def plot_coverage_maps(coverage_map, original_uv_fits_path,
original_cc_fits_path, data_dir, outname,
path_to_script):
i_image_cc = create_clean_image_from_fits_file(original_cc_fits_path)
i_image = create_image_from_fits_file(original_cc_fits_path)
rms = rms_image_shifted(original_uv_fits_path,
tmp_dir=data_dir,
image_fits=original_cc_fits_path,
path_to_script=path_to_script)
blc, trc = find_bbox(i_image.image,
2 * rms,
delta=int(i_image_cc._beam.bmaj / 2))
fig = iplot(i_image.image,
coverage_map - 0.68,
x=i_image.x,
y=i_image.y,
min_abs_level=2. * rms,
outdir=data_dir,
beam=i_image_cc.beam,
show=True,
beam_place='lr',
show_beam=True,
cmap='viridis',
blc=blc,
trc=trc,
color_clim=[-0.3, 0.3],
colors_mask=i_image.image < 2 * rms)
fig.savefig(os.path.join(data_dir, '{}.pdf'.format(outname)),
bbox_inches='tight',
format='pdf',
dpi=600)
for i, boot_uv in enumerate(sorted(booted_uv_fits)):
clean_difmap(boot_uv,
'booted_cc_{}.fits'.format(i),
'I', (512, 0.1),
path=data_dir,
path_to_script=path_to_script,
outpath=data_dir)
booted_im_fits = glob.glob(os.path.join(data_dir, 'booted_cc*.fits'))
images = Images()
images.add_from_fits(booted_im_fits)
error_image = images.create_error_image()
orig_image = create_clean_image_from_fits_file(os.path.join(data_dir, im_fits))
rms = rms_image_shifted(os.path.join(data_dir, uv_fits),
image_fits=os.path.join(data_dir, im_fits),
path_to_script=path_to_script)
blc, trc = find_bbox(orig_image.image, level=1.5 * rms)
image_mask = orig_image.image < 2 * rms
label_size = 14
matplotlib.rcParams['xtick.labelsize'] = label_size
matplotlib.rcParams['ytick.labelsize'] = label_size
matplotlib.rcParams['axes.titlesize'] = label_size
matplotlib.rcParams['axes.labelsize'] = label_size
matplotlib.rcParams['font.size'] = label_size
matplotlib.rcParams['legend.fontsize'] = label_size
# 3c273
# fig = iplot(orig_image.image, error_image.image/rms, x=orig_image.x,
# y=orig_image.y, show_beam=True, min_abs_level=2*rms, cmap='hsv',
# beam=orig_image.beam, colors_mask=image_mask,
# beam_face_color='black', blc=(495, 385), trc=(680, 540))
def create_coverage_map_classic(original_uv_fits_path,
ci_type,
original_cc_fits_path=None,
imsize=None,
outdir=None,
n_boot=200,
path_to_script=None,
alpha=0.68,
n_cov=100,
n_rms=1.,
stokes='I',
sample_cc_fits_paths=None,
sample_uv_fits_paths=None):
"""
Conduct coverage analysis of image pixels flux CI. Find number of times
when CI of `sample` values contains `true` value.
:param original_uv_fits_path:
Path to original FITS-file with uv-data.
:param ci_type:
Type of CI to test. ``boot`` or ``rms``. If ``boot`` then use residuals
bootstrap CI. If ``rms`` then use Hovatta corrected image rms CI.
:param original_cc_fits_path: (optional)
Path to original FITS-file with CC model. If ``None`` then use
``imsize`` parameter to get `original` CC model from
``original_uv_fits_path``. (default: ``None``)
:param imsize: (optional)
Image parameters (image size [pix], pixel size [mas]) to use
when doing first CC with ``original_cc_fits_path = None``. (default:
``None``)
:param outdir: (optional)
Directory to store intermediate results. If ``None`` then use CWD.
(default: ``None``)
:param n_boot: (optional)
Number of bootstrap replications to use when calculating bootstrap CI
for ``ci_type = boot`` option when ``boot_cc_fits_paths`` hasn't
specified. (default: ``200``)
:param path_to_script: (optional)
Path to Dan Homan's script for final clean. If ``None`` then use CWD.
(default: ``None``)
:param alpha: (optional)
Level of significance when calculating bootstrap CI for ``ci_type =
boot`` case. E.g. ``0.68`` corresponds to `1 \sigma`. (default:
``0.68``)
:param n_cov: (optional)
Number of `samples` from infinite population to consider in coverage
analysis of intervals. Here `samples` - observations of known source
with different realisations of noise with known parameters. (default:
``100``)
:param n_rms: (optional)
Number of rms to use in ``ci_type = rms`` case. (default: ``1.``)
:param stokes: (optional)
Stokes parameter to use. If ``None`` then use ``I``. (default: ``None``)
:param boot_cc_fits_paths: (optional)
If ``ci_type = boot`` then this parameter could specify paths to cleaned
bootstrapped uv-data.
:param sample_uv_fits_paths: (optional)
Path to FITS-files with `sample` uv-data. If ``None`` then create
``n_cov`` `sample` uv-data from noise of `original` uv-data and
`original` CLEAN model. (default: ``None``)
:param sample_cc_fits_paths: (optional)
Path to FITS-files with CLEAN models of `sample` uv-data. If ``None``
then create ``n_cov`` `sample` uv-data from noise of `original` uv-data
and `original` CLEAN model. (default: ``None``)
:return:
Coverage map. Each pixel contain frequency of times when CI for samples
from population contain `true` value for given pixel.
"""
if original_cc_fits_path is None:
print(
"No `original` CLEAN model specified! Will CLEAN `original`"
" uv-data.")
if imsize is None:
raise Exception("Specify ``imsize``")
uv_fits_dir, uv_fits_fname = os.path.split(original_uv_fits_path)
print("Cleaning `original` uv-data to"
" {}".format(os.path.join(outdir, 'original_cc.fits')))
clean_difmap(uv_fits_fname,
'original_cc.fits',
stokes,
imsize,
path=uv_fits_dir,
path_to_script=path_to_script,
outpath=outdir)
original_cc_fits_path = os.path.join(outdir, 'original_cc.fits')
# Find images parameters for cleaning if necessary
if imsize is None:
print(
"Getting image parameters from `original`"
" CLEAN FITS file {}.".format(original_cc_fits_path))
image_params = get_fits_image_info(original_cc_fits_path)
imsize = (image_params['imsize'][0],
abs(image_params['pixsize'][0]) / mas_to_rad)
# This is `true` values. Will check how often they arise in `sample` CIs.
original_image = create_image_from_fits_file(original_cc_fits_path)
# If `sample` data doesn't ready - create it!
if sample_uv_fits_paths is None:
# Create `sample`
sample_uv_fits_paths, sample_cc_fits_paths =\
create_sample(original_uv_fits_path,
original_cc_fits_path=original_cc_fits_path,
imsize=imsize, outdir=outdir,
path_to_script=path_to_script, n_sample=n_cov,
stokes=stokes)
# For each pixel check how often CI of `sample` images contains `model`]
# values.
print("Creating coverage array")
cov_array = np.zeros((imsize[0], imsize[0]), dtype=float)
# Testing coverage of bootstrapped CI
# For each `sample` uv-data and model generate bootstrap CI
print sample_uv_fits_paths
print sample_cc_fits_paths
for sample_cc_fits_path, sample_uv_fits_path in zip(
sample_cc_fits_paths, sample_uv_fits_paths):
print("Sample : {}".format(sample_cc_fits_path))
if ci_type == 'boot':
n__ = sample_uv_fits_path.split('.')[0].split('_')[-1]
n_ = sample_cc_fits_path.split('.')[0].split('_')[-1]
assert n_ == n__
print(
"Bootstrapping sample uv-data {} with sample model {} using"
" {} replications".format(sample_uv_fits_path,
sample_cc_fits_path, n_boot))
print("Removing old bootstrapped files...")
boot_cc_fits_paths = glob.glob(
os.path.join(outdir, 'sample_cc_boot_*.fits'))
for rmfile in boot_cc_fits_paths:
print("Removing CC file {}".format(rmfile))
os.unlink(rmfile)
boot_uv_fits_paths = \
sorted(glob.glob(os.path.join(outdir, 'sample_uv_boot_*.uvf')))
for rmfile in boot_uv_fits_paths:
print("Removing UV file {}".format(rmfile))
os.unlink(rmfile)
bootstrap_uv_fits(sample_uv_fits_path, [sample_cc_fits_path],
n_boot,
outpath=outdir,
outname=['sample_uv_boot', '.uvf'])
boot_uv_fits_paths =\
sorted(glob.glob(os.path.join(outdir, 'sample_uv_boot_*.uvf')))
# Clean each bootstrapped uv-data
for i, uv_fits_path in enumerate(boot_uv_fits_paths):
uv_fits_dir, uv_fits_fname = os.path.split(uv_fits_path)
j = uv_fits_fname.split('.')[0].split('_')[-1]
cc_fname = 'sample_cc_boot_{}.fits'.format(j)
print("Cleaning {} bootstrapped sample"
" uv-data to {}".format(uv_fits_path,
os.path.join(outdir, cc_fname)))
clean_difmap(uv_fits_fname,
cc_fname,
stokes,
imsize,
path=uv_fits_dir,
path_to_script=path_to_script,
outpath=outdir)
boot_cc_fits_paths = sorted(
glob.glob(os.path.join(outdir, 'sample_cc_boot_*.fits')))
# Calculate bootstrap CI for current `sample`
# hdi_low, hdi_high = boot_ci_bc(boot_cc_fits_paths,
# observed_cc_fits_path, alpha=alpha)
hdi_low, hdi_high = boot_ci(boot_cc_fits_paths,
sample_cc_fits_path,
alpha=alpha)
elif ci_type == 'rms':
# Calculate ``n_rms`` CI
print("Calculating rms...")
sample_image = create_image_from_fits_file(sample_cc_fits_path)
rms = rms_image_shifted(sample_uv_fits_path,
image_fits=sample_cc_fits_path,
path_to_script=path_to_script)
# rms = rms_image(sample_image)
hdi_low = sample_image.image - n_rms * rms
hdi_high = sample_image.image + n_rms * rms
else:
raise Exception("CI intervals must be `boot` or `rms`!")
# Check if `model` value falls in current `sample` CI
print("Calculating hits of `true` values for"
" {}".format(sample_cc_fits_path))
for (x, y), value in np.ndenumerate(cov_array):
cov_array[x, y] += float(
np.logical_and(hdi_low[x, y] < original_image.image[x, y],
original_image.image[x, y] < hdi_high[x, y]))
return cov_array / n_cov
create_coverage_map_classic(original_uv_fits_path, ci_type=ci_type,
original_cc_fits_path=original_cc_fits_path,
imsize=imsize, outdir=data_dir,
path_to_script=path_to_script,
sample_cc_fits_paths=sample_cc_fits_paths,
sample_uv_fits_paths=sample_uv_fits_paths,
n_rms=n_rms, alpha=alpha, n_boot=n_boot,
n_cov=n_cov, stokes=stokes)
np.savetxt(os.path.join(data_dir, "{}.txt".format(outfile)),
coverage_map)
original_cc_fits_path = os.path.join(data_dir, 'original_cc.fits')
i_image_cc = create_clean_image_from_fits_file(original_cc_fits_path)
i_image = create_image_from_fits_file(original_cc_fits_path)
rms = rms_image_shifted(original_uv_fits_path,
tmp_dir=data_dir,
image_fits=original_cc_fits_path,
path_to_script=path_to_script)
blc, trc = find_bbox(i_image.image,
2 * rms,
delta=int(i_image_cc._beam.bmaj / 2))
# iplot(i_image.image, coverage_map, x=i_image.x, y=i_image.y,
# min_abs_level=2. * rms, outfile=outfile, outdir=data_dir,
# beam=i_image_cc.beam, show=True, beam_corner='lr',
# show_beam=True, cmap='viridis', blc=blc, trc=trc)
fig = iplot(i_image.image,
coverage_map - 0.68,
x=i_image.x,
y=i_image.y,
min_abs_level=2. * rms,
outfile=outfile,
outdir=data_dir,
def process_mf(uvdata_dict,
beam,
data_dir,
path_to_script,
clean_after=True,
rms_cs_dict=None,
mapsize_clean=(512, 0.1)):
images_dict = dict()
print(" === CLEANing each band and Stokes ===")
clean_original_data(uvdata_dict,
data_dir,
beam,
mapsize_clean=mapsize_clean)
for band in bands:
images_dict[band] = dict()
for stokes in ("I", "Q", "U"):
ccimage = create_clean_image_from_fits_file(
os.path.join(data_dir, "cc_{}_{}.fits".format(band, stokes)))
images_dict[band].update({stokes: ccimage})
# Cacluate RMS for each band and Stokes
print(" === Cacluate RMS for each band and Stokes ===")
if rms_cs_dict is None:
rms_cs_dict = dict()
for band in bands:
rms_cs_dict[band] = {
stokes: rms_image_shifted(os.path.join(data_dir,
uvdata_dict[band]),
stokes=stokes,
image=images_dict[band][stokes],
path_to_script=path_to_script)
for stokes in ("I", "Q", "U")
}
for band in bands:
for stokes in ("I", "Q", "U"):
print("rms = {}".format(rms_cs_dict[band][stokes]))
# Find mask for "I" for each band and combine them into single mask
print("Calculating masks for I at each band and combining them")
spix_mask_image = spix_mask(
{band: images_dict[band]["I"]
for band in bands}, {band: rms_cs_dict[band]["I"]
for band in bands},
n_sigma=3,
path_to_script=path_to_script)
# Find mask for "PPOL" for each band and combine them into single mask
print("Calculating masks for PPOL at each band and combining them")
ppol_mask_image = dict()
for band in bands:
ppol_mask_image[band] = pol_mask(
{stokes: images_dict[band][stokes]
for stokes in ("I", "Q", "U")},
{stokes: rms_cs_dict[band][stokes]
for stokes in ("I", "Q", "U")},
n_sigma=2,
path_to_script=path_to_script)
ppol_mask_image = np.logical_or.reduce(
[ppol_mask_image[band] for band in bands])
spix_image, sigma_spix_image, chisq_spix_image =\
spix_map(freqs, [images_dict[band]["I"].image for band in bands],
mask=spix_mask_image)
print("Calculating PANG and it's error for each band")
pang_images = dict()
sigma_pang_images = dict()
for band in bands:
pang_images[band] = pang_map(images_dict[band]["Q"].image,
images_dict[band]["U"].image,
mask=ppol_mask_image)
sigma_pang_images[band] = np.hypot(
images_dict[band]["Q"].image * 1.8 * rms_cs_dict[band]["U"],
images_dict[band]["U"].image * 1.8 * rms_cs_dict[band]["Q"])
sigma_pang_images[band] = sigma_pang_images[band] / (
2. * (images_dict[band]["Q"].image**2. +
images_dict[band]["U"].image**2.))
print("Calculating ROTM image")
rotm_image, sigma_rotm_image, chisq_rotm_image = rotm_map(
freqs, [pang_images[band] for band in bands],
[sigma_pang_images[band] for band in bands],
mask=ppol_mask_image)
if clean_after:
print("Removing maps")
for band in bands:
for stokes in ("I", "Q", "U"):
os.unlink(
os.path.join(data_dir,
"cc_{}_{}.fits".format(band, stokes)))
result = {
"ROTM": {
"value": rotm_image,
"sigma": sigma_rotm_image,
"chisq": chisq_rotm_image
},
"SPIX": {
"value": spix_image,
"sigma": sigma_spix_image,
"chisq": chisq_spix_image
},
"RMS": rms_cs_dict
}
return result
def analyze_rotm_conv(self, sigma_evpa=None, sigma_d_term=None,
rotm_slices=None, colors_clim=None, n_sigma=None,
pxls_plot=None, plot_points=None, slice_ylim=None):
print("Estimate RM map and it's error using conventional method...")
if sigma_evpa is None:
sigma_evpa = self.sigma_evpa
if sigma_d_term is None:
sigma_d_term = self.sigma_d_term
if rotm_slices is None:
rotm_slices = self.rotm_slices
if n_sigma is not None:
self.set_common_mask(n_sigma)
# Find EVPA error for each frequency
print("Calculating maps of PANG errors for each band using Hovatta's"
" approach...")
# Fetch common size `I` map on highest frequency for plotting PANG error
# maps
i_image = self.cc_cs_image_dict[self.freqs[-1]]['I']
if pxls_plot is not None:
pxls_plot = [i_image._convert_coordinate(pxl) for pxl in pxls_plot]
rms = rms_image(i_image)
blc, trc = find_bbox(i_image.image, 2.*rms,
delta=int(i_image._beam.beam[0]))
for i, freq in enumerate(self.freqs):
n_ant = len(self.uvdata_dict[freq].antennas)
n_if = self.uvdata_dict[freq].nif
d_term = sigma_d_term[i]
s_evpa = sigma_evpa[i]
# Get number of scans
if self.n_scans is not None:
try:
# If `NX` table is present
n_scans = len(self.uvdata_dict[freq].scans)
except TypeError:
scans_dict = self.uvdata_dict[freq].scans_bl
scan_lengths = list()
for scans in scans_dict.values():
if scans is not None:
scan_lengths.append(len(scans))
n_scans = mode(scan_lengths)[0][0]
else:
n_scans = self.n_scans[i]
q = self.cc_cs_image_dict[freq]['Q']
u = self.cc_cs_image_dict[freq]['U']
i = self.cc_cs_image_dict[freq]['I']
# We need ``s_evpa = 0`` for testing RM gradient significance but
# ``s_evpa != 0`` for calculating RM errors
pang_std, ppol_std = hovatta_find_sigma_pang(q, u, i, s_evpa,
d_term, n_ant, n_if,
n_scans)
self.evpa_sigma_dict[freq] = pang_std
fig = iplot(i_image.image, pang_std, x=i_image.x, y=i_image.y,
min_abs_level=3. * rms, colors_mask=self._cs_mask,
color_clim=[0, 1], blc=blc, trc=trc,
beam=self.common_beam, colorbar_label='sigma EVPA, [rad]',
show_beam=True, show=False)
self.figures['EVPA_sigma_{}'.format(freq)] = fig
sigma_pang_arrays = [self.evpa_sigma_dict[freq] for freq in self.freqs]
sigma_pang_arrays_grad = [np.sqrt(self.evpa_sigma_dict[freq]**2 -
np.deg2rad(self.sigma_evpa[i])**2) for
i, freq in enumerate(self.freqs)]
rotm_image, sigma_rotm_image, chisq_image =\
self.original_cs_images.create_rotm_image(sigma_pang_arrays,
mask=self._cs_mask,
return_chisq=True,
plot_pxls=pxls_plot,
outdir=self.data_dir,
mask_on_chisq=False)
rotm_image_grad, sigma_rotm_image_grad, _ =\
self.original_cs_images.create_rotm_image(sigma_pang_arrays_grad,
mask=self._cs_mask,
return_chisq=True,
plot_pxls=pxls_plot,
outdir=self.data_dir,
mask_on_chisq=False)
self._rotm_image_conv = rotm_image
self._rotm_image_conv_grad = rotm_image_grad
self._rotm_image_sigma_conv = sigma_rotm_image
self._rotm_image_sigma_conv_grad = sigma_rotm_image_grad
self._rotm_chisq_image_conv = chisq_image
i_image = self.cc_cs_image_dict[self.freqs[-1]]['I']
uv_fits_path = self.uvfits_dict[self.freqs[-1]]
image_fits_path = self.cc_cs_fits_dict[self.freqs[-1]]['I']
# RMS using Hovatta-style
rms = rms_image_shifted(uv_fits_path, image_fits=image_fits_path,
path_to_script=self.path_to_script)
blc, trc = find_bbox(i_image.image, 2.*rms,
delta=int(i_image._beam.beam[0]))
fig = iplot(i_image.image, rotm_image.image, x=i_image.x, y=i_image.y,
min_abs_level=3. * rms, colors_mask=self._cs_mask,
color_clim=colors_clim, blc=blc, trc=trc,
beam=self.common_beam, slice_points=rotm_slices,
show_beam=True, show=False, show_points=plot_points,
cmap='viridis')
self.figures['rotm_image_conv'] = fig
fig = iplot(i_image.image, sigma_rotm_image.image, x=i_image.x,
y=i_image.y, min_abs_level=3. * rms,
colors_mask=self._cs_mask, color_clim=[0, 200], blc=blc,
trc=trc, beam=self.common_beam, slice_points=rotm_slices,
show_beam=True, show=False, cmap='viridis', beam_place='ul')
self.figures['rotm_image_conv_sigma'] = fig
fig = iplot(i_image.image, sigma_rotm_image_grad.image, x=i_image.x,
y=i_image.y, min_abs_level=3. * rms,
colors_mask=self._cs_mask, color_clim=[0, 200], blc=blc,
trc=trc, beam=self.common_beam, slice_points=rotm_slices,
show_beam=True, show=False, cmap='viridis', beam_place='ul')
self.figures['rotm_image_conv_sigma_grad'] = fig
fig = iplot(i_image.image, chisq_image.image, x=i_image.x, y=i_image.y,
min_abs_level=3. * rms, colors_mask=self._cs_mask,
outfile='rotm_chisq_image_conv', outdir=self.data_dir,
color_clim=[0., self._chisq_crit], blc=blc, trc=trc,
beam=self.common_beam,
colorbar_label='Chi-squared', slice_points=rotm_slices,
show_beam=True, show=False, cmap='viridis')
self.figures['rotm_chisq_image_conv'] = fig
if rotm_slices is not None:
self.figures['slices_conv'] = dict()
for rotm_slice in rotm_slices:
rotm_slice_ = i_image._convert_coordinates(rotm_slice[0],
rotm_slice[1])
# Here we use RM image error calculated using PANG errors
# without contribution of the EVPA calibration errors.
fig = analyze_rotm_slice(rotm_slice_, rotm_image,
sigma_rotm_image=sigma_rotm_image_grad,
outdir=self.data_dir,
beam_width=int(i_image._beam.beam[0]),
outfname="ROTM_{}_slice".format(rotm_slice),
ylim=slice_ylim)
self.figures['slices_conv'][str(rotm_slice)] = fig
return rotm_image, sigma_rotm_image