def clean_boot_common(self):
print("Clean bootstrap replications with common "
"restoring beams and map sizes...")
for freq in self.freqs:
self.cc_boot_cs_fits_dict.update({freq: dict()})
uv_fits_paths = self.uvfits_boot_dict[freq]
for stokes in self.stokes:
for i, uv_fits_path in enumerate(uv_fits_paths):
uv_dir, uv_fname = os.path.split(uv_fits_path)
outfname = 'cs_boot_{}_{}_cc_{}.fits'.format(freq, stokes,
str(i + 1).zfill(3))
# Check if it is already done
if not os.path.exists(os.path.join(self.data_dir,
outfname)):
clean_difmap(uv_fname, outfname, stokes,
self.common_imsize, path=uv_dir,
path_to_script=self.path_to_script,
beam_restore=self.common_beam,
outpath=self.data_dir,
command_file=self._difmap_commands_file)
else:
print("Found CLEAN model in file {}".format(outfname))
files = sorted(glob.glob(os.path.join(self.data_dir,
'cs_boot_{}_{}_cc_*.fits'.format(freq, stokes))))
self.cc_boot_cs_fits_dict[freq].update({stokes: files})
if self.clear_difmap_logs:
print("Removing difmap log-files...")
difmap_logs = glob.glob(os.path.join(self.data_dir, "difmap.log*"))
for difmap_log in difmap_logs:
os.unlink(difmap_log)
def clean_original_data(uvdata_dict,
data_dir,
beam=None,
plot=False,
mapsize_clean=(512, 0.1),
outfname_postfix=None):
if not isinstance(mapsize_clean, dict):
assert len(mapsize_clean) == 2
mapsize_clean = {band: mapsize_clean for band in bands}
for band in bands:
print("Band - {}".format(band))
for stokes in ('I', 'Q', 'U'):
print("Stokes - {}".format(stokes))
if outfname_postfix is None:
outfname = "cc_{}_{}.fits".format(band, stokes)
else:
outfname = "cc_{}_{}_{}.fits".format(band, stokes,
outfname_postfix)
print("Cleaning {} to {}".format(uvdata_dict[band], outfname))
clean_difmap(fname=uvdata_dict[band],
outfname=outfname,
stokes=stokes.lower(),
path=data_dir,
outpath=data_dir,
mapsize_clean=mapsize_clean[band],
path_to_script=path_to_script,
show_difmap_output=False,
beam_restore=beam)
# Rarely need this one
if plot:
if outfname_postfix is None:
outfname = "cc_{}_{}.fits".format(band, "I")
else:
outfname = "cc_{}_{}_{}.fits".format(band, "I",
outfname_postfix)
ccimage = create_clean_image_from_fits_file(
os.path.join(data_dir, outfname))
beam = ccimage.beam
rms = rms_image(ccimage)
blc, trc = find_bbox(ccimage.image, 1.0 * rms, 10)
fig = iplot(ccimage.image,
x=ccimage.x,
y=ccimage.y,
min_abs_level=2.0 * rms,
beam=beam,
show_beam=True,
blc=blc,
trc=trc,
close=False,
colorbar_label="Jy/beam",
show=True)
if outfname_postfix is None:
outfname = "cc_{}.png".format(band)
else:
outfname = "cc_{}_{}.png".format(band, outfname_postfix)
fig.savefig(os.path.join(data_dir, outfname))
def clean_original_native(self, freq_stokes_dict=None):
"""
Clean original FITS-files with uv-data using native resolution.
"""
for freq in self.freqs:
self.cc_image_dict.update({freq: dict()})
self.cc_fits_dict.update({freq: dict()})
self.cc_beam_dict.update({freq: dict()})
if freq_stokes_dict is not None:
print("Found CLEANed images of original uvdata with naitive map &"
" beam parameters...")
for freq in self.freqs:
for stokes in self.stokes:
image = create_clean_image_from_fits_file(
freq_stokes_dict[freq][stokes])
self.cc_image_dict[freq].update({stokes: image})
if stokes == 'I':
self.cc_beam_dict.update({freq: image.beam})
else:
print(
"Clean original uv-data with native map & beam parameters...")
for freq in self.freqs:
print("Cleaning frequency {} with image "
"parameters {}".format(freq, self.imsizes_dict[freq]))
uv_fits_path = self.uvfits_dict[freq]
uv_dir, uv_fname = os.path.split(uv_fits_path)
for stokes in self.stokes:
outfname = '{}_{}_cc.fits'.format(freq, stokes)
outpath = os.path.join(self.data_dir, outfname)
# Check if it is already done
if not os.path.exists(outpath):
clean_difmap(uv_fname,
outfname,
stokes,
self.imsizes_dict[freq],
path=uv_dir,
path_to_script=self.path_to_script,
outpath=self.data_dir,
command_file=self._difmap_commands_file)
else:
print("Found CLEAN model in file {}".format(outfname))
self.cc_fits_dict[freq].update(
{stokes: os.path.join(self.data_dir, outfname)})
image = create_clean_image_from_fits_file(outpath)
self.cc_image_dict[freq].update({stokes: image})
if stokes == 'I':
self.cc_beam_dict.update({freq: image.beam})
if self.clear_difmap_logs:
print("Removing difmap log-files...")
difmap_logs = glob.glob(os.path.join(self.data_dir, "difmap.log*"))
for difmap_log in difmap_logs:
os.unlink(difmap_log)
def rms_image_shifted(uv_fits_path,
hovatta_factor=True,
shift=(1000, 1000),
tmp_name='shifted_clean_map.fits',
tmp_dir=None,
stokes='I',
image=None,
image_fits=None,
mapsize_clean=None,
path_to_script=None,
niter=None):
"""
Estimate image per-pixel rms using shifted image.
"""
path, uv_fits_fname = os.path.split(uv_fits_path)
if tmp_dir is None:
tmp_dir = os.getcwd()
if path_to_script is None:
raise Exception("Provide location of difmap final CLEAN script!")
if mapsize_clean is None and image is not None:
import utils
pixsize = abs(image.pixsize[0]) / utils.mas_to_rad
mapsize_clean = (image.imsize[0], pixsize)
if mapsize_clean is None and image_fits is not None:
import from_fits
image = from_fits.create_image_from_fits_file(image_fits)
import utils
pixsize = abs(image.pixsize[0]) / utils.mas_to_rad
mapsize_clean = (image.imsize[0], pixsize)
import spydiff
if niter is None:
spydiff.clean_difmap(uv_fits_fname,
tmp_name,
stokes=stokes,
mapsize_clean=mapsize_clean,
path=path,
path_to_script=path_to_script,
outpath=tmp_dir,
shift=shift)
else:
spydiff.clean_n(uv_fits_path,
tmp_name,
stokes=stokes,
mapsize_clean=mapsize_clean,
niter=niter,
path_to_script=path_to_script,
outpath=tmp_dir,
shift=shift)
import from_fits
image = from_fits.create_image_from_fits_file(
os.path.join(tmp_dir, tmp_name))
return rms_image(image, hovatta_factor=hovatta_factor)
def clean_original_common(self, freq_stokes_dict=None):
# if freq_stokes_dict is not None:
# print("Found CLEANed images of original uvdata with common map &"
# " beam parameters...")
# for freq in self.freqs:
# self.cc_cs_image_dict.update({freq: dict()})
# self.cc_cs_fits_dict.update({freq: dict()})
# for stokes in self.stokes:
# image = create_clean_image_from_fits_file(freq_stokes_dict[freq][stokes])
# self.cc_image_dict[freq].update({stokes: image})
# if stokes == 'I':
# self.cc_beam_dict.update({freq: image.beam})
print("Clean original uv-data with common map parameters "
" {} and beam {}".format(self.common_imsize, self.common_beam))
for freq in self.freqs:
self.cc_cs_image_dict.update({freq: dict()})
self.cc_cs_fits_dict.update({freq: dict()})
uv_fits_path = self.uvfits_dict[freq]
uv_dir, uv_fname = os.path.split(uv_fits_path)
for stokes in self.stokes:
outfname = 'cs_{}_{}_cc.fits'.format(freq, stokes)
outpath = os.path.join(self.data_dir, outfname)
# Check if it is already done
if not os.path.exists(outpath):
clean_difmap(uv_fname,
outfname,
stokes,
self.common_imsize,
path=uv_dir,
path_to_script=self.path_to_script,
beam_restore=self.common_beam,
outpath=self.data_dir,
command_file=self._difmap_commands_file)
else:
print("Found CLEAN model in file {}".format(outfname))
self.cc_cs_fits_dict[freq].update(
{stokes: os.path.join(self.data_dir, outfname)})
image = create_clean_image_from_fits_file(outpath)
self.cc_cs_image_dict[freq].update({stokes: image})
if self.clear_difmap_logs:
print("Removing difmap log-files...")
difmap_logs = glob.glob(os.path.join(self.data_dir, "difmap.log*"))
for difmap_log in difmap_logs:
os.unlink(difmap_log)
def abc_simulations(param,
z,
freqs,
uv_fits_templates,
cc_images,
pickle_history,
out_dir=None):
"""
Simulation function for ABC. Simulates data given parameters (``b, n, los``)
and returns the vector of the summary statistics.
:param b:
Value of the magnetic field at 1 pc [G].
:param n:
Value of particle density at 1 pc [cm^(-3)].
:param los:
Jet angle to the line of site [rad].
:param z:
Redshift of the source.
:param freqs:
Iterable of frequencies.
:param uv_fits_templates:
Iterable of paths to FITS files with self-calibrated uv-data. In order
of ``freqs``.
:param cc_images:
Iterable of paths to FITS files with CLEAN maps. In order of ``freqs``.
:param out_dir: (optional)
Directory to store files. If ``None`` then use CWD. (default: ``None``)
:param pickle_history:
Path to pickle file with dictionary of simulations history.
:return:
Summary statistics. E.g. "observed" core flux at highest frequency,
"observed" core size at highest frequency, "observed" ellipticity of the
core at highest frequency, core shift between frequencies (distance
between "core shift - frequency" curves).
"""
if out_dir is None:
out_dir = os.getcwd()
else:
# FIXME: Create directory if it doesn't exist
if not os.path.exists(out_dir):
pass
result_dict = dict()
b, n, los = np.exp(param)
p = Parameters(b, n, los)
with open(pickle_history, 'r') as fo:
history = pickle.load(fo)
print("Running simulations with b={}, n={}, los={}".format(b, n, los))
for freq, uv_fits_template, cc_image in zip(freqs, uv_fits_templates,
cc_images):
# Cleaning old results if any
simulated_maps_old = glob.glob(os.path.join(exe_dir, "map*.txt"))
for to_remove in simulated_maps_old:
os.unlink(to_remove)
# Checking if simulations on highest frequency are done. If so then use
# scaled image parameters
if freqs[0] in history[p].keys():
pixel_size_mas = history[p][
freqs[0]]["pixel_size_mas"] * freqs[0] / freq
number_of_pixels = history[p][
freqs[0]]["number_of_pixels"] * freq / freqs[0]
map_size = (number_of_pixels, pixel_size_mas)
print("Using scaled image parameters: {}, {}".format(
number_of_pixels, pixel_size_mas))
else:
pixel_size_mas = 0.01
number_of_pixels = 400
map_size = None
update_dict = {
"jet": {
"bfield": {
"parameters": {
"b_1": b
}
},
"nfield": {
"parameters": {
"n_1": n
}
}
},
"observation": {
"los_angle": los,
"frequency_ghz": freq,
"redshift": z
},
"image": {
"pixel_size_mas": pixel_size_mas,
"number_of_pixels": number_of_pixels
}
}
update_config(cfg_file, update_dict)
# FIXME: Handle ``FailedFindBestImageParamsException`` during ABC run
simulation_params = run_simulations(cfg_file,
path_to_executable,
map_size=map_size)
# Find total flux on simulated image
image = os.path.join(exe_dir, "map_i.txt")
image = np.loadtxt(image)
# Rare case of strange fluxes
image[image < 0] = 0
image[image > 10.0] = 0
# Total model flux at current frequency
total_flux = image.sum()
# Maximum model pixel flux at current frequency
max_flux = image.max()
cc_image = create_clean_image_from_fits_file(cc_image)
noise_factor = 1.0
initial_dfm_model = os.path.join(main_dir, 'initial_cg.mdl')
out_dfm_model_fn = "bk_{}.mdl".format(freq)
uv_fits_save_fname = "bk_{}.fits".format(freq)
modelfit_simulation_result(exe_dir,
initial_dfm_model,
noise_factor=noise_factor,
out_dfm_model_fn=out_dfm_model_fn,
out_dir=out_dir,
params=simulation_params,
uv_fits_save_fname=uv_fits_save_fname,
uv_fits_template=uv_fits_template)
# Find measured and true distance of core to jet component
dr_obs = find_core_separation_from_jet_using_difmap_model(
os.path.join(out_dir, out_dfm_model_fn))
dr_true = find_core_separation_from_center_using_simulations(
os.path.join(exe_dir, "map_i.txt"), simulation_params)
# This means something wrong with jetshow
if total_flux < 0:
print("b = {}, n = {}, los = {}".format(b, n, los))
open(
os.path.join(
out_dir,
"total_disaster_{}_{}_{}_{}.txt".format(b, n, los, freq)),
'a').close()
raise TotalDisasterException
# Plot map with components superimposed
cc_fits_save_fname = "bk_cc_{}.fits".format(freq)
# For 18 cm we need large pixel size
cellsize = 0.1
if freq == 1.665:
cellsize = 0.5
elif freq == 8.1:
cellsize = 0.2
clean_difmap(uv_fits_save_fname,
cc_fits_save_fname,
'I', (1024, cellsize),
path=out_dir,
path_to_script=path_to_script,
show_difmap_output=False,
outpath=out_dir)
ccimage = create_clean_image_from_fits_file(
os.path.join(out_dir, cc_fits_save_fname))
beam = ccimage.beam
rms = rms_image(ccimage)
blc, trc = find_bbox(ccimage.image, rms, 10)
comps = import_difmap_model(out_dfm_model_fn, out_dir)
plot_fitted_model(os.path.join(out_dir, uv_fits_save_fname),
comps,
savefig=os.path.join(
out_dir,
"difmap_model_uvplot_{}.png".format(freq)))
fig = iplot(ccimage.image,
x=ccimage.x,
y=ccimage.y,
min_abs_level=3 * rms,
beam=beam,
show_beam=True,
blc=blc,
trc=trc,
components=comps,
close=True,
colorbar_label="Jy/beam")
fig.savefig(os.path.join(out_dir, "cc_{}.png".format(freq)))
# Move simulated images to data directory
cut_image(os.path.join(exe_dir, "map_i.txt"))
cut_image(os.path.join(exe_dir, "map_l.txt"))
cut_image(os.path.join(exe_dir, "map_q.txt"))
cut_image(os.path.join(exe_dir, "map_u.txt"))
cut_image(os.path.join(exe_dir, "map_v.txt"))
cut_image(os.path.join(exe_dir, "map_tau.txt"))
for name in ('i', 'q', 'u', 'v', 'tau', 'l'):
shutil.move(
os.path.join(exe_dir, "map_{}.txt".format(name)),
os.path.join(out_dir, "map_{}_{}.txt".format(name, freq)))
# Calculate some info
dr_pc = distance_from_SMBH(dr_true, los, z=z)
b_core = b_field(b, dr_pc)
t_syn_years = t_syn(b_core, freq) / (np.pi * 10**7)
result_dict[freq] = dict()
to_results = {
"dr_obs":
dr_obs,
"dr_true":
dr_true,
"flux":
total_flux,
"flux_obs":
comps[0].p[0],
"bmaj_obs":
comps[0].p[3],
"tb_difmap":
np.log10(tb_comp(comps[0].p[0], comps[0].p[3], freq, z=z)),
"tb_pix":
np.log10(
tb(max_flux,
freq,
simulation_params[u'image'][u'pixel_size_mas'],
z=z)),
"b_core":
b_core,
"dr_core_pc":
dr_pc,
"t_syn_core":
t_syn_years,
"pixel_size_mas":
simulation_params[u'image'][u'pixel_size_mas'],
"number_of_pixels":
simulation_params[u'image'][u'number_of_pixels']
}
result_dict[freq] = to_results
history[p] = result_dict
with open(pickle_history, 'w') as fo:
pickle.dump(history, fo)
return create_summary_from_result_dict(result_dict, (0.3, 0.2, 0.1))
source = '2230+114'
epoch = '2005-02-05'
uv_fits = mojave_uv_fits_fname(source, 'u', epoch.replace('-', '_'))
path_to_script = '/home/ilya/github/vlbi_errors/difmap/final_clean_nw'
mdl_fname = '{}_{}.mdl'.format(source, epoch)
# Fetch uv-fits
download_mojave_uv_fits(source, [epoch.replace('-', '_')],
bands=['u'],
download_dir=data_dir)
# Fetch model file
get_mojave_mdl_file(tsv_table, source, epoch, outdir=data_dir)
# Clean uv-fits
clean_difmap(uv_fits,
'cc.fits',
'I', [1024, 0.1],
path=data_dir,
path_to_script=path_to_script,
outpath=data_dir)
# Create clean image instance
cc_image = create_clean_image_from_fits_file(os.path.join(data_dir, 'cc.fits'))
comps = import_difmap_model(mdl_fname, data_dir)
model = Model(stokes='I')
model.add_components(*comps)
# Check that model fits UV-data well
uv_data = UVData(os.path.join(data_dir, uv_fits))
uv_data.uvplot()
mdl_data = copy.deepcopy(uv_data)
mdl_data.substitute([model])
mdl_data.uvplot(sym='.r')
def create_coverage_map_classic(original_uv_fits_path,
ci_type,
original_cc_fits_path=None,
imsize=None,
outdir=None,
n_boot=200,
path_to_script=None,
alpha=0.68,
n_cov=100,
n_rms=1.,
stokes='I',
sample_cc_fits_paths=None,
sample_uv_fits_paths=None):
"""
Conduct coverage analysis of image pixels flux CI. Find number of times
when CI of `sample` values contains `true` value.
:param original_uv_fits_path:
Path to original FITS-file with uv-data.
:param ci_type:
Type of CI to test. ``boot`` or ``rms``. If ``boot`` then use residuals
bootstrap CI. If ``rms`` then use Hovatta corrected image rms CI.
:param original_cc_fits_path: (optional)
Path to original FITS-file with CC model. If ``None`` then use
``imsize`` parameter to get `original` CC model from
``original_uv_fits_path``. (default: ``None``)
:param imsize: (optional)
Image parameters (image size [pix], pixel size [mas]) to use
when doing first CC with ``original_cc_fits_path = None``. (default:
``None``)
:param outdir: (optional)
Directory to store intermediate results. If ``None`` then use CWD.
(default: ``None``)
:param n_boot: (optional)
Number of bootstrap replications to use when calculating bootstrap CI
for ``ci_type = boot`` option when ``boot_cc_fits_paths`` hasn't
specified. (default: ``200``)
:param path_to_script: (optional)
Path to Dan Homan's script for final clean. If ``None`` then use CWD.
(default: ``None``)
:param alpha: (optional)
Level of significance when calculating bootstrap CI for ``ci_type =
boot`` case. E.g. ``0.68`` corresponds to `1 \sigma`. (default:
``0.68``)
:param n_cov: (optional)
Number of `samples` from infinite population to consider in coverage
analysis of intervals. Here `samples` - observations of known source
with different realisations of noise with known parameters. (default:
``100``)
:param n_rms: (optional)
Number of rms to use in ``ci_type = rms`` case. (default: ``1.``)
:param stokes: (optional)
Stokes parameter to use. If ``None`` then use ``I``. (default: ``None``)
:param boot_cc_fits_paths: (optional)
If ``ci_type = boot`` then this parameter could specify paths to cleaned
bootstrapped uv-data.
:param sample_uv_fits_paths: (optional)
Path to FITS-files with `sample` uv-data. If ``None`` then create
``n_cov`` `sample` uv-data from noise of `original` uv-data and
`original` CLEAN model. (default: ``None``)
:param sample_cc_fits_paths: (optional)
Path to FITS-files with CLEAN models of `sample` uv-data. If ``None``
then create ``n_cov`` `sample` uv-data from noise of `original` uv-data
and `original` CLEAN model. (default: ``None``)
:return:
Coverage map. Each pixel contain frequency of times when CI for samples
from population contain `true` value for given pixel.
"""
if original_cc_fits_path is None:
print(
"No `original` CLEAN model specified! Will CLEAN `original`"
" uv-data.")
if imsize is None:
raise Exception("Specify ``imsize``")
uv_fits_dir, uv_fits_fname = os.path.split(original_uv_fits_path)
print("Cleaning `original` uv-data to"
" {}".format(os.path.join(outdir, 'original_cc.fits')))
clean_difmap(uv_fits_fname,
'original_cc.fits',
stokes,
imsize,
path=uv_fits_dir,
path_to_script=path_to_script,
outpath=outdir)
original_cc_fits_path = os.path.join(outdir, 'original_cc.fits')
# Find images parameters for cleaning if necessary
if imsize is None:
print(
"Getting image parameters from `original`"
" CLEAN FITS file {}.".format(original_cc_fits_path))
image_params = get_fits_image_info(original_cc_fits_path)
imsize = (image_params['imsize'][0],
abs(image_params['pixsize'][0]) / mas_to_rad)
# This is `true` values. Will check how often they arise in `sample` CIs.
original_image = create_image_from_fits_file(original_cc_fits_path)
# If `sample` data doesn't ready - create it!
if sample_uv_fits_paths is None:
# Create `sample`
sample_uv_fits_paths, sample_cc_fits_paths =\
create_sample(original_uv_fits_path,
original_cc_fits_path=original_cc_fits_path,
imsize=imsize, outdir=outdir,
path_to_script=path_to_script, n_sample=n_cov,
stokes=stokes)
# For each pixel check how often CI of `sample` images contains `model`]
# values.
print("Creating coverage array")
cov_array = np.zeros((imsize[0], imsize[0]), dtype=float)
# Testing coverage of bootstrapped CI
# For each `sample` uv-data and model generate bootstrap CI
print sample_uv_fits_paths
print sample_cc_fits_paths
for sample_cc_fits_path, sample_uv_fits_path in zip(
sample_cc_fits_paths, sample_uv_fits_paths):
print("Sample : {}".format(sample_cc_fits_path))
if ci_type == 'boot':
n__ = sample_uv_fits_path.split('.')[0].split('_')[-1]
n_ = sample_cc_fits_path.split('.')[0].split('_')[-1]
assert n_ == n__
print(
"Bootstrapping sample uv-data {} with sample model {} using"
" {} replications".format(sample_uv_fits_path,
sample_cc_fits_path, n_boot))
print("Removing old bootstrapped files...")
boot_cc_fits_paths = glob.glob(
os.path.join(outdir, 'sample_cc_boot_*.fits'))
for rmfile in boot_cc_fits_paths:
print("Removing CC file {}".format(rmfile))
os.unlink(rmfile)
boot_uv_fits_paths = \
sorted(glob.glob(os.path.join(outdir, 'sample_uv_boot_*.uvf')))
for rmfile in boot_uv_fits_paths:
print("Removing UV file {}".format(rmfile))
os.unlink(rmfile)
bootstrap_uv_fits(sample_uv_fits_path, [sample_cc_fits_path],
n_boot,
outpath=outdir,
outname=['sample_uv_boot', '.uvf'])
boot_uv_fits_paths =\
sorted(glob.glob(os.path.join(outdir, 'sample_uv_boot_*.uvf')))
# Clean each bootstrapped uv-data
for i, uv_fits_path in enumerate(boot_uv_fits_paths):
uv_fits_dir, uv_fits_fname = os.path.split(uv_fits_path)
j = uv_fits_fname.split('.')[0].split('_')[-1]
cc_fname = 'sample_cc_boot_{}.fits'.format(j)
print("Cleaning {} bootstrapped sample"
" uv-data to {}".format(uv_fits_path,
os.path.join(outdir, cc_fname)))
clean_difmap(uv_fits_fname,
cc_fname,
stokes,
imsize,
path=uv_fits_dir,
path_to_script=path_to_script,
outpath=outdir)
boot_cc_fits_paths = sorted(
glob.glob(os.path.join(outdir, 'sample_cc_boot_*.fits')))
# Calculate bootstrap CI for current `sample`
# hdi_low, hdi_high = boot_ci_bc(boot_cc_fits_paths,
# observed_cc_fits_path, alpha=alpha)
hdi_low, hdi_high = boot_ci(boot_cc_fits_paths,
sample_cc_fits_path,
alpha=alpha)
elif ci_type == 'rms':
# Calculate ``n_rms`` CI
print("Calculating rms...")
sample_image = create_image_from_fits_file(sample_cc_fits_path)
rms = rms_image_shifted(sample_uv_fits_path,
image_fits=sample_cc_fits_path,
path_to_script=path_to_script)
# rms = rms_image(sample_image)
hdi_low = sample_image.image - n_rms * rms
hdi_high = sample_image.image + n_rms * rms
else:
raise Exception("CI intervals must be `boot` or `rms`!")
# Check if `model` value falls in current `sample` CI
print("Calculating hits of `true` values for"
" {}".format(sample_cc_fits_path))
for (x, y), value in np.ndenumerate(cov_array):
cov_array[x, y] += float(
np.logical_and(hdi_low[x, y] < original_image.image[x, y],
original_image.image[x, y] < hdi_high[x, y]))
return cov_array / n_cov
def create_coverage_map(original_uv_fits_path,
ci_type,
original_cc_fits_path=None,
imsize=None,
outdir=None,
n_boot=200,
path_to_script=None,
alpha=0.68,
n_cov=100,
n_rms=1.,
stokes='I',
boot_cc_fits_paths=None,
sample_cc_fits_paths=None):
"""
Conduct coverage analysis of image pixels flux CI. Find number of times
when CI of `observed` value contains values of `samples`.
:param original_uv_fits_path:
Path to original FITS-file with uv-data.
:param ci_type:
Type of CI to test. ``boot`` or ``rms``. If ``boot`` then use residuals
bootstrap CI. If ``rms`` then use Hovatta corrected image rms CI.
:param original_cc_fits_path: (optional)
Path to original FITS-file with CC model. If ``None`` then use
``imsize`` parameter to get `original` CC model from
``original_uv_fits_path``. (default: ``None``)
:param imsize: (optional)
Image parameters (image size [pix], pixel size [mas]) to use
when doing first CC with ``original_cc_fits_path = None``. (default:
``None``)
:param outdir: (optional)
Directory to store intermediate results. If ``None`` then use CWD.
(default: ``None``)
:param n_boot: (optional)
Number of bootstrap replications to use when calculating bootstrap CI
for ``ci_type = boot`` option when ``boot_cc_fits_paths`` hasn't
specified. (default: ``200``)
:param path_to_script: (optional)
Path to Dan Homan's script for final clean. If ``None`` then use CWD.
(default: ``None``)
:param alpha: (optional)
Level of significance when calculating bootstrap CI for ``ci_type =
boot`` case. E.g. ``0.68`` corresponds to `1 \sigma`. (default:
``0.68``)
:param n_cov: (optional)
Number of `samples` from infinite population to consider in coverage
analysis of intervals. Here `samples` - observations of known source
with different realisations of noise with known parameters. (default:
``100``)
:param n_rms: (optional)
Number of rms to use in ``ci_type = rms`` case. (default: ``1.``)
:param stokes: (optional)
Stokes parameter to use. If ``None`` then use ``I``. (default: ``None``)
:param boot_cc_fits_paths: (optional)
If ``ci_type = boot`` then this parameter could specify paths to cleaned
bootstrapped uv-data.
:param sample_cc_fits_paths: (optional)
Path to FITS-files with CLEAN models of `sample` uv-data. If ``None``
then create ``n_cov`` `sample` uv-data from noise of `original` uv-data
and `original` CLEAN model. (default: ``None``)
:return:
Coverage map. Each pixel contain frequency of times when samples from
population hit inside CI for given pixel.
"""
# If not given `original` CLEAN model - get it by cleaning `original`
# uv-data
if original_cc_fits_path is None:
print(
"No `original` CLEAN model specified! Will CLEAN `original`"
" uv-data.")
if imsize is None:
raise Exception("Specify ``imsize``")
uv_fits_dir, uv_fits_fname = os.path.split(original_uv_fits_path)
print("Cleaning `original` uv-data to"
" {}".format(os.path.join(outdir, 'cc.fits')))
clean_difmap(uv_fits_fname,
'cc.fits',
stokes,
imsize,
path=uv_fits_dir,
path_to_script=path_to_script,
outpath=outdir)
original_cc_fits_path = os.path.join(outdir, 'cc.fits')
original_uv_data = UVData(original_uv_fits_path)
noise = original_uv_data.noise()
original_model = create_model_from_fits_file(original_cc_fits_path)
# Find images parameters for cleaning if necessary
if imsize is None:
print(
"Getting image parameters from `original`"
" CLEAN FITS file {}.".format(original_cc_fits_path))
image_params = get_fits_image_info(original_cc_fits_path)
imsize = (image_params['imsize'][0],
abs(image_params['pixsize'][0]) / mas_to_rad)
# Substitute uv-data with original model and create `model` uv-data
print("Substituting original uv-data with CLEAN model...")
model_uv_data = copy.deepcopy(original_uv_data)
model_uv_data.substitute([original_model])
# Add noise to `model` uv-data to get `observed` uv-data
observed_uv_data = copy.deepcopy(model_uv_data)
observed_uv_data.noise_add(noise)
observed_uv_fits_path = os.path.join(outdir, 'observed_uv.uvf')
if os.path.isfile(observed_uv_fits_path):
os.unlink(observed_uv_fits_path)
print("Adding noise to `model` uv-data to get `observed` uv-data...")
observed_uv_data.save(fname=observed_uv_fits_path)
observed_cc_fits_path = os.path.join(outdir, 'observed_cc.fits')
if os.path.isfile(observed_cc_fits_path):
os.unlink(observed_cc_fits_path)
# Clean `observed` uv-data to get `observed` image and model
print("Cleaning `observed` uv-data to `observed` CLEAN model...")
clean_difmap('observed_uv.uvf',
'observed_cc.fits',
original_model.stokes,
imsize,
path=outdir,
path_to_script=path_to_script,
outpath=outdir)
# Get `observed` model and image
observed_model = create_model_from_fits_file(observed_cc_fits_path)
observed_image = create_image_from_fits_file(observed_cc_fits_path)
# Testing coverage of bootstrapped CI
if ci_type == 'boot':
# Bootstrap and clean only when necessary
if boot_cc_fits_paths is None:
# Bootstrap `observed` uv-data with `observed` model
boot = CleanBootstrap([observed_model], observed_uv_data)
cwd = os.getcwd()
path_to_script = path_to_script or cwd
os.chdir(outdir)
print("Bootstrapping uv-data with {} replications".format(n_boot))
boot.run(outname=['observed_uv_boot', '.uvf'], n=n_boot)
os.chdir(cwd)
boot_uv_fits_paths = sorted(
glob.glob(os.path.join(outdir, 'observed_uv_boot*.uvf')))
# Clean each bootstrapped uv-data
for i, uv_fits_path in enumerate(boot_uv_fits_paths):
uv_fits_dir, uv_fits_fname = os.path.split(uv_fits_path)
print("Cleaning {} bootstrapped observed"
" uv-data to {}".format(
uv_fits_path,
os.path.join(
outdir,
'observed_cc_boot_{}.fits'.format(i + 1))))
clean_difmap(uv_fits_fname,
'observed_cc_boot_{}.fits'.format(i + 1),
original_model.stokes,
imsize,
path=uv_fits_dir,
path_to_script=path_to_script,
outpath=outdir)
boot_cc_fits_paths = glob.glob(
os.path.join(outdir, 'observed_cc_*.fits'))
# Calculate bootstrap CI
# hdi_low, hdi_high = boot_ci_bc(boot_cc_fits_paths,
# observed_cc_fits_path, alpha=alpha)
hdi_low, hdi_high = boot_ci(boot_cc_fits_paths,
observed_cc_fits_path,
alpha=alpha)
elif ci_type == 'rms':
# Calculate ``n_rms`` CI
rms = observed_image.rms(region=(50, 50, 50, None))
rms = np.sqrt(rms**2. + (1.5 * rms**2.)**2.)
hdi_low = observed_image.image - rms
hdi_high = observed_image.image + rms
else:
raise Exception("CI intervals must be `boot` or `rms`!")
# Create `sample` uv-data and clean it only when necessary
if sample_cc_fits_paths is None:
# Add noise to `model` uv-data ``n_cov`` times and get ``n_cov``
# `samples` from population
sample_uv_fits_paths = list()
for i in range(n_cov):
sample_uv_data = copy.deepcopy(model_uv_data)
sample_uv_data.noise_add(noise)
sample_uv_fits_path = os.path.join(outdir,
'samle_uv_{}.uvf'.format(i + 1))
sample_uv_data.save(sample_uv_fits_path)
sample_uv_fits_paths.append(sample_uv_fits_path)
# Clean each `sample` FITS-file
for i, uv_fits_path in enumerate(sample_uv_fits_paths):
uv_fits_dir, uv_fits_fname = os.path.split(uv_fits_path)
print("Cleaning {} sample uv-data to"
" {}".format(
uv_fits_path,
os.path.join(outdir, 'sample_cc_{}.fits'.format(i + 1))))
clean_difmap(uv_fits_fname,
'sample_cc_{}.fits'.format(i + 1),
original_model.stokes,
imsize,
path=uv_fits_dir,
path_to_script=path_to_script,
outpath=outdir)
sample_cc_fits_paths = glob.glob(
os.path.join(outdir, 'sample_cc_*.fits'))
sample_images = list()
for sample_cc_fits_path in sample_cc_fits_paths:
image = create_image_from_fits_file(sample_cc_fits_path)
sample_images.append(image.image)
# For each pixel check how often flux in `sample` images lies in CI derived
# for observed image.
cov_array = np.zeros((imsize[0], imsize[0]), dtype=float)
print("calculating CI intervals")
for (x, y), value in np.ndenumerate(cov_array):
for image in sample_images:
cov_array[x, y] += float(
np.logical_and(hdi_low[x, y] < image[x, y],
image[x, y] < hdi_high[x, y]))
return cov_array / n_cov
continue
# Plot map with components superimposed
cc_fits_save_fname = "bk_cc_{}_{}.fits".format(str(i).zfill(2), freq)
# For 18 cm we need large pixel size
cellsize = 0.1
if freq == 1.665:
cellsize = 0.5
elif freq == 8.1:
cellsize = 0.2
clean_difmap(uv_fits_save_fname,
cc_fits_save_fname,
'I', (1024, cellsize),
path=out_dir,
path_to_script=path_to_script,
show_difmap_output=False,
outpath=out_dir)
ccimage = create_clean_image_from_fits_file(
os.path.join(out_dir, cc_fits_save_fname))
beam = ccimage.beam
rms = rms_image(ccimage)
blc, trc = find_bbox(ccimage.image, rms, 10)
comps = import_difmap_model(out_dfm_model_fn, out_dir)
plot_fitted_model(os.path.join(out_dir, uv_fits_save_fname),
comps,
savefig=os.path.join(
out_dir, "difmap_model_uvplot_{}_{}.png".format(
data_dir = os.path.join(base_dir, source)
if not os.path.exists(data_dir):
os.makedirs(data_dir)
source_images = dict()
download_mojave_uv_fits(source,
epochs=[source_epoch_dict[source]],
bands=['u'],
download_dir=data_dir)
epoch = source_epoch_dict[source]
fname = mojave_uv_fits_fname(source, 'u', epoch)
stokes = 'I'
cc_fits = map_fname(source, epoch, stokes)
clean_difmap(fname,
cc_fits,
stokes,
mapsize,
path=data_dir,
path_to_script=path_to_script,
outpath=data_dir)
i_image = create_clean_image_from_fits_file(os.path.join(
data_dir, cc_fits))
# imsize = i_image.imsize
# i_rms = i_image.rms(region=(imsize[0] / 10., imsize[0] / 10., imsize[0] / 10., None))
# # Calculate distance to most distant pixel with rms > 7 * rms
# mask = i_image.image < 10. * i_rms
# i_image_zeroed = i_image.image.copy()
# i_image_zeroed[mask] = 0.
# y, x = np.nonzero(i_image_zeroed)
# y -= i_image.pixref[0]
# x -= i_image.pixref[1]
source_images = dict()
download_mojave_uv_fits(source,
epochs=[source_epoch_dict[source]],
bands=['u'],
download_dir=data_dir)
for epoch in [source_epoch_dict[source]]:
print "Working with epoch {}".format(epoch)
fname = mojave_uv_fits_fname(source, 'u', epoch)
for stokes in ('I', 'Q', 'U'):
# First restore with naitive beam to get it parameters
print "Cleaning stokes {} with naitive restoring" \
" beam".format(stokes)
clean_difmap(fname,
map_fname(source, epoch, stokes),
stokes,
mapsize,
path=data_dir,
path_to_script=path_to_script,
outpath=data_dir)
i_image = create_clean_image_from_fits_file(
os.path.join(data_dir, map_fname(source, epoch, 'I')))
imsize = i_image.imsize
# Get beam parameters
beam = i_image.beam
print "Beam : {} [mas, mas, deg]".format(beam)
circ_beam = (beam[0], beam[0], 0)
print "Using circular beam {} [mas, mas, deg]".format(circ_beam)
# Now clean and restore with circular beam
for stokes in ('I', 'Q', 'U'):
print "Cleaning stokes {} with circular restoring" \
),
bmaj=(
sp.stats.uniform.logpdf,
[0, 20.],
dict(),
))
model = Model(stokes='I')
model.add_components(cg1, cg2)
uvdata.substitute([model])
uvdata.noise_add(noise)
uvdata.save(os.path.join(data_dir, 'fake.uvf'))
# Clean uv-data
clean_difmap(
'fake.uvf',
'fake_cc.fits',
'I', (1024, 0.1),
path=data_dir,
path_to_script='/home/ilya/code/vlbi_errors/difmap/final_clean_nw',
outpath=data_dir,
show_difmap_output=True)
image = create_clean_image_from_fits_file(
os.path.join(data_dir, 'fake_cc.fits'))
rms = rms_image(image)
blc, trc = find_bbox(image.image, 2. * rms, delta=int(image._beam.beam[0]))
# Plot image
iplot(image.image,
x=image.x,
y=image.y,
min_abs_level=3. * rms,
outfile='fake_image',
outdir=data_dir,
blc=blc,
# uv_fits = '/home/ilya/data/check_cv_misha/1226+023.X1.2010_01_26.UV_CAL'
# windows = '/home/ilya/data/check_cv_misha/1226+023.X1.2010_01_26.win'
cv_scores = dict()
n_folds = 10
# for niter in cc_pars:
for k, script in path_to_scripts.items():
# print("Using niter = {}".format(niter))
print("Using k = {}".format(k))
kfold = KFoldCV(uv_fits, n_folds)
cv = list()
for j, (tr_fname, ts_fname) in enumerate(kfold):
# clean_n(kfold.train_fname, 'trained_model_{}.FITS'.format(niter), 'i',
# (1024, 0.1), niter=niter, path_to_script=path_to_script,
# show_difmap_output=True)
clean_difmap(kfold.train_fname, 'trained_model_{}.FITS'.format(k), 'i',
(1024, 0.1), path_to_script=path_to_scripts[k],
show_difmap_output=True)
tr_model = create_model_from_fits_file('trained_model_{}.FITS'.format(k))
ts_uvdata = UVData(ts_fname)
# score = ts_uvdata.cv_score(tr_model)
score = score_(ts_fname, 'trained_model_{}.FITS'.format(k))
print("{} of {} gives {}".format(j+1, n_folds, score))
cv.append(score)
cv_scores[k] = (np.nanmean(cv), np.nanstd(cv))
print("CV gives {} +/- {}".format(np.nanmean(cv), np.nanstd(cv)))
print(cv_scores)
n = cv_scores.keys()
scores = [cv_scores[i][0] for i in n]
errors = [cv_scores[i][1] for i in n]
def get_stacked_map(source, mojave_dir=None, out_dir=None, imsize=(512, 0.1),
path_to_script=None, epochs_slice=None):
"""
Functions that returns stacked image of given source using MOJAVE 15 GHz
data downloading it directly from MOJAVE DB or getting it from
user-specified directory.
:param source:
Source name [B1950].
:param mojave_dir: (optional)
Path to directory with MOJAVE 15 GHz data. If ``None`` then download
from MOJAVE DB. (default: ``None``)
:param out_dir: (optional)
Directory where to store files. If ``None`` then use CWD. (default:
``None``)
:param imsize: (optional)
Tuple of image size [pix], pixel size [mas] to put to difmap. (default:
``(512, 0.1)``)
:param path_to_script: (optional)
Path to difmap CLEANing script. If ``None`` then use CWD. (default:
``None``)
:param epochs_slice: (optional)
Slice of epochs sorted to process. If ``None`` then use all available
epochs. (default: ``None``)
:return:
Numpy 2D array with stacked image.
"""
if path_to_script is None:
path_to_script = os.getcwd()
epochs = get_epochs_for_source(source)
if epochs_slice is not None:
epochs = epochs[::-1][epochs_slice]
if out_dir is None:
out_dir = os.getcwd()
elif not os.path.exists(out_dir):
os.makedirs(out_dir)
if mojave_dir is None:
download_mojave_uv_fits(source, epochs, bands=['u'],
download_dir=out_dir)
else:
out_dir = mojave_dir
beams_dict = dict()
print "Output directory : {}".format(out_dir)
# First clean and restore with native beam
epoch_stokes_dict = dict()
for epoch in sorted(epochs):
print "Cleaning epoch {} with naitive restoring beam".format(epoch)
uv_fits_fname = mojave_uv_fits_fname(source, 'u', epoch)
uvdata = UVData(os.path.join(out_dir, uv_fits_fname))
uv_stokes = uvdata.stokes
if 'RR' in uv_stokes and 'LL' in uv_stokes:
stokes = 'I'
elif 'RR' in uv_stokes and 'LL' not in uv_stokes:
stokes = 'RR'
elif 'LL' in uv_stokes and 'RR' not in uv_stokes:
stokes = 'LL'
else:
continue
epoch_stokes_dict.update({epoch: stokes})
im_fits_fname = "{}_{}_{}_{}.fits".format(source, 'U', epoch, stokes)
print "Difmap params: "
print "uv_fits_fname : {}".format(uv_fits_fname)
print "im_fits_fname : {}".format(im_fits_fname)
print "path : {}".format(out_dir)
print "outpath: {}".format(out_dir)
clean_difmap(uv_fits_fname, im_fits_fname, stokes, imsize,
path=out_dir, path_to_script=path_to_script,
outpath=out_dir)
ccimage = create_clean_image_from_fits_file(os.path.join(out_dir,
im_fits_fname))
beam = ccimage.beam
print "Beam for epoch {} : {} [mas, mas, deg]".format(epoch, beam)
beams_dict.update({epoch: (beam[0], beam[0], 0)})
circ_beam = np.mean([beam[0] for beam in beams_dict.values()])
# Now clean and restore with circular beam
images = list()
for epoch in sorted(epochs):
stokes = epoch_stokes_dict[epoch]
uv_fits_fname = mojave_uv_fits_fname(source, 'u', epoch)
im_fits_fname = "{}_{}_{}_{}_circ.fits".format(source, 'U', epoch,
stokes)
print "Difmap params: "
print "uv_fits_fname : {}".format(uv_fits_fname)
print "im_fits_fname : {}".format(im_fits_fname)
print "path : {}".format(out_dir)
print "outpath: {}".format(out_dir)
clean_difmap(uv_fits_fname, im_fits_fname, stokes, imsize,
path=out_dir, path_to_script=path_to_script,
outpath=out_dir, beam_restore=(circ_beam, circ_beam, 0))
image = create_image_from_fits_file(os.path.join(out_dir,
im_fits_fname))
images.append(image.image.copy())
images = np.dstack(images)
return np.mean(images, axis=2)
import matplotlib.pyplot as plt
import matplotlib
matplotlib.rcParams['xtick.labelsize'] = 14
matplotlib.rcParams['ytick.labelsize'] = 14
matplotlib.rcParams['axes.labelsize'] = 14
path_to_script = '/home/ilya/github/vlbi_errors/difmap/final_clean_nw'
data_dir = '/home/ilya/Dropbox/ACC/3c120/boot'
uvdata_dir = '/home/ilya/Dropbox/ACC/3c120/uvdata'
uv_fits_u = '0430+052.u.2006_05_24.uvf'
uv_fits_x = '0430+052.x.2006_05_24.uvf'
# Clean original uv-data with native beam
clean_difmap(uv_fits_u, 'u_cc.fits', 'I', (1024, 0.1), path=uvdata_dir,
path_to_script=path_to_script, show_difmap_output=True,
outpath=data_dir)
clean_difmap(uv_fits_x, 'x_cc.fits', 'I', (1024, 0.1), path=uvdata_dir,
path_to_script=path_to_script, show_difmap_output=True,
outpath=data_dir)
# Clean original uv-data with common beam
clean_difmap(uv_fits_x, 'x_cc_same.fits', 'I', (1024, 0.1), path=uvdata_dir,
path_to_script=path_to_script, show_difmap_output=True,
outpath=data_dir)
ccimage_x = create_clean_image_from_fits_file(os.path.join(data_dir, 'x_cc.fits'))
clean_difmap(uv_fits_u, 'u_cc_same.fits', 'I', (1024, 0.1), path=uvdata_dir,
path_to_script=path_to_script, show_difmap_output=True,
outpath=data_dir, beam_restore=ccimage_x.beam)
u_model = create_model_from_fits_file(os.path.join(data_dir, 'u_cc.fits'))
info_h = get_fits_image_info(os.path.join(data_dir,
'1038+064.l22.2010_05_21.icn.fits'))
mapsize_clean = (info_h[0][0], abs(info_h[-3][0] / mas_to_rad))
# # Check that we got rigth model
# uvdata = create_uvdata_from_fits_file(os.path.join(data_dir,
# '1038+064.l22.2010_05_21.uvf'))
# ccmodel = create_ccmodel_from_fits_file(os.path.join(data_dir,
# '1038+064.l22.2010_05_21.icn.fits'))
# uvdata.uvplot()
# uvdata.substitute([ccmodel])
# uvdata.uvplot(sym='.r')
# Clean longest wavelength uv-data for current source and find common beam
clean_difmap('1038+064.l18.2010_05_21.uvf', '1038+064.l18.2010_05_21.icn.fits',
'i', mapsize_clean, data_dir, path_to_script, outpath=data_dir,
show_difmap_output=True)
info_l = get_fits_image_info(os.path.join(data_dir,
'1038+064.l18.2010_05_21.icn.fits'))
beam_restore = (info_l[3][0] / mas_to_rad, info_l[3][1] / mas_to_rad,
info_l[3][2] / degree_to_rad)
print "Longest wavelength beam : {}".format(beam_restore)
# Cleaning all uv-files with naitive resolution
for uv_path in uv_paths:
uv_dir, uv_file = os.path.split(uv_path)
l = uv_file.split('.')[1]
epoch = uv_file.split('.')[2]
for stoke in ('i', 'q', 'u'):
outfname = "{}.{}.{}.{}cn.fits".format(source, l, epoch, stoke)
cv_scores[overclean_coeff] = dict()
for i, (train_uvfits_fname, test_uvfits_fname) in enumerate(scans_cv):
print("BASELINE = ", scans_cv.cur_bl, ", SCAN# = ",
scans_cv.cur_scan, " ====================")
# Optionally create entry in dict
if scans_cv.cur_bl not in cv_scores[overclean_coeff]:
cv_scores[overclean_coeff][scans_cv.cur_bl] = list()
print(train_uvfits_fname, test_uvfits_fname)
# CLEAN train data set
clean_difmap(fname=train_uvfits_fname,
outfname="trained_cc.fits",
stokes='i',
mapsize_clean=(512, 0.1),
path=data_dir,
outpath=data_dir,
path_to_script=path_to_script,
show_difmap_output=False,
text_box="/home/ilya/data/cv/box.clean")
# Score trained model on test data set
cv_score = score(os.path.join(data_dir, test_uvfits_fname),
os.path.join(data_dir, "trained_cc.fits"),
bmaj=beam,
score="l2",
use_weights=False)
print("CV score = ", cv_score)
cv_scores[overclean_coeff][scans_cv.cur_bl].append(cv_score)
print("Result for current k = {} is {}".format(
overclean_coeff, cv_scores[overclean_coeff]))
pixsize = abs(image_stack.pixsize[0])
imsize = image_stack.imsize
model = Model(stokes='I')
model.from_2darray(image, (0.01, 0.01))
# model.add_component(ImageComponent(image, x[0,:],
# y[:,0]))
# model.add_component(CGComponent(2.25, 0., 0., 0.01))
import copy
model_uvdata = copy.deepcopy(original_uvdata)
model_uvdata.substitute([model])
model_uvdata.noise_add(noise)
#fig = original_uvdata.uvplot()
model_uvdata.uvplot()
model_uvdata.save('opening.fits', rewrite=True)
# import astropy.io.fits as pf
# original_hdu = pf.open(os.path.join('/home/ilya/Dropbox/0235/VLBA/',
# '0235+164.q1.2008_09_02.uvf_difmap'))[0]
# hdu = pf.open('opening.fits')[0]
# print original_hdu.data[0]
# print hdu.data[0]
from spydiff import clean_difmap
clean_difmap(
'opening.fits',
'opening_cc.fits',
'I', (512, 0.03),
path_to_script='/home/ilya/code/vlbi_errors/difmap/final_clean_nw',
show_difmap_output=True)
image = create_image_from_fits_file('opening_cc.fits')
plt.matshow(image.image)
plt.show()
def clean_boot_data(sources,
epochs,
bands,
stokes,
base_path=None,
path_to_script=None,
pixels_per_beam=None,
imsize=None):
"""
:param sources:
Iterable of sources names.
:param epochs:
Iterable of sources epochs.
:param bands:
Iterable of bands.
:param stokes:
Iterable of stokes parameters.
:param base_path: (optional)
Path to route of directory tree. If ``None`` then use current directory.
(default: ``None``)
:param path_to_script: (optional)
Path to ``clean`` difmap script. If ``None`` then use current directory.
(default: ``None``)
:param beam: (optional)
Beam parameter for cleaning (bmaj, bmin, bpa). If ``None`` then use
naitive beam. (default: ``None``)
:param mapsize_clean: (optional)
Parameters of map for cleaning (map size, pixel size). If ``None``
then use those of map in map directory (not bootstrapped).
(default: ``None``)
:param mapsize_restore: (optional)
Parameters of map for restoring CC (map size, pixel size). If
``None`` then use ``mapsize_clean``. (default: ``None``)
"""
if base_path is None:
base_path = os.getcwd()
elif not base_path.endswith("/"):
base_path += "/"
stokes = list(stokes)
# Now ``I`` goes first
stokes.sort()
curdir = os.getcwd()
print "Cleaning bootstrapped and original data..."
for source in sources:
print " for source ", source
for epoch in epochs:
print " for epoch ", epoch
stoke = 'i'
# Find ``mapsize`` using highest frequency data
band = bands[-1]
map_path = im_fits_path(source,
band,
epoch,
stoke,
base_path=base_path)
try:
map_info = get_fits_image_info(map_path + 'cc.fits')
except IOError:
continue
mapsize_clean = (map_info[0][0], map_info[-3][0] / mas_to_rad)
# Find ``beam_restore`` using lowest frequency data
band = bands[0]
map_path = im_fits_path(source,
band,
epoch,
stoke,
base_path=base_path)
map_info = get_fits_image_info(map_path + 'cc.fits')
beam_restore = (map_info[3][0] / mas_to_rad,
map_info[3][1] / mas_to_rad,
map_info[3][2] / degree_to_rad)
# If we told to use some pixel size (in units of low frequency beam)
if pixels_per_beam is not None:
pixsize = beam_restore[0] / pixels_per_beam
else:
pixsize = mapsize_clean[1]
# If we don't told to use some image size we construct it to keep
# physical image size as in low frequency map
if imsize is None:
# imsize = imsize_low * pix_size_low / new_pixsize
imsize = map_info[0][0] * (map_info[-3][0] /
mas_to_rad) / pixsize
powers = [imsize // (2**i) for i in range(15)]
indx = powers.index(0)
imsize = 2**indx
# Chosen image & pixel sizes
mapsize_clean = (imsize, pixsize)
print "Common mapsize: {}".format(mapsize_clean)
for band in bands:
print " for band ", band
uv_path = uv_fits_path(source,
band.upper(),
epoch,
base_path=base_path)
n = len(glob.glob(uv_path + '*boot*_*.fits'))
if n == 0:
print "skippin source {}, epoch {}, band {}".format(
source, epoch, band)
continue
# Cleaning bootstrapped data & restore with low resolution
for i in range(n):
uv_fname = uv_path + 'boot_' + str(i + 1) + '.fits'
if not os.path.isfile(uv_fname):
print "...skipping absent file ", uv_fname
continue
print " Using uv-file ", uv_fname
# Sort stokes with ``I`` first and use it's beam
for stoke in stokes:
print " working with stokes parameter ", stoke
map_path = im_fits_path(source,
band,
epoch,
stoke,
base_path=base_path)
clean_difmap(fname='boot_' + str(i + 1) + '.fits',
outfname='cc_' + str(i + 1) + '.fits',
stokes=stoke,
mapsize_clean=mapsize_clean,
path=uv_path,
path_to_script=path_to_script,
mapsize_restore=None,
beam_restore=beam_restore,
outpath=map_path)
# Cleaning original data & restore with low_freq resolution
for stoke in stokes:
print " working with stokes parameter ", stoke
map_path = im_fits_path(source,
band,
epoch,
stoke,
base_path=base_path)
clean_difmap(fname='sc_uv.fits',
outfname='cc_orig.fits',
stokes=stoke,
mapsize_clean=mapsize_clean,
path=uv_path,
path_to_script=path_to_script,
mapsize_restore=None,
beam_restore=beam_restore,
outpath=map_path)
os.chdir(curdir)
x1_uvdata.substitute([ccmodel_x1])
# FIXME: !!!
noise[264] = 0.05 * noise[noise.keys()[0]]
x1_uvdata.noise_add(noise)
x1_uvdata.save(x1_uvdata.data, os.path.join(base_path, 'x1_uv_real.fits'))
# Clean parameters
mapsize_clean = (1024, abs(pixsize_x1[0]) / mas_to_rad)
beam_restore = (map_info_c1[3][0] / mas_to_rad, map_info_c1[3][1] / mas_to_rad,
map_info_c1[3][2] / degree_to_rad)
# Clean
clean_difmap('c1_uv_real.fits',
'c1_cc_real.fits',
'i',
mapsize_clean,
path=base_path,
path_to_script=path_to_script,
beam_restore=beam_restore,
show_difmap_output=True,
outpath=base_path)
# Clean
clean_difmap('x1_uv_real.fits',
'x1_cc_real.fits',
'i',
mapsize_clean,
path=base_path,
path_to_script=path_to_script,
beam_restore=beam_restore,
show_difmap_output=True,
outpath=base_path)
def create_sample(original_uv_fits_path,
original_cc_fits_path=None,
imsize=None,
outdir=None,
path_to_script=None,
n_sample=100,
stokes='I'):
"""
Create `sample` from `true` or `model` source
:param original_uv_fits_path:
Path to original FITS-file with uv-data.
:param original_cc_fits_path: (optional)
Path to original FITS-file with CC model. If ``None`` then use
``imsize`` parameter to get `original` CC model from
``original_uv_fits_path``. (default: ``None``)
:param imsize: (optional)
Image parameters (image size [pix], pixel size [mas]) to use
when doing first CC with ``original_cc_fits_path = None``. (default:
``None``)
:param outdir: (optional)
Directory to store intermediate results. If ``None`` then use CWD.
(default: ``None``)
:param path_to_script: (optional)
Path to Dan Homan's script for final clean. If ``None`` then use CWD.
(default: ``None``)
:param n_sample: (optional)
Number of `samples` from infinite population to consider in coverage
analysis of intervals. Here `samples` - observations of known source
with different realisations of noise with known parameters. (default:
``100``)
:param stokes: (optional)
Stokes parameter to use. If ``None`` then use ``I``. (default: ``None``)
:return:
Creates FITS-files with uv-data and CLEAN models of `sample`.
"""
# If not given `original` CLEAN model - get it by cleaning `original`
# uv-data
if original_cc_fits_path is None:
print(
"No `original` CLEAN model specified! Will CLEAN `original`"
" uv-data.")
if imsize is None:
raise Exception("Specify ``imsize``")
uv_fits_dir, uv_fits_fname = os.path.split(original_uv_fits_path)
original_cc_fits_path = os.path.join(outdir, 'original_cc.fits')
print(
"Cleaning `original` uv-data to {}".format(original_cc_fits_path))
clean_difmap(uv_fits_fname,
'original_cc.fits',
stokes,
imsize,
path=uv_fits_dir,
path_to_script=path_to_script,
outpath=outdir)
original_uv_data = UVData(original_uv_fits_path)
noise = original_uv_data.noise()
original_model = create_model_from_fits_file(original_cc_fits_path)
# Find images parameters for cleaning if necessary
if imsize is None:
print(
"Getting image parameters from `original`"
" CLEAN FITS file {}.".format(original_cc_fits_path))
image_params = get_fits_image_info(original_cc_fits_path)
imsize = (image_params['imsize'][0],
abs(image_params['pixsize'][0]) / mas_to_rad)
# Substitute uv-data with original model and create `model` uv-data
print("Substituting `original` uv-data with CLEAN model...")
model_uv_data = copy.deepcopy(original_uv_data)
model_uv_data.substitute([original_model])
# Create `sample` uv-data
# Add noise to `model` uv-data ``n_cov`` times and get ``n_cov`` `samples`
# from population
sample_uv_fits_paths = list()
print("Creating {} `samples` from population".format(n_sample))
for i in range(n_sample):
sample_uv_data = copy.deepcopy(model_uv_data)
sample_uv_data.noise_add(noise)
sample_uv_fits_path = os.path.join(
outdir, 'sample_uv_{}.uvf'.format(str(i + 1).zfill(3)))
sample_uv_data.save(sample_uv_fits_path)
sample_uv_fits_paths.append(sample_uv_fits_path)
# Clean each `sample` FITS-file
print("CLEANing `samples` uv-data")
for uv_fits_path in sample_uv_fits_paths:
uv_fits_dir, uv_fits_fname = os.path.split(uv_fits_path)
j = uv_fits_fname.split('.')[0].split('_')[-1]
print("Cleaning {} sample uv-data to"
" {}".format(uv_fits_path,
os.path.join(outdir,
'sample_cc_{}.fits'.format(j))))
clean_difmap(uv_fits_fname,
'sample_cc_{}.fits'.format(j),
original_model.stokes,
imsize,
path=uv_fits_dir,
path_to_script=path_to_script,
outpath=outdir)
sample_cc_fits_paths = sorted(
glob.glob(os.path.join(outdir, 'sample_cc_*.fits')))
sample_uv_fits_paths = sorted(
glob.glob(os.path.join(outdir, 'sample_uv_*.uvf')))
return sample_uv_fits_paths, sample_cc_fits_paths
data_dir = "/home/ilya/data/selfcal"
mapsize_clean = 512, 0.1
n_mc = 100
epochs = glob.glob(os.path.join(data_dir, "*_*_*.mod"))
epochs = [os.path.split(epoch)[-1] for epoch in epochs]
epochs = sorted([epoch.split(".")[0] for epoch in epochs])
# epochs = ["2019_08_23", "2019_10_11"]
epochs = ["2015_02_20"]
for epoch in epochs:
# Obtain CLEAN model
clean_difmap(fname="2200+420.u.{}.uvf".format(epoch),
outfname="cc_I.fits",
path=data_dir,
stokes="I",
outpath=data_dir,
mapsize_clean=(512, 0.1),
path_to_script="/home/ilya/github/ve/difmap/final_clean_nw",
show_difmap_output=False)
ccmodel = create_model_from_fits_file(os.path.join(data_dir, "cc_I.fits"))
# Self-cal with script
original_raw_uvf = "2200+420.u.{}.uvf_raw_edt".format(epoch)
selfcal_difmap(fname=original_raw_uvf,
outfname="myselfcaled.uvf",
path=data_dir,
path_to_script="/home/ilya/github/ve/difmap/auto_selfcal",
outpath=data_dir,
show_difmap_output=True)
selfcaled_uvf = "myselfcaled.uvf"
def coverage_of_model(original_uv_fits,
original_mdl_file,
outdir=None,
n_cov=100,
n_boot=300,
mapsize=(1024, 0.1),
path_to_script=None):
"""
Conduct coverage analysis of uv-data & model
:param original_uv_fits:
Self-calibrated uv-fits file.
:param original_mdl_file:
Difmap txt-file with model.
:param outdir:
Output directory to store results.
:param n_cov:
Number of samples to create.
"""
# Create sample of 100 uv-fits data & models
sample_uv_fits_paths, sample_model_paths = create_sample(original_uv_fits,
original_mdl_file,
outdir=outdir,
n_sample=n_cov)
# For each sample uv-fits & model find 1) conventional errors & 2) bootstrap
# errors
for j, (sample_uv_fits_path, sample_mdl_path) in enumerate(
zip(sample_uv_fits_paths, sample_model_paths)):
sample_uv_fits, dir = os.path.split(sample_uv_fits_path)
sample_mdl_file, dir = os.path.split(sample_mdl_path)
try:
comps = import_difmap_model(sample_mdl_file, dir)
except ValueError:
print('Problem import difmap model')
model = Model(stokes='I')
model.add_components(*comps)
# Find errors by using Fomalont way
# 1. Clean uv-data
clean_difmap(sample_uv_fits,
'sample_cc_{}.fits'.format(j),
'I',
mapsize,
path=dir,
path_to_script=path_to_script,
outpath=dir)
# 2. Get beam
ccimage = create_clean_image_from_fits_file(
os.path.join(dir, 'sample_cc_{}.fits'.format(j)))
beam = ccimage.beam_image
# 2. Subtract components convolved with beam
ccimage.substract_model(model)
# Find errors by using Lee way
# a) fit uv-data and find model
# b) CLEAN uv-data
# c) substract model from CLEAN image
# d) find errors
pass
# Find errors by using bootstrap
# FT model to uv-plane
uvdata = UVData(sample_uv_fits_path)
try:
boot = CleanBootstrap([model], uvdata)
# If uv-data contains only one Stokes parameter (e.g. `0838+133`)
except IndexError:
print('Problem bootstrapping')
curdir = os.getcwd()
os.chdir(dir)
boot.run(n=n_boot, nonparametric=True, outname=[outname, '.fits'])
os.chdir(curdir)
booted_uv_paths = sorted(
glob.glob(os.path.join(data_dir, outname + "*")))
# Modelfit bootstrapped uvdata
for booted_uv_path in booted_uv_paths:
path, booted_uv_file = os.path.split(booted_uv_path)
i = booted_uv_file.split('_')[-1].split('.')[0]
modelfit_difmap(booted_uv_file,
dfm_model_fname,
dfm_model_fname + '_' + i,
path=path,
mdl_path=data_dir,
out_path=data_dir)
# Get params of initial model used for bootstrap
comps = import_difmap_model(dfm_model_fname, data_dir)
comps_params0 = {i: [] for i in range(len(comps))}
for i, comp in enumerate(comps):
comps_params0[i].extend(list(comp.p))
# Load bootstrap models
booted_mdl_paths = glob.glob(
os.path.join(data_dir, dfm_model_fname + "_*"))
comps_params = {i: [] for i in range(len(comps))}
for booted_mdl_path in booted_mdl_paths:
path, booted_mdl_file = os.path.split(booted_mdl_path)
comps = import_difmap_model(booted_mdl_file, path)
for i, comp in enumerate(comps):
comps_params[i].extend(list(comp.p))
# Print 65-% intervals (1 sigma)
for i, comp in enumerate(comps):
errors_fname = '68_{}_{}_comp{}.txt'.format(source, last_epoch, i)
fn = open(os.path.join(data_dir, errors_fname), 'w')
print "Component #{}".format(i + 1)
for j in range(len(comp)):
low, high, mean, median = hdi_of_mcmc(np.array(
comps_params[i]).reshape((n_boot, len(comp))).T[j],
cred_mass=0.68,
return_mean_median=True)
fn.write("{} {} {} {} {}".format(comp.p[j], low, high, mean,
median))
fn.write("\n")
fn.close()
# For source in sources with component close to core
# 1. Find residuals or estimate noise
# 2. N times add resampled residuals (or just gaussian noise) to model and
# create N new datasets
# 3. Fit them using difmap.
# 4. Find errors using Fomalont, Yee and using bootstrap. Check coverage.
base_dir = '/home/ilya/vlbi_errors/model_cov'
n_boot = 300
outname = 'boot_uv'
names = [
'source', 'id', 'trash', 'epoch', 'flux', 'r', 'pa', 'bmaj', 'e', 'bpa'
]
df = pd.read_table(os.path.join(base_dir, 'asu.tsv'),
sep=';',
header=None,
names=names,
dtype={key: str
for key in names},
index_col=False)
# Mow for all sources get the latest epoch and create directory for analysis
for source in df['source'].unique():
epochs = df.loc[df['source'] == source]['epoch']
last_epoch_ = list(epochs)[-1]
last_epoch = last_epoch_.replace('-', '_')
data_dir = os.path.join(base_dir, source, last_epoch)
if not os.path.exists(data_dir):
os.makedirs(data_dir)
try:
download_mojave_uv_fits(source,
epochs=[last_epoch],
bands=['u'],
download_dir=data_dir)
except:
open(
'problem_download_from_mojave_{}_{}'.format(
source, last_epoch), 'a').close()
continue
uv_fits_fname = mojave_uv_fits_fname(source, 'u', last_epoch)
# Create instance of Model and bootstrap uv-data
dfm_model_fname = 'dfmp_original_model.mdl'
fn = open(os.path.join(data_dir, dfm_model_fname), 'w')
model_df = df.loc[np.logical_and(df['source'] == source,
df['epoch'] == last_epoch_)]
for (flux, r, pa, bmaj, e, bpa) in np.asarray(
model_df[['flux', 'r', 'pa', 'bmaj', 'e', 'bpa']]):
print flux, r, pa, bmaj, e, bpa
if not r.strip(' '):
r = '0.0'
if not pa.strip(' '):
pa = '0.0'
if not bmaj.strip(' '):
bmaj = '0.0'
if not e.strip(' '):
e = "1.0"
if np.isnan(float(bpa)):
bpa = "0.0"
else:
bpa = bpa + 'v'
if bmaj == '0.0':
type_ = 0
bpa = "0.0"
else:
bmaj = bmaj + 'v'
type_ = 1
fn.write("{}v {}v {}v {} {} {} {} {} {}".format(
flux, r, pa, bmaj, e, bpa, type_, "0", "0\n"))
fn.close()
from from_fits import create_model_from_fits_file
from spydiff import clean_difmap, modelfit_difmap, import_difmap_model
from uv_data import UVData
from model import Model
data_dir = '/home/ilya/Dropbox/0235/tmp/test_mdl_cc/'
uv_fits = '0235+164.q1.2008_06_12.uvp'
model_fname = '0235+164.q1.2008_06_12.mdl'
path_to_script = '/home/ilya/code/vlbi_errors/difmap/final_clean_nw'
uvdata = UVData(os.path.join(data_dir, uv_fits))
# First CLEAN data
clean_difmap(uv_fits,
'cc.fits',
'I', (1024, 0.03),
path=data_dir,
path_to_script=path_to_script,
show_difmap_output=True,
outpath=data_dir)
model = create_model_from_fits_file(os.path.join(data_dir, 'cc.fits'))
comps = model._components
comps = sorted(comps,
key=lambda x: np.sqrt(x._p[1]**2 + x._p[2]**2),
reverse=True)
for i, comp in enumerate(comps[350:]):
print "Substracting {} components".format((i + 350))
uvdata_ = UVData(os.path.join(data_dir, uv_fits))
mdl1 = Model(stokes='I')
mdl1.add_component(comp)
from image import plot as iplot
from image import find_bbox
# Plotting bootstrap error of I Stokes
path_to_script = '/home/ilya/code/vlbi_errors/difmap/final_clean_nw'
# data_dir = '/home/ilya/Dropbox/papers/boot/new_pics/uvdata'
data_dir = '/home/ilya/vlbi_errors/examples/coverage/1749+701/errors'
# uv_fits = '1633+382.l22.2010_05_21.uvf'
# im_fits = '1633+382.l22.2010_05_21.icn.fits'
# uv_fits = '1226+023.u.2006_03_09.uvf'
uv_fits = '1749+701.x.2006_04_05.uvf'
# im_fits = '3c273_orig_cc.fits'
im_fits = '1749+701.x.2006_04_05.cc.fits'
clean_difmap(uv_fits,
im_fits,
'I', (512, 0.1),
path=data_dir,
path_to_script=path_to_script,
outpath=data_dir)
model = create_model_from_fits_file(os.path.join(data_dir, im_fits))
uvdata = UVData(os.path.join(data_dir, uv_fits))
boot = CleanBootstrap([model], uvdata)
os.chdir(data_dir)
boot.run(100, nonparametric=False, use_v=False)
booted_uv_fits = glob.glob(os.path.join(data_dir, 'bootstrapped_data*.fits'))
for i, boot_uv in enumerate(sorted(booted_uv_fits)):
clean_difmap(boot_uv,
'booted_cc_{}.fits'.format(i),
'I', (512, 0.1),
path=data_dir,
path_to_script=path_to_script,
outpath=data_dir)
print "Source {}, epoch {}".format(source, epoch)
data_dir = os.path.join(base_dir, source)
if not os.path.exists(data_dir):
os.makedirs(data_dir)
# First, download lowest frequency band and find beam width
download_mojave_uv_fits(source,
epochs=[epoch],
bands=[bands[0]],
download_dir=data_dir)
uv_fits_fname = mojave_uv_fits_fname(source, bands[0], epoch)
cc_fits_fname = "{}_{}_{}_{}_naitive_cc.fits".format(
source, epoch, bands[0], 'I')
clean_difmap(uv_fits_fname,
cc_fits_fname,
'I',
mapsize_dict[bands[0]],
path=data_dir,
path_to_script=path_to_script,
outpath=data_dir)
cc_image = create_clean_image_from_fits_file(
os.path.join(data_dir, cc_fits_fname))
image = create_image_from_fits_file(
os.path.join(data_dir, cc_fits_fname))
# Beam in mas, mas, deg
beam = cc_image.beam
print "BW [mas, mas, deg] : ", beam
# pixel size in mas
pixsize = abs(cc_image.pixsize[0]) / mas_to_rad
# This is characteristic jet width in pixels (half of that)
width = 0.5 * (beam[0] + beam[1]) / pixsize
print "BW scale [pixels] : ", width
for baseline, std in noise.items():
noise[baseline] = noise_factor * std
q1_uvdata.substitute([q1_model])
q1_uvdata.noise_add(noise)
q1_uvdata.save(q1_uvdata.data, os.path.join(path, 'q1_uv.fits'))
# Clean parameters
mapsize_clean = (imsize[0], abs(pixsize[0]) / mas_to_rad)
beam_restore = (map_info[3][0] / mas_to_rad, map_info[3][1] / mas_to_rad,
map_info[3][2] / degree_to_rad)
# Clean
clean_difmap('k1_uv.fits',
'k1_cc.fits',
'i',
mapsize_clean,
path=path,
path_to_script=path_to_script,
beam_restore=beam_restore,
show_difmap_output=True,
outpath=path)
# Clean
clean_difmap('q1_uv.fits',
'q1_cc.fits',
'i',
mapsize_clean,
path=path,
path_to_script=path_to_script,
beam_restore=beam_restore,
show_difmap_output=True,
outpath=path)
def analyze_source(uv_fits_paths,
n_boot,
imsizes=None,
common_imsize=None,
common_beam=None,
find_shifts=False,
outdir=None,
path_to_script=None,
clear_difmap_logs=True,
rotm_slices=None):
"""
Function that uses multifrequency self-calibration data for in-depth
analysis.
:param uv_fits_paths:
Iterable of paths to self-calibrated uv-data FITS-files.
:param n_boot:
Number of bootstrap replications to use in analysis.
:param imsizes: (optional)
Iterable of image parameters (imsize, pixsize) that should be used for
CLEANing of uv-data if no CLEAN-images are supplied. Should be sorted in
increasing frequency order. If ``None`` then specify parameters by CLEAN
images. (default: ``None``)
:param common_imsize: (optional)
Image parameters that will be used in making common size images for
multifrequency analysis. If ``None`` then use physical image size of
lowest frequency and pixel size of highest frequency. (default:
``None``)
:param outdir: (optional)
Output directory. This directory will be used for saving picture, data,
etc. If ``None`` then use CWD. (default: ``None``)
:param path_to_script: (optional)
Path ot difmap CLEAN script. If ``None`` then use CWD. (default:
``None``)
:notes:
Workflow:
1) Чистка с родным разрешением всех N диапазонов и получение родных
моделей I, Q, U.
2) Выбор общей ДН из N возможных
3) (Опционально) Выбор uv-tapering
4) Чистка uv-данных для всех диапазонов с (опционально применяемым
uv-tapering) общей ДН
5) Оценка сдвига ядра
6) Создание B наборов из N многочастотных симулированных данных
используя родные модели
7) (Опционально) Чистка B наборов из N многочастотных симданных с
родным разрешением для получения карт ошибок I для каждой из N
частот
8) Чистка B наборов из N многочастотных симданных для всех диапазонов с
(опционально применяемым uv-tapering) общей ДН
9) Оценка ошибки определения сдвига ядра
10) Оценка RM и ее ошибки
11) Оценка alpha и ее ошибки
"""
# Fail early
if imsizes is None:
raise Exception("Provide imsizes argument!")
if common_imsize is not None:
print("Using common image size {}".format(common_imsize))
else:
raise Exception("Provide common_imsize argument!")
# Setting up the output directory
if outdir is None:
outdir = os.getcwd()
print("Using output directory {}".format(outdir))
os.chdir(outdir)
# Assume input self-calibrated uv-data FITS files have different frequencies
n_freq = len(uv_fits_paths)
print("Using {} frequencies".format(n_freq))
# Assuming full multifrequency analysis
stokes = ('I', 'Q', 'U')
# Container for original self-calibrated uv-data
uv_data_dict = dict()
# Container for original self-calibrated uv-data FITS-file paths
uv_fits_dict = dict()
for uv_fits_path in uv_fits_paths:
uvdata = UVData(uv_fits_path)
# Mark frequencies by total band center [Hz] for consistency with image.
uv_data_dict.update({uvdata.band_center: uvdata})
uv_fits_dict.update({uvdata.band_center: uv_fits_path})
# Lowest frequency goes first
freqs = sorted(uv_fits_dict.keys())
print("Frequencies are: {}".format(freqs))
# Assert we have original map parameters for all frequencies
assert len(imsizes) == n_freq
# Container for original CLEAN-images of self-calibrated uv-data
cc_image_dict = dict()
# Container for paths to FITS-files with original CLEAN-images of
# self-calibrated uv-data
cc_fits_dict = dict()
# Container for original CLEAN-image's beam parameters
cc_beam_dict = dict()
for freq in freqs:
cc_image_dict.update({freq: dict()})
cc_fits_dict.update({freq: dict()})
cc_beam_dict.update({freq: dict()})
# 1.
# Clean original uv-data with specified map parameters
print("1. Clean original uv-data with specified map parameters...")
imsizes_dict = dict()
for i, freq in enumerate(freqs):
imsizes_dict.update({freq: imsizes[i]})
for freq in freqs:
uv_fits_path = uv_fits_dict[freq]
uv_dir, uv_fname = os.path.split(uv_fits_path)
for stoke in stokes:
outfname = '{}_{}_cc.fits'.format(freq, stoke)
outpath = os.path.join(outdir, outfname)
clean_difmap(uv_fname,
outfname,
stoke,
imsizes_dict[freq],
path=uv_dir,
path_to_script=path_to_script,
outpath=outdir)
cc_fits_dict[freq].update({stoke: os.path.join(outdir, outfname)})
image = create_clean_image_from_fits_file(outpath)
cc_image_dict[freq].update({stoke: image})
if stoke == 'I':
cc_beam_dict.update({freq: image.beam})
# Containers for images and paths to FITS files with common size images
cc_cs_image_dict = dict()
cc_cs_fits_dict = dict()
# 2.
# Choose common beam size
print("2. Choosing common beam size...")
if common_beam is None:
common_beam = cc_beam_dict[freqs[0]]
print("Using common beam [mas, mas, deg] : {}".format(common_beam))
# 3.
# Optionally uv-tapering uv-data
print("3. Optionally uv-tapering uv-data...")
print("skipping...")
# 4.
# Clean original uv-data with common map parameters
print("4. Clean original uv-data with common map parameters...")
for freq in freqs:
cc_cs_image_dict.update({freq: dict()})
cc_cs_fits_dict.update({freq: dict()})
uv_fits_path = uv_fits_dict[freq]
uv_dir, uv_fname = os.path.split(uv_fits_path)
for stoke in stokes:
outfname = 'cs_{}_{}_cc.fits'.format(freq, stoke)
outpath = os.path.join(outdir, outfname)
# clean_difmap(uv_fname_cc, outfname, stoke, common_imsize,
# path=uv_dir, path_to_script=path_to_script,
# outpath=outdir, show_difmap_output=False)
cc_cs_fits_dict[freq].update(
{stoke: os.path.join(outdir, outfname)})
image = create_image_from_fits_file(outpath)
cc_cs_image_dict[freq].update({stoke: image})
# 5.
# Optionally find shifts between original CLEAN-images
print("5. Optionally find shifts between original CLEAN-images...")
if find_shifts:
print("Determining images shift...")
shift_dict = dict()
freq_1 = freqs[0]
image_1 = cc_image_dict[freq_1]['I']
for freq_2 in freqs[1:]:
image_2 = cc_image_dict[freq_2]['I']
# Coarse grid of possible shifts
shift = find_shift(image_1,
image_2,
100,
5,
max_mask_r=200,
mask_step=5)
# More accurate grid of possible shifts
print("Using fine grid for accurate estimate")
coarse_grid = range(0, 100, 5)
idx = coarse_grid.index(shift)
if idx > 0:
min_shift = coarse_grid[idx - 1]
else:
min_shift = 0
shift = find_shift(image_1,
image_2,
coarse_grid[idx + 1],
1,
min_shift=min_shift,
max_mask_r=200,
mask_step=5)
shift_dict.update({str((
freq_1,
freq_2,
)): shift})
# Dumping shifts to json file in target directory
with open(os.path.join(outdir, "shifts_original.json"), 'w') as fp:
json.dump(shift_dict, fp)
else:
print("skipping...")
# 6.
# Bootstrap self-calibrated uv-data with CLEAN-models
print("6. Bootstrap self-calibrated uv-data with CLEAN-models...")
uv_boot_fits_dict = dict()
for freq, uv_fits_path in uv_fits_dict.items():
# cc_fits_paths = [cc_fits_dict[freq][stoke] for stoke in stokes]
# bootstrap_uv_fits(uv_fits_path, cc_fits_paths, n_boot, outpath=outdir,
# outname=('boot_{}'.format(freq), '_uv.fits'))
files = glob.glob(os.path.join(outdir, 'boot_{}*.fits'.format(freq)))
uv_boot_fits_dict.update({freq: sorted(files)})
# 7.
# Optionally clean bootstrap replications with original restoring beams and
# map sizes to get error estimates for original resolution maps of I, PPOL,
# FPOL, ...
print(
"7. Optionally clean bootstrap replications with original restoring"
" beams and map sizes...")
print("skipping...")
# 8.
# Optionally clean bootstrap replications with common restoring beams and
# map sizes
print(
"8. Optionally clean bootstrap replications with common restoring"
" beams and map sizes...")
cc_boot_fits_dict = dict()
for freq in freqs:
cc_boot_fits_dict.update({freq: dict()})
uv_fits_paths = uv_boot_fits_dict[freq]
for stoke in stokes:
for i, uv_fits_path in enumerate(uv_fits_paths):
uv_dir, uv_fname = os.path.split(uv_fits_path)
outfname = 'boot_{}_{}_cc_{}.fits'.format(
freq, stoke,
str(i + 1).zfill(3))
# clean_difmap(uv_fname_cc, outfname, stoke, common_imsize,
# path=uv_dir, path_to_script=path_to_script,
# outpath=outdir, show_difmap_output=False)
files = sorted(
glob.glob(
os.path.join(outdir,
'boot_{}_{}_cc_*.fits'.format(freq, stoke))))
cc_boot_fits_dict[freq].update({stoke: files})
# 9. Optionally estimate RM map and it's error
print("9. Optionally estimate RM map and it's error...")
original_cs_images = Images()
for freq in freqs:
for stoke in stokes:
original_cs_images.add_images(cc_cs_image_dict[freq][stoke])
# Find rough mask for creating bootstrap images of RM, alpha, ...
print("Finding rough mask for creating bootstrap images of RM, alpha, ...")
cs_mask = pol_mask(
{stoke: cc_cs_image_dict[freqs[-1]][stoke]
for stoke in stokes},
n_sigma=3.)
rotm_image, _ = original_cs_images.create_rotm_image(mask=cs_mask)
boot_images = Images()
fnames = sorted(glob.glob(os.path.join(data_dir, "boot_*_*_cc_*.fits")))
for freq in freqs:
for stoke in stokes:
boot_images.add_from_fits(cc_boot_fits_dict[freq][stoke])
boot_rotm_images = boot_images.create_rotm_images(mask=cs_mask)
s_rotm_image = boot_rotm_images.create_error_image(cred_mass=0.95)
if rotm_slices is not None:
fnames = [
'rotm_slice_spread_{}.png'.format(i + 1)
for i in range(len(rotm_slices))
]
for rotm_slice, fname in zip(rotm_slices, fnames):
analyze_rotm_slice(rotm_slice,
rotm_image,
boot_rotm_images,
outdir=outdir,
outfname=fname)
# # Calculate simulataneous confidence bands
# # Bootstrap slices
# slices = list()
# for image in rotm_images_sym.images:
# slice_ = image.slice((216, 276), (296, 276))
# slices.append(slice_[~np.isnan(slice_)])
# # Find means
# obs_slice = rotm_image_sym.slice((216, 276), (296, 276))
# x = np.arange(216, 296, 1)
# x = x[~np.isnan(obs_slice)]
# obs_slice = obs_slice[~np.isnan(obs_slice)]
# # Find sigmas
# slices_ = [arr.reshape((1, len(obs_slice))) for arr in slices]
# sigmas = hdi_of_arrays(slices_).squeeze()
# means = np.mean(np.vstack(slices), axis=0)
# diff = obs_slice - means
# # Move bootstrap curves to original simulated centers
# slices_ = [slice_ + diff for slice_ in slices]
# # Find low and upper confidence band
# low, up = create_sim_conf_band(slices_, obs_slice, sigmas,
# alpha=conf_band_alpha)
# # Plot confidence bands and model values
# fig = plt.figure()
# ax = fig.add_subplot(1, 1, 1)
# ax.plot(x, low[::-1], 'g')
# ax.plot(x, up[::-1], 'g')
# [ax.plot(x, slice_[::-1], 'r', lw=0.15) for slice_ in slices_]
# ax.plot(x, obs_slice[::-1], '.k')
# # Plot ROTM model
# ax.plot(np.arange(216, 296, 1),
# rotm_grad_value * (np.arange(216, 296, 1) - 256.)[::-1] +
# rotm_value_0)
# fig.savefig(os.path.join(data_dir, 'rotm_slice_spread.png'),
# bbox_inches='tight', dpi=200)
# plt.close()
if clear_difmap_logs:
print("Removing difmap log-files...")
difmap_logs = glob.glob(os.path.join(outdir, "difmap.log*"))
for difmpa_log in difmap_logs:
os.unlink(difmpa_log)
