def create_sample(original_uv_fits_path,
original_cc_fits_path=None,
imsize=None,
outdir=None,
path_to_script=None,
n_sample=100,
stokes='I'):
"""
Create `sample` from `true` or `model` source
:param original_uv_fits_path:
Path to original FITS-file with uv-data.
:param original_cc_fits_path: (optional)
Path to original FITS-file with CC model. If ``None`` then use
``imsize`` parameter to get `original` CC model from
``original_uv_fits_path``. (default: ``None``)
:param imsize: (optional)
Image parameters (image size [pix], pixel size [mas]) to use
when doing first CC with ``original_cc_fits_path = None``. (default:
``None``)
:param outdir: (optional)
Directory to store intermediate results. If ``None`` then use CWD.
(default: ``None``)
:param path_to_script: (optional)
Path to Dan Homan's script for final clean. If ``None`` then use CWD.
(default: ``None``)
:param n_sample: (optional)
Number of `samples` from infinite population to consider in coverage
analysis of intervals. Here `samples` - observations of known source
with different realisations of noise with known parameters. (default:
``100``)
:param stokes: (optional)
Stokes parameter to use. If ``None`` then use ``I``. (default: ``None``)
:return:
Creates FITS-files with uv-data and CLEAN models of `sample`.
"""
# If not given `original` CLEAN model - get it by cleaning `original`
# uv-data
if original_cc_fits_path is None:
print(
"No `original` CLEAN model specified! Will CLEAN `original`"
" uv-data.")
if imsize is None:
raise Exception("Specify ``imsize``")
uv_fits_dir, uv_fits_fname = os.path.split(original_uv_fits_path)
original_cc_fits_path = os.path.join(outdir, 'original_cc.fits')
print(
"Cleaning `original` uv-data to {}".format(original_cc_fits_path))
clean_difmap(uv_fits_fname,
'original_cc.fits',
stokes,
imsize,
path=uv_fits_dir,
path_to_script=path_to_script,
outpath=outdir)
original_uv_data = UVData(original_uv_fits_path)
noise = original_uv_data.noise()
original_model = create_model_from_fits_file(original_cc_fits_path)
# Find images parameters for cleaning if necessary
if imsize is None:
print(
"Getting image parameters from `original`"
" CLEAN FITS file {}.".format(original_cc_fits_path))
image_params = get_fits_image_info(original_cc_fits_path)
imsize = (image_params['imsize'][0],
abs(image_params['pixsize'][0]) / mas_to_rad)
# Substitute uv-data with original model and create `model` uv-data
print("Substituting `original` uv-data with CLEAN model...")
model_uv_data = copy.deepcopy(original_uv_data)
model_uv_data.substitute([original_model])
# Create `sample` uv-data
# Add noise to `model` uv-data ``n_cov`` times and get ``n_cov`` `samples`
# from population
sample_uv_fits_paths = list()
print("Creating {} `samples` from population".format(n_sample))
for i in range(n_sample):
sample_uv_data = copy.deepcopy(model_uv_data)
sample_uv_data.noise_add(noise)
sample_uv_fits_path = os.path.join(
outdir, 'sample_uv_{}.uvf'.format(str(i + 1).zfill(3)))
sample_uv_data.save(sample_uv_fits_path)
sample_uv_fits_paths.append(sample_uv_fits_path)
# Clean each `sample` FITS-file
print("CLEANing `samples` uv-data")
for uv_fits_path in sample_uv_fits_paths:
uv_fits_dir, uv_fits_fname = os.path.split(uv_fits_path)
j = uv_fits_fname.split('.')[0].split('_')[-1]
print("Cleaning {} sample uv-data to"
" {}".format(uv_fits_path,
os.path.join(outdir,
'sample_cc_{}.fits'.format(j))))
clean_difmap(uv_fits_fname,
'sample_cc_{}.fits'.format(j),
original_model.stokes,
imsize,
path=uv_fits_dir,
path_to_script=path_to_script,
outpath=outdir)
sample_cc_fits_paths = sorted(
glob.glob(os.path.join(outdir, 'sample_cc_*.fits')))
sample_uv_fits_paths = sorted(
glob.glob(os.path.join(outdir, 'sample_uv_*.uvf')))
return sample_uv_fits_paths, sample_cc_fits_paths
def create_sample(original_uv_fits,
original_mdl_file,
outdir=None,
n_sample=100,
stokes='I'):
"""
Create `sample` from `true` or `model` source
:param outdir: (optional)
Directory to store intermediate results. If ``None`` then use CWD.
(default: ``None``)
:param n_sample: (optional)
Number of `samples` from infinite population to consider in coverage
analysis of intervals. Here `samples` - observations of known source
with different realisations of noise with known parameters. (default:
``100``)
:param stokes: (optional)
Stokes parameter to use. If ``None`` then use ``I``. (default: ``None``)
"""
original_uv_data = UVData(original_uv_fits)
noise = original_uv_data.noise()
path, _ = os.path.split(original_mdl_file)
comps = import_difmap_model(original_mdl_file, path)
original_model = Model(stokes=stokes)
original_model.add_components(*comps)
# Substitute uv-data with original model and create `model` uv-data
print("Substituting `original` uv-data with CLEAN model...")
model_uv_data = copy.deepcopy(original_uv_data)
model_uv_data.substitute([original_model])
# Create `sample` uv-data
# Add noise to `model` uv-data ``n_cov`` times and get ``n_cov`` `samples`
# from population
sample_uv_fits_paths = list()
print("Creating {} `samples` from population".format(n_sample))
for i in range(n_sample):
sample_uv_data = copy.deepcopy(model_uv_data)
sample_uv_data.noise_add(noise)
sample_uv_fits_path = os.path.join(
outdir, 'sample_uv_{}.uvf'.format(str(i + 1).zfill(3)))
sample_uv_data.save(sample_uv_fits_path)
sample_uv_fits_paths.append(sample_uv_fits_path)
# Fitting in difmap each `sample` FITS-file
print("Fitting `samples` uv-data")
for uv_fits_path in sample_uv_fits_paths:
uv_fits_dir, uv_fits_fname = os.path.split(uv_fits_path)
j = uv_fits_fname.split('.')[0].split('_')[-1]
print("Fitting {} sample uv-data to"
" {}".format(
uv_fits_path,
os.path.join(outdir, 'sample_model_{}.mdl'.format(j))))
modelfit_difmap(uv_fits_fname,
original_mdl_file,
'sample_model_{}.mdl'.format(j),
path=uv_fits_dir,
mdl_path=uv_fits_dir,
out_path=uv_fits_dir)
sample_mdl_paths = sorted(
glob.glob(os.path.join(outdir, 'sample_model_*.mdl')))
sample_uv_fits_paths = sorted(
glob.glob(os.path.join(outdir, 'sample_uv_*.uvf')))
return sample_uv_fits_paths, sample_mdl_paths
# download_mojave_uv_fits(source, epochs=[epoch], bands=['u'],
# download_dir=data_dir)
uv_fits_fnames = {
freq: mojave_uv_fits_fname(source, freq, epoch)
for freq in ('x', 'j', 'u')
}
for freq, uv_fits_fname in uv_fits_fnames.items():
uv_fits_path = os.path.join(data_dir, uv_fits_fname)
cg1 = CGComponent(2.0, 0., 0., 0.2)
cg2 = CGComponent(1.0, 0., 0.3, 0.3)
cg3 = CGComponent(0.5, 0., 1.5, 0.4)
mdl = Model(stokes='I')
mdl.add_components(cg1, cg2, cg3)
uvdata = UVData(uv_fits_path)
noise = uvdata.noise()
for i in range(1, 101):
uvdata = UVData(uv_fits_path)
uvdata.substitute([mdl])
uvdata.noise_add(noise)
art_fits_fname = 'art_{}_{}.fits'.format(freq, i)
art_fits_path = os.path.join(data_dir, art_fits_fname)
uvdata.save(art_fits_path)
# Here we should MCMC posterior
modelfit_difmap(art_fits_fname,
'initial.mdl',
'out_{}_{}.mdl'.format(freq, i),
niter=100,
path=data_dir,
mdl_path=data_dir,
def create_coverage_map(original_uv_fits_path,
ci_type,
original_cc_fits_path=None,
imsize=None,
outdir=None,
n_boot=200,
path_to_script=None,
alpha=0.68,
n_cov=100,
n_rms=1.,
stokes='I',
boot_cc_fits_paths=None,
sample_cc_fits_paths=None):
"""
Conduct coverage analysis of image pixels flux CI. Find number of times
when CI of `observed` value contains values of `samples`.
:param original_uv_fits_path:
Path to original FITS-file with uv-data.
:param ci_type:
Type of CI to test. ``boot`` or ``rms``. If ``boot`` then use residuals
bootstrap CI. If ``rms`` then use Hovatta corrected image rms CI.
:param original_cc_fits_path: (optional)
Path to original FITS-file with CC model. If ``None`` then use
``imsize`` parameter to get `original` CC model from
``original_uv_fits_path``. (default: ``None``)
:param imsize: (optional)
Image parameters (image size [pix], pixel size [mas]) to use
when doing first CC with ``original_cc_fits_path = None``. (default:
``None``)
:param outdir: (optional)
Directory to store intermediate results. If ``None`` then use CWD.
(default: ``None``)
:param n_boot: (optional)
Number of bootstrap replications to use when calculating bootstrap CI
for ``ci_type = boot`` option when ``boot_cc_fits_paths`` hasn't
specified. (default: ``200``)
:param path_to_script: (optional)
Path to Dan Homan's script for final clean. If ``None`` then use CWD.
(default: ``None``)
:param alpha: (optional)
Level of significance when calculating bootstrap CI for ``ci_type =
boot`` case. E.g. ``0.68`` corresponds to `1 \sigma`. (default:
``0.68``)
:param n_cov: (optional)
Number of `samples` from infinite population to consider in coverage
analysis of intervals. Here `samples` - observations of known source
with different realisations of noise with known parameters. (default:
``100``)
:param n_rms: (optional)
Number of rms to use in ``ci_type = rms`` case. (default: ``1.``)
:param stokes: (optional)
Stokes parameter to use. If ``None`` then use ``I``. (default: ``None``)
:param boot_cc_fits_paths: (optional)
If ``ci_type = boot`` then this parameter could specify paths to cleaned
bootstrapped uv-data.
:param sample_cc_fits_paths: (optional)
Path to FITS-files with CLEAN models of `sample` uv-data. If ``None``
then create ``n_cov`` `sample` uv-data from noise of `original` uv-data
and `original` CLEAN model. (default: ``None``)
:return:
Coverage map. Each pixel contain frequency of times when samples from
population hit inside CI for given pixel.
"""
# If not given `original` CLEAN model - get it by cleaning `original`
# uv-data
if original_cc_fits_path is None:
print(
"No `original` CLEAN model specified! Will CLEAN `original`"
" uv-data.")
if imsize is None:
raise Exception("Specify ``imsize``")
uv_fits_dir, uv_fits_fname = os.path.split(original_uv_fits_path)
print("Cleaning `original` uv-data to"
" {}".format(os.path.join(outdir, 'cc.fits')))
clean_difmap(uv_fits_fname,
'cc.fits',
stokes,
imsize,
path=uv_fits_dir,
path_to_script=path_to_script,
outpath=outdir)
original_cc_fits_path = os.path.join(outdir, 'cc.fits')
original_uv_data = UVData(original_uv_fits_path)
noise = original_uv_data.noise()
original_model = create_model_from_fits_file(original_cc_fits_path)
# Find images parameters for cleaning if necessary
if imsize is None:
print(
"Getting image parameters from `original`"
" CLEAN FITS file {}.".format(original_cc_fits_path))
image_params = get_fits_image_info(original_cc_fits_path)
imsize = (image_params['imsize'][0],
abs(image_params['pixsize'][0]) / mas_to_rad)
# Substitute uv-data with original model and create `model` uv-data
print("Substituting original uv-data with CLEAN model...")
model_uv_data = copy.deepcopy(original_uv_data)
model_uv_data.substitute([original_model])
# Add noise to `model` uv-data to get `observed` uv-data
observed_uv_data = copy.deepcopy(model_uv_data)
observed_uv_data.noise_add(noise)
observed_uv_fits_path = os.path.join(outdir, 'observed_uv.uvf')
if os.path.isfile(observed_uv_fits_path):
os.unlink(observed_uv_fits_path)
print("Adding noise to `model` uv-data to get `observed` uv-data...")
observed_uv_data.save(fname=observed_uv_fits_path)
observed_cc_fits_path = os.path.join(outdir, 'observed_cc.fits')
if os.path.isfile(observed_cc_fits_path):
os.unlink(observed_cc_fits_path)
# Clean `observed` uv-data to get `observed` image and model
print("Cleaning `observed` uv-data to `observed` CLEAN model...")
clean_difmap('observed_uv.uvf',
'observed_cc.fits',
original_model.stokes,
imsize,
path=outdir,
path_to_script=path_to_script,
outpath=outdir)
# Get `observed` model and image
observed_model = create_model_from_fits_file(observed_cc_fits_path)
observed_image = create_image_from_fits_file(observed_cc_fits_path)
# Testing coverage of bootstrapped CI
if ci_type == 'boot':
# Bootstrap and clean only when necessary
if boot_cc_fits_paths is None:
# Bootstrap `observed` uv-data with `observed` model
boot = CleanBootstrap([observed_model], observed_uv_data)
cwd = os.getcwd()
path_to_script = path_to_script or cwd
os.chdir(outdir)
print("Bootstrapping uv-data with {} replications".format(n_boot))
boot.run(outname=['observed_uv_boot', '.uvf'], n=n_boot)
os.chdir(cwd)
boot_uv_fits_paths = sorted(
glob.glob(os.path.join(outdir, 'observed_uv_boot*.uvf')))
# Clean each bootstrapped uv-data
for i, uv_fits_path in enumerate(boot_uv_fits_paths):
uv_fits_dir, uv_fits_fname = os.path.split(uv_fits_path)
print("Cleaning {} bootstrapped observed"
" uv-data to {}".format(
uv_fits_path,
os.path.join(
outdir,
'observed_cc_boot_{}.fits'.format(i + 1))))
clean_difmap(uv_fits_fname,
'observed_cc_boot_{}.fits'.format(i + 1),
original_model.stokes,
imsize,
path=uv_fits_dir,
path_to_script=path_to_script,
outpath=outdir)
boot_cc_fits_paths = glob.glob(
os.path.join(outdir, 'observed_cc_*.fits'))
# Calculate bootstrap CI
# hdi_low, hdi_high = boot_ci_bc(boot_cc_fits_paths,
# observed_cc_fits_path, alpha=alpha)
hdi_low, hdi_high = boot_ci(boot_cc_fits_paths,
observed_cc_fits_path,
alpha=alpha)
elif ci_type == 'rms':
# Calculate ``n_rms`` CI
rms = observed_image.rms(region=(50, 50, 50, None))
rms = np.sqrt(rms**2. + (1.5 * rms**2.)**2.)
hdi_low = observed_image.image - rms
hdi_high = observed_image.image + rms
else:
raise Exception("CI intervals must be `boot` or `rms`!")
# Create `sample` uv-data and clean it only when necessary
if sample_cc_fits_paths is None:
# Add noise to `model` uv-data ``n_cov`` times and get ``n_cov``
# `samples` from population
sample_uv_fits_paths = list()
for i in range(n_cov):
sample_uv_data = copy.deepcopy(model_uv_data)
sample_uv_data.noise_add(noise)
sample_uv_fits_path = os.path.join(outdir,
'samle_uv_{}.uvf'.format(i + 1))
sample_uv_data.save(sample_uv_fits_path)
sample_uv_fits_paths.append(sample_uv_fits_path)
# Clean each `sample` FITS-file
for i, uv_fits_path in enumerate(sample_uv_fits_paths):
uv_fits_dir, uv_fits_fname = os.path.split(uv_fits_path)
print("Cleaning {} sample uv-data to"
" {}".format(
uv_fits_path,
os.path.join(outdir, 'sample_cc_{}.fits'.format(i + 1))))
clean_difmap(uv_fits_fname,
'sample_cc_{}.fits'.format(i + 1),
original_model.stokes,
imsize,
path=uv_fits_dir,
path_to_script=path_to_script,
outpath=outdir)
sample_cc_fits_paths = glob.glob(
os.path.join(outdir, 'sample_cc_*.fits'))
sample_images = list()
for sample_cc_fits_path in sample_cc_fits_paths:
image = create_image_from_fits_file(sample_cc_fits_path)
sample_images.append(image.image)
# For each pixel check how often flux in `sample` images lies in CI derived
# for observed image.
cov_array = np.zeros((imsize[0], imsize[0]), dtype=float)
print("calculating CI intervals")
for (x, y), value in np.ndenumerate(cov_array):
for image in sample_images:
cov_array[x, y] += float(
np.logical_and(hdi_low[x, y] < image[x, y],
image[x, y] < hdi_high[x, y]))
return cov_array / n_cov
import os
import copy
from astropy.io import fits as pf
from uv_data import UVData
from model import Model
from components import EGComponent, CGComponent
from spydiff import modelfit_difmap, import_difmap_model
data_dir = '/home/ilya/code/vlbi_errors/tests/ft'
uv_fits = '1308+326.U1.2009_08_28.UV_CAL'
hdus_orig = pf.open(os.path.join(data_dir, uv_fits))
print "orig", hdus_orig[0].data[0][0]
uvdata = UVData(os.path.join(data_dir, uv_fits))
noise = uvdata.noise(use_V=False)
# noise = {bl: 0.1*noise_ for bl, noise_ in noise.items()}
eg1 = EGComponent(5., 0, 0, 0.15, 0.33, 0.2)
eg2 = EGComponent(2.5, 1, 1, 0.5, 0.5, 0.)
model = Model(stokes='I')
model.add_components(eg1, eg2)
# model.add_components(eg1, eg2)
uvdata_c = copy.deepcopy(uvdata)
uvdata_c.substitute([model])
uvdata_c.noise_add(noise)
uvdata_c.save(os.path.join(data_dir, 'fake.fits'), rewrite=True)
modelfit_difmap('fake.fits',
'mod_c2_2ee.mdl',
'out_2c.mdl',
path=data_dir,
mdl_path=data_dir,
out_path=data_dir,
niter=100)
# print([cg.p for cg in new_dfm_model])
# print([cg.p for cg in original_dfm_model])
# Create template model file
# export_difmap_model([cg], os.path.join(data_dir, "template.mdl"), uvdata_template.frequency/10**9)
# Modelfit artificial self-calibrated data
# cg = CGComponent(0.5, 0, 0, 0.5)
# model = Model(stokes=stokes)
# model.add_components(*new_dfm_model)
# model.add_components(ccmodel)
if stokes == "I":
use_V = True
else:
use_V = False
noise = uvdata_template.noise(use_V=use_V)
params = list()
for i in range(n_mc):
uvdata_template.substitute([ccmodel])
uvdata_template.uvdata = uvdata_template.uvdata * corrections
uvdata_template.noise_add(noise)
uvdata_template.save(os.path.join(data_dir, "artificial.uvf"),
rewrite=True,
downscale_by_freq=True)
# Self-calibrate
selfcal_difmap(
fname="artificial.uvf",
outfname="artificial.uvf",
# image += gaussian_(x, y)
# image[image < cut] = 0.
import matplotlib.pyplot as plt
# plt.matshow(np.log(image))
# plt.show()
from model import Model
from components import ImageComponent, CGComponent
from from_fits import create_image_from_fits_file
import os
from uv_data import UVData
original_uvdata = UVData(
os.path.join('/home/ilya/Dropbox/0235/VLBA/',
'0235+164.q1.2008_09_02.uvf_difmap'))
noise = original_uvdata.noise()
noise = {key: 0.25 * value for key, value in noise.items()}
image_stack = create_image_from_fits_file(
os.path.join('/home/ilya/Dropbox/0235/VLBA/', '0235+164.q1.stack.fits'))
pixsize = abs(image_stack.pixsize[0])
imsize = image_stack.imsize
model = Model(stokes='I')
model.from_2darray(image, (0.01, 0.01))
# model.add_component(ImageComponent(image, x[0,:],
# y[:,0]))
# model.add_component(CGComponent(2.25, 0., 0., 0.01))
import copy
model_uvdata = copy.deepcopy(original_uvdata)
model_uvdata.substitute([model])
model_uvdata.noise_add(noise)
#fig = original_uvdata.uvplot()
transfer = pickle.load(fo)
# 2D coordinates (mas)
x = transfer.pixsize[0]*transfer.image_coordinates[..., 0]
y = transfer.pixsize[1]*transfer.image_coordinates[..., 1]
x *= mas_to_rad
y *= mas_to_rad
image = transfer.image()
icomp = ImageComponent(image, x[0], y[..., 0])
uvdata = UVData(os.path.join(data_dir, '0952+179.U1.2007_04_30.PINAL'))
# uvdata = UVData('/home/ilya/Dropbox/ACC/3c120/uvdata/0430+052.u.2006_05_24.uvf')
# uvdata = UVData('/home/ilya/Dropbox/ACC/3c120/uvdata/0430+052.u.2006_05_24.uvf')
model = Model(stokes='I')
model.add_component(icomp)
noise = uvdata.noise(use_V=True)
uvdata.substitute([model])
for bl in noise:
noise[bl] *= 10
uvdata.noise_add(noise)
uvdata.save(os.path.join(data_dir, '0952_15GHz_BK.fits'))
# clean_difmap('15GHz_BK.fits', 'u_BK_cc.fits', 'I', (1024, 0.1), path=data_dir,
# path_to_script=path_to_script, show_difmap_output=True,
# outpath=data_dir)
ccimage = create_clean_image_from_fits_file(os.path.join(data_dir, '0952_15GHz_BK_cc.fits'))
beam = ccimage.beam
rms = rms_image(ccimage)
blc, trc = find_bbox(ccimage.image, rms, 10)
iplot(ccimage.image, x=ccimage.x, y=ccimage.y, min_abs_level=3*rms, beam=beam,
corrections = uvdata_raw.uvdata/uvdata_sc.uvdata
# Create artificial raw data with known sky model and given corrections
original_dfm_model = import_difmap_model(os.path.join(data_dir, "2019_08_27.mod"))
modelfit_difmap("myselfcaled.uvf", "2019_08_27.mod", "artificial.mdl", niter=100, stokes='I',
path=data_dir, mdl_path=data_dir, out_path=data_dir, show_difmap_output=True)
new_dfm_model = import_difmap_model("artificial.mdl", data_dir)
print([cg.p for cg in new_dfm_model])
print([cg.p for cg in original_dfm_model])
# cg = CGComponent(0.5, 0, 0, 0.5)
model = Model(stokes="I")
model.add_components(*new_dfm_model)
noise = uvdata_template.noise(use_V=True)
params = list()
for i in range(30):
uvdata_template.substitute([model])
# uvdata_template.uvdata = uvdata_template.uvdata*corrections
uvdata_template.noise_add(noise)
uvdata_template.save(os.path.join(data_dir, "artificial.uvf"), rewrite=True, downscale_by_freq=True)
# uvdata_artificial_raw = UVData(os.path.join(data_dir, "artificial.uvf"))
# Self-calibrate
selfcal_difmap(fname="artificial.uvf", outfname="artificial.uvf",
path=data_dir, path_to_script="/home/ilya/github/ve/difmap/auto_selfcal", outpath=data_dir,
show_difmap_output=True)
