import copy
import matplotlib.pyplot as plt
import os
import numpy as np
from spydiff import import_difmap_model, modelfit_difmap, clean_difmap
from uv_data import UVData
from model import Model
from from_fits import create_clean_image_from_fits_file
from mojave import (download_mojave_uv_fits, get_mojave_mdl_file,
mojave_uv_fits_fname)
data_dir = '/home/ilya/vlbi_errors/model_cov/check_image_add'
tsv_table = os.path.join(data_dir, 'asu.tsv')
source = '2230+114'
epoch = '2005-02-05'
uv_fits = mojave_uv_fits_fname(source, 'u', epoch.replace('-', '_'))
path_to_script = '/home/ilya/github/vlbi_errors/difmap/final_clean_nw'
mdl_fname = '{}_{}.mdl'.format(source, epoch)
# Fetch uv-fits
download_mojave_uv_fits(source, [epoch.replace('-', '_')],
bands=['u'],
download_dir=data_dir)
# Fetch model file
get_mojave_mdl_file(tsv_table, source, epoch, outdir=data_dir)
# Clean uv-fits
clean_difmap(uv_fits,
'cc.fits',
'I', [1024, 0.1],
path=data_dir,
path_to_script=path_to_script,
def coverage_of_model(original_uv_fits,
original_mdl_file,
outdir=None,
n_cov=100,
n_boot=300,
mapsize=(1024, 0.1),
path_to_script=None):
"""
Conduct coverage analysis of uv-data & model
:param original_uv_fits:
Self-calibrated uv-fits file.
:param original_mdl_file:
Difmap txt-file with model.
:param outdir:
Output directory to store results.
:param n_cov:
Number of samples to create.
"""
# Create sample of 100 uv-fits data & models
sample_uv_fits_paths, sample_model_paths = create_sample(original_uv_fits,
original_mdl_file,
outdir=outdir,
n_sample=n_cov)
# For each sample uv-fits & model find 1) conventional errors & 2) bootstrap
# errors
for j, (sample_uv_fits_path, sample_mdl_path) in enumerate(
zip(sample_uv_fits_paths, sample_model_paths)):
sample_uv_fits, dir = os.path.split(sample_uv_fits_path)
sample_mdl_file, dir = os.path.split(sample_mdl_path)
try:
comps = import_difmap_model(sample_mdl_file, dir)
except ValueError:
print('Problem import difmap model')
model = Model(stokes='I')
model.add_components(*comps)
# Find errors by using Fomalont way
# 1. Clean uv-data
clean_difmap(sample_uv_fits,
'sample_cc_{}.fits'.format(j),
'I',
mapsize,
path=dir,
path_to_script=path_to_script,
outpath=dir)
# 2. Get beam
ccimage = create_clean_image_from_fits_file(
os.path.join(dir, 'sample_cc_{}.fits'.format(j)))
beam = ccimage.beam_image
# 2. Subtract components convolved with beam
ccimage.substract_model(model)
# Find errors by using Lee way
# a) fit uv-data and find model
# b) CLEAN uv-data
# c) substract model from CLEAN image
# d) find errors
pass
# Find errors by using bootstrap
# FT model to uv-plane
uvdata = UVData(sample_uv_fits_path)
try:
boot = CleanBootstrap([model], uvdata)
# If uv-data contains only one Stokes parameter (e.g. `0838+133`)
except IndexError:
print('Problem bootstrapping')
curdir = os.getcwd()
os.chdir(dir)
boot.run(n=n_boot, nonparametric=True, outname=[outname, '.fits'])
os.chdir(curdir)
booted_uv_paths = sorted(
glob.glob(os.path.join(data_dir, outname + "*")))
# Modelfit bootstrapped uvdata
for booted_uv_path in booted_uv_paths:
path, booted_uv_file = os.path.split(booted_uv_path)
i = booted_uv_file.split('_')[-1].split('.')[0]
modelfit_difmap(booted_uv_file,
dfm_model_fname,
dfm_model_fname + '_' + i,
path=path,
mdl_path=data_dir,
out_path=data_dir)
# Get params of initial model used for bootstrap
comps = import_difmap_model(dfm_model_fname, data_dir)
comps_params0 = {i: [] for i in range(len(comps))}
for i, comp in enumerate(comps):
comps_params0[i].extend(list(comp.p))
# Load bootstrap models
booted_mdl_paths = glob.glob(
os.path.join(data_dir, dfm_model_fname + "_*"))
comps_params = {i: [] for i in range(len(comps))}
for booted_mdl_path in booted_mdl_paths:
path, booted_mdl_file = os.path.split(booted_mdl_path)
comps = import_difmap_model(booted_mdl_file, path)
for i, comp in enumerate(comps):
comps_params[i].extend(list(comp.p))
# Print 65-% intervals (1 sigma)
for i, comp in enumerate(comps):
errors_fname = '68_{}_{}_comp{}.txt'.format(source, last_epoch, i)
fn = open(os.path.join(data_dir, errors_fname), 'w')
print "Component #{}".format(i + 1)
for j in range(len(comp)):
low, high, mean, median = hdi_of_mcmc(np.array(
comps_params[i]).reshape((n_boot, len(comp))).T[j],
cred_mass=0.68,
return_mean_median=True)
fn.write("{} {} {} {} {}".format(comp.p[j], low, high, mean,
median))
fn.write("\n")
fn.close()
# For source in sources with component close to core
# 1. Find residuals or estimate noise
# 2. N times add resampled residuals (or just gaussian noise) to model and
# create N new datasets
# 3. Fit them using difmap.
# 4. Find errors using Fomalont, Yee and using bootstrap. Check coverage.
base_dir = '/home/ilya/vlbi_errors/model_cov'
n_boot = 300
outname = 'boot_uv'
names = [
'source', 'id', 'trash', 'epoch', 'flux', 'r', 'pa', 'bmaj', 'e', 'bpa'
]
df = pd.read_table(os.path.join(base_dir, 'asu.tsv'),
sep=';',
header=None,
names=names,
dtype={key: str
for key in names},
index_col=False)
# Mow for all sources get the latest epoch and create directory for analysis
for source in df['source'].unique():
epochs = df.loc[df['source'] == source]['epoch']
last_epoch_ = list(epochs)[-1]
last_epoch = last_epoch_.replace('-', '_')
data_dir = os.path.join(base_dir, source, last_epoch)
if not os.path.exists(data_dir):
os.makedirs(data_dir)
try:
download_mojave_uv_fits(source,
epochs=[last_epoch],
bands=['u'],
download_dir=data_dir)
except:
open(
'problem_download_from_mojave_{}_{}'.format(
source, last_epoch), 'a').close()
continue
uv_fits_fname = mojave_uv_fits_fname(source, 'u', last_epoch)
# Create instance of Model and bootstrap uv-data
dfm_model_fname = 'dfmp_original_model.mdl'
fn = open(os.path.join(data_dir, dfm_model_fname), 'w')
model_df = df.loc[np.logical_and(df['source'] == source,
df['epoch'] == last_epoch_)]
for (flux, r, pa, bmaj, e, bpa) in np.asarray(
model_df[['flux', 'r', 'pa', 'bmaj', 'e', 'bpa']]):
print flux, r, pa, bmaj, e, bpa
if not r.strip(' '):
r = '0.0'
if not pa.strip(' '):
pa = '0.0'
if not bmaj.strip(' '):
bmaj = '0.0'
if not e.strip(' '):
e = "1.0"
if np.isnan(float(bpa)):
bpa = "0.0"
else:
bpa = bpa + 'v'
if bmaj == '0.0':
type_ = 0
bpa = "0.0"
else:
bmaj = bmaj + 'v'
type_ = 1
fn.write("{}v {}v {}v {} {} {} {} {} {}".format(
flux, r, pa, bmaj, e, bpa, type_, "0", "0\n"))
fn.close()
def download_mojave_models(base_dir):
"""
Download MOJAVE models & uv-fits files.
:param base_dir:
Directory with MOJAVE table.
"""
names = [
'source', 'id', 'trash', 'epoch', 'flux', 'r', 'pa', 'bmaj', 'e', 'bpa'
]
df = pd.read_table(os.path.join(base_dir, 'asu.tsv'),
sep=';',
header=None,
names=names,
dtype={key: str
for key in names},
index_col=False)
# Mow for all sources get the latest epoch and create directory for analysis
for source in df['source'].unique():
epochs = df.loc[df['source'] == source]['epoch']
last_epoch_ = list(epochs)[-1]
last_epoch = last_epoch_.replace('-', '_')
data_dir = os.path.join(base_dir, source, last_epoch)
if not os.path.exists(data_dir):
os.makedirs(data_dir)
try:
download_mojave_uv_fits(source,
epochs=[last_epoch],
bands=['u'],
download_dir=data_dir)
except:
open(
'problem_download_from_mojave_{}_{}'.format(
source, last_epoch), 'a').close()
continue
uv_fits_fname = mojave_uv_fits_fname(source, 'u', last_epoch)
# Create instance of Model and bootstrap uv-data
dfm_model_fname = 'dfmp_original_model.mdl'
fn = open(os.path.join(data_dir, dfm_model_fname), 'w')
model_df = df.loc[np.logical_and(df['source'] == source,
df['epoch'] == last_epoch_)]
for (flux, r, pa, bmaj, e, bpa) in np.asarray(
model_df[['flux', 'r', 'pa', 'bmaj', 'e', 'bpa']]):
print flux, r, pa, bmaj, e, bpa
if not r.strip(' '):
r = '0.0'
if not pa.strip(' '):
pa = '0.0'
if not bmaj.strip(' '):
bmaj = '0.0'
if not e.strip(' '):
e = "1.0"
if np.isnan(float(bpa)):
bpa = "0.0"
else:
bpa = bpa + 'v'
if bmaj == '0.0':
type_ = 0
bpa = "0.0"
else:
bmaj = bmaj + 'v'
type_ = 1
fn.write("{}v {}v {}v {} {} {} {} {} {}".format(
flux, r, pa, bmaj, e, bpa, type_, "0", "0\n"))
fn.close()
if len(source_dict[source]) > 5:
epoch = source_dict[source][0][5]
source_dir = os.path.join(base_dir, source, epoch)
if not os.path.exists(source_dir):
os.makedirs(source_dir)
get_mojave_mdl_file(os.path.join(base_dir, 'asu.tsv'),
source,
epoch,
outdir=source_dir)
epoch_ = "{}_{}_{}".format(*epoch.split('-'))
download_mojave_uv_fits(source,
epochs=[epoch_],
download_dir=source_dir,
bands=['u'])
fname = mojave_uv_fits_fname(source, 'u', epoch_)
uvdata = UVData(os.path.join(source_dir, fname))
print(uvdata.stokes)
if 'RR' not in uvdata.stokes or 'LL' not in uvdata.stokes:
continue
# Refit difmap model
modelfit_difmap(fname,
"{}_{}.mdl".format(source, epoch),
"{}_{}.mdl".format(source, epoch),
niter=200,
path=source_dir,
mdl_path=source_dir,
out_path=source_dir,
show_difmap_output=True)
raise Exception("ci_type must be rms or boot!")
data_dir = os.path.join(base_dir, source, stokes, ci_type, str(perc))
print "Calculating {} CI with nominal coverage {}".format(
ci_type, perc)
print "Using directory {}".format(data_dir)
if not os.path.exists(data_dir):
os.makedirs(data_dir)
download_mojave_uv_fits(source,
epochs=epochs,
bands=bands,
download_dir=data_dir)
# Classic coverage analysis
fname = mojave_uv_fits_fname(source, bands[0], epochs[0])
original_uv_fits_path = os.path.join(data_dir, fname)
# sample_uv_fits_paths, sample_cc_fits_paths =\
# create_sample(original_uv_fits_path, imsize=imsize, outdir=data_dir,
# path_to_script=path_to_script)
sample_cc_fits_paths = sorted(
glob.glob(os.path.join(data_dir, 'sample_cc_*.fits')))
# sample_cc_fits_paths = None
sample_uv_fits_paths = sorted(
glob.glob(os.path.join(data_dir, 'sample_uv_*.uvf')))
# sample_uv_fits_paths = None
original_cc_fits_path = os.path.join(data_dir, 'original_cc.fits')
# original_cc_fits_path = None
coverage_map =\
create_coverage_map_classic(original_uv_fits_path, ci_type=ci_type,
original_cc_fits_path=original_cc_fits_path,
import corner
from components import CGComponent
from model import Model
from uv_data import UVData
from mojave import mojave_uv_fits_fname, download_mojave_uv_fits
from spydiff import modelfit_difmap, import_difmap_model
from bootstrap import bootstrap_uvfits_with_difmap_model
source = '1514-241'
epoch = '2006_04_28'
data_dir = '/home/ilya/Dropbox/papers/boot/bias/new/stationary'
# download_mojave_uv_fits(source, epochs=[epoch], bands=['u'],
# download_dir=data_dir)
uv_fits_fnames = {
freq: mojave_uv_fits_fname(source, freq, epoch)
for freq in ('x', 'j', 'u')
}
for freq, uv_fits_fname in uv_fits_fnames.items():
uv_fits_path = os.path.join(data_dir, uv_fits_fname)
cg1 = CGComponent(2.0, 0., 0., 0.2)
cg2 = CGComponent(1.0, 0., 0.3, 0.3)
cg3 = CGComponent(0.5, 0., 1.5, 0.4)
mdl = Model(stokes='I')
mdl.add_components(cg1, cg2, cg3)
uvdata = UVData(uv_fits_path)
noise = uvdata.noise()
for i in range(1, 101):
uvdata = UVData(uv_fits_path)
uvdata.substitute([mdl])
uvdata.noise_add(noise)
# FIXME: Cutting IFs results in different uvw
import os
import numpy as np
from mojave import mojave_uv_fits_fname
from spydiff import import_difmap_model, modelfit_difmap
from model import Model
from uv_data import UVData
import astropy.io.fits as pf
data_dir = '/home/ilya/silke/if'
boot_dir = '/home/ilya/silke/if/boot'
epoch = '2017_01_28'
uv_fits_fname = mojave_uv_fits_fname('0851+202', 'u', epoch)
uv_fits_path = os.path.join(data_dir, uv_fits_fname)
# # Create sequence of FITS files with different number of bands - from 1 to 7
# for i in range(1, 8):
# uvdata = UVData(uv_fits_path)
# imdata = uvdata.hdu.data['DATA'][:, :, :, :i, :, :, :]
# pardata = [uvdata.hdu.data[key] for key in uvdata.hdu.data.parnames]
# pardata[5] = np.zeros(len(pardata[4]))
# pardata[0] *= uvdata.frequency
# pardata[1] *= uvdata.frequency
# pardata[2] *= uvdata.frequency
# pscales = [uvdata.hdu.header['PSCAL{}'.format(i)] for i in
# range(1, len(uvdata.hdu.parnames)+1)]
# pzeros = [uvdata.hdu.header['PZERO{}'.format(i)] for i in
# range(1, len(uvdata.hdu.parnames)+1)]
# x = pf.GroupData(imdata, parnames=uvdata.hdu.data.parnames, pardata=pardata,
# bitpix=-32, parbzeros=pzeros)
# for epoch_ready in epochs_ready:
# dir, fname = os.path.split(epoch_ready)
# epochs_ready_.append(fname[7:-4])
# epochs_ready = ['2001_04_01', '2001_01_04', '1998_10_30', '2007_08_09',
# '1999_01_09', '2002_11_23']
epochs_ready = []
for path in original_dfm_models:
fname = os.path.split(path)[-1]
epoch = fname[:-4]
print("Processing epoch : {}".format(epoch))
if epoch in epochs_ready:
print("Skipping epoch {}".format(epoch))
continue
original_model_fname = fname
original_model_path = os.path.join(data_dir, original_model_fname)
uv_fits_fname = mojave_uv_fits_fname('0316+413', 'u', epoch)
uv_fits_path = os.path.join(data_dir, uv_fits_fname)
uvdata = UVData(uv_fits_path)
try:
del uvdata.hdu.header['HISTORY']
uvdata.save(rewrite=True, downscale_by_freq=False)
except KeyError:
pass
out_txt_file = os.path.join(txt_file_dir, 'errors_{}.mod'.format(epoch))
out_png_file = os.path.join(txt_file_dir, 'errors_{}.png'.format(epoch))
try:
bootstrap_uvfits_with_difmap_model(uv_fits_path,
original_model_path,
n_boot=100,
import corner
from components import CGComponent
from model import Model
from uv_data import UVData
from mojave import mojave_uv_fits_fname, download_mojave_uv_fits
from spydiff import modelfit_difmap, import_difmap_model
from bootstrap import bootstrap_uvfits_with_difmap_model
source = '1514-241'
epoch = '2006_04_28'
data_dir = '/home/ilya/Dropbox/papers/boot/bias/new/stationary'
# download_mojave_uv_fits(source, epochs=[epoch], bands=['u'],
# download_dir=data_dir)
uv_fits_fnames = {freq: mojave_uv_fits_fname(source, freq, epoch) for freq in
('x', 'j', 'u')}
for freq, uv_fits_fname in uv_fits_fnames.items():
uv_fits_path = os.path.join(data_dir, uv_fits_fname)
cg1 = CGComponent(2.0, 0., 0., 0.2)
cg2 = CGComponent(1.0, 0., 0.3, 0.3)
cg3 = CGComponent(0.5, 0., 1.5, 0.4)
mdl = Model(stokes='I')
mdl.add_components(cg1, cg2, cg3)
uvdata = UVData(uv_fits_path)
noise = uvdata.noise()
for i in range(1, 101):
uvdata = UVData(uv_fits_path)
uvdata.substitute([mdl])
uvdata.noise_add(noise)
art_fits_fname = 'art_{}_{}.fits'.format(freq, i)
'y': (512, 0.1),
'j': (512, 0.1),
'u': (512, 0.1)
}
mapsize_common = (512, 0.1)
data_dir = os.path.join(base_dir, source, epoch)
bands = ['x', 'y', 'j', 'u']
if not os.path.exists(data_dir):
os.makedirs(data_dir)
from mojave import download_mojave_uv_fits
download_mojave_uv_fits(source,
epochs=[epoch],
bands=bands,
download_dir=data_dir)
from mojave import mojave_uv_fits_fname
uv_fits_fnames = [
mojave_uv_fits_fname(source, band, epoch) for band in bands
]
uv_fits_paths = [
os.path.join(data_dir, uv_fits_fname)
for uv_fits_fname in uv_fits_fnames
]
analyze_source(uv_fits_paths,
n_boot=30,
outdir=data_dir,
path_to_script=path_to_script,
imsizes=mapsize_dict.values(),
common_imsize=mapsize_common,
find_shifts=False)
