def download_mojave_models(base_dir):
"""
Download MOJAVE models & uv-fits files.
:param base_dir:
Directory with MOJAVE table.
"""
names = [
'source', 'id', 'trash', 'epoch', 'flux', 'r', 'pa', 'bmaj', 'e', 'bpa'
]
df = pd.read_table(os.path.join(base_dir, 'asu.tsv'),
sep=';',
header=None,
names=names,
dtype={key: str
for key in names},
index_col=False)
# Mow for all sources get the latest epoch and create directory for analysis
for source in df['source'].unique():
epochs = df.loc[df['source'] == source]['epoch']
last_epoch_ = list(epochs)[-1]
last_epoch = last_epoch_.replace('-', '_')
data_dir = os.path.join(base_dir, source, last_epoch)
if not os.path.exists(data_dir):
os.makedirs(data_dir)
try:
download_mojave_uv_fits(source,
epochs=[last_epoch],
bands=['u'],
download_dir=data_dir)
except:
open(
'problem_download_from_mojave_{}_{}'.format(
source, last_epoch), 'a').close()
continue
uv_fits_fname = mojave_uv_fits_fname(source, 'u', last_epoch)
# Create instance of Model and bootstrap uv-data
dfm_model_fname = 'dfmp_original_model.mdl'
fn = open(os.path.join(data_dir, dfm_model_fname), 'w')
model_df = df.loc[np.logical_and(df['source'] == source,
df['epoch'] == last_epoch_)]
for (flux, r, pa, bmaj, e, bpa) in np.asarray(
model_df[['flux', 'r', 'pa', 'bmaj', 'e', 'bpa']]):
print flux, r, pa, bmaj, e, bpa
if not r.strip(' '):
r = '0.0'
if not pa.strip(' '):
pa = '0.0'
if not bmaj.strip(' '):
bmaj = '0.0'
if not e.strip(' '):
e = "1.0"
if np.isnan(float(bpa)):
bpa = "0.0"
else:
bpa = bpa + 'v'
if bmaj == '0.0':
type_ = 0
bpa = "0.0"
else:
bmaj = bmaj + 'v'
type_ = 1
fn.write("{}v {}v {}v {} {} {} {} {} {}".format(
flux, r, pa, bmaj, e, bpa, type_, "0", "0\n"))
fn.close()
from model import Model
from from_fits import create_clean_image_from_fits_file
from mojave import (download_mojave_uv_fits, get_mojave_mdl_file,
mojave_uv_fits_fname)
data_dir = '/home/ilya/vlbi_errors/model_cov/check_image_add'
tsv_table = os.path.join(data_dir, 'asu.tsv')
source = '2230+114'
epoch = '2005-02-05'
uv_fits = mojave_uv_fits_fname(source, 'u', epoch.replace('-', '_'))
path_to_script = '/home/ilya/github/vlbi_errors/difmap/final_clean_nw'
mdl_fname = '{}_{}.mdl'.format(source, epoch)
# Fetch uv-fits
download_mojave_uv_fits(source, [epoch.replace('-', '_')],
bands=['u'],
download_dir=data_dir)
# Fetch model file
get_mojave_mdl_file(tsv_table, source, epoch, outdir=data_dir)
# Clean uv-fits
clean_difmap(uv_fits,
'cc.fits',
'I', [1024, 0.1],
path=data_dir,
path_to_script=path_to_script,
outpath=data_dir)
# Create clean image instance
cc_image = create_clean_image_from_fits_file(os.path.join(data_dir, 'cc.fits'))
comps = import_difmap_model(mdl_fname, data_dir)
model = Model(stokes='I')
stokes)
elif ci_type == 'rms':
perc = {1: 68, 2: 95, 3: 99}[n_rms]
outfile = 'cov_{}_cov{}_{}'.format(perc, n_cov, stokes)
else:
raise Exception("ci_type must be rms or boot!")
data_dir = os.path.join(base_dir, source, stokes, ci_type, str(perc))
print "Calculating {} CI with nominal coverage {}".format(
ci_type, perc)
print "Using directory {}".format(data_dir)
if not os.path.exists(data_dir):
os.makedirs(data_dir)
download_mojave_uv_fits(source,
epochs=epochs,
bands=bands,
download_dir=data_dir)
# Classic coverage analysis
fname = mojave_uv_fits_fname(source, bands[0], epochs[0])
original_uv_fits_path = os.path.join(data_dir, fname)
# sample_uv_fits_paths, sample_cc_fits_paths =\
# create_sample(original_uv_fits_path, imsize=imsize, outdir=data_dir,
# path_to_script=path_to_script)
sample_cc_fits_paths = sorted(
glob.glob(os.path.join(data_dir, 'sample_cc_*.fits')))
# sample_cc_fits_paths = None
sample_uv_fits_paths = sorted(
glob.glob(os.path.join(data_dir, 'sample_uv_*.uvf')))
# sample_uv_fits_paths = None
original_cc_fits_path = os.path.join(data_dir, 'original_cc.fits')
def coverage_of_model(original_uv_fits,
original_mdl_file,
outdir=None,
n_cov=100,
n_boot=300,
mapsize=(1024, 0.1),
path_to_script=None):
"""
Conduct coverage analysis of uv-data & model
:param original_uv_fits:
Self-calibrated uv-fits file.
:param original_mdl_file:
Difmap txt-file with model.
:param outdir:
Output directory to store results.
:param n_cov:
Number of samples to create.
"""
# Create sample of 100 uv-fits data & models
sample_uv_fits_paths, sample_model_paths = create_sample(original_uv_fits,
original_mdl_file,
outdir=outdir,
n_sample=n_cov)
# For each sample uv-fits & model find 1) conventional errors & 2) bootstrap
# errors
for j, (sample_uv_fits_path, sample_mdl_path) in enumerate(
zip(sample_uv_fits_paths, sample_model_paths)):
sample_uv_fits, dir = os.path.split(sample_uv_fits_path)
sample_mdl_file, dir = os.path.split(sample_mdl_path)
try:
comps = import_difmap_model(sample_mdl_file, dir)
except ValueError:
print('Problem import difmap model')
model = Model(stokes='I')
model.add_components(*comps)
# Find errors by using Fomalont way
# 1. Clean uv-data
clean_difmap(sample_uv_fits,
'sample_cc_{}.fits'.format(j),
'I',
mapsize,
path=dir,
path_to_script=path_to_script,
outpath=dir)
# 2. Get beam
ccimage = create_clean_image_from_fits_file(
os.path.join(dir, 'sample_cc_{}.fits'.format(j)))
beam = ccimage.beam_image
# 2. Subtract components convolved with beam
ccimage.substract_model(model)
# Find errors by using Lee way
# a) fit uv-data and find model
# b) CLEAN uv-data
# c) substract model from CLEAN image
# d) find errors
pass
# Find errors by using bootstrap
# FT model to uv-plane
uvdata = UVData(sample_uv_fits_path)
try:
boot = CleanBootstrap([model], uvdata)
# If uv-data contains only one Stokes parameter (e.g. `0838+133`)
except IndexError:
print('Problem bootstrapping')
curdir = os.getcwd()
os.chdir(dir)
boot.run(n=n_boot, nonparametric=True, outname=[outname, '.fits'])
os.chdir(curdir)
booted_uv_paths = sorted(
glob.glob(os.path.join(data_dir, outname + "*")))
# Modelfit bootstrapped uvdata
for booted_uv_path in booted_uv_paths:
path, booted_uv_file = os.path.split(booted_uv_path)
i = booted_uv_file.split('_')[-1].split('.')[0]
modelfit_difmap(booted_uv_file,
dfm_model_fname,
dfm_model_fname + '_' + i,
path=path,
mdl_path=data_dir,
out_path=data_dir)
# Get params of initial model used for bootstrap
comps = import_difmap_model(dfm_model_fname, data_dir)
comps_params0 = {i: [] for i in range(len(comps))}
for i, comp in enumerate(comps):
comps_params0[i].extend(list(comp.p))
# Load bootstrap models
booted_mdl_paths = glob.glob(
os.path.join(data_dir, dfm_model_fname + "_*"))
comps_params = {i: [] for i in range(len(comps))}
for booted_mdl_path in booted_mdl_paths:
path, booted_mdl_file = os.path.split(booted_mdl_path)
comps = import_difmap_model(booted_mdl_file, path)
for i, comp in enumerate(comps):
comps_params[i].extend(list(comp.p))
# Print 65-% intervals (1 sigma)
for i, comp in enumerate(comps):
errors_fname = '68_{}_{}_comp{}.txt'.format(source, last_epoch, i)
fn = open(os.path.join(data_dir, errors_fname), 'w')
print "Component #{}".format(i + 1)
for j in range(len(comp)):
low, high, mean, median = hdi_of_mcmc(np.array(
comps_params[i]).reshape((n_boot, len(comp))).T[j],
cred_mass=0.68,
return_mean_median=True)
fn.write("{} {} {} {} {}".format(comp.p[j], low, high, mean,
median))
fn.write("\n")
fn.close()
# For source in sources with component close to core
# 1. Find residuals or estimate noise
# 2. N times add resampled residuals (or just gaussian noise) to model and
# create N new datasets
# 3. Fit them using difmap.
# 4. Find errors using Fomalont, Yee and using bootstrap. Check coverage.
base_dir = '/home/ilya/vlbi_errors/model_cov'
n_boot = 300
outname = 'boot_uv'
names = [
'source', 'id', 'trash', 'epoch', 'flux', 'r', 'pa', 'bmaj', 'e', 'bpa'
]
df = pd.read_table(os.path.join(base_dir, 'asu.tsv'),
sep=';',
header=None,
names=names,
dtype={key: str
for key in names},
index_col=False)
# Mow for all sources get the latest epoch and create directory for analysis
for source in df['source'].unique():
epochs = df.loc[df['source'] == source]['epoch']
last_epoch_ = list(epochs)[-1]
last_epoch = last_epoch_.replace('-', '_')
data_dir = os.path.join(base_dir, source, last_epoch)
if not os.path.exists(data_dir):
os.makedirs(data_dir)
try:
download_mojave_uv_fits(source,
epochs=[last_epoch],
bands=['u'],
download_dir=data_dir)
except:
open(
'problem_download_from_mojave_{}_{}'.format(
source, last_epoch), 'a').close()
continue
uv_fits_fname = mojave_uv_fits_fname(source, 'u', last_epoch)
# Create instance of Model and bootstrap uv-data
dfm_model_fname = 'dfmp_original_model.mdl'
fn = open(os.path.join(data_dir, dfm_model_fname), 'w')
model_df = df.loc[np.logical_and(df['source'] == source,
df['epoch'] == last_epoch_)]
for (flux, r, pa, bmaj, e, bpa) in np.asarray(
model_df[['flux', 'r', 'pa', 'bmaj', 'e', 'bpa']]):
print flux, r, pa, bmaj, e, bpa
if not r.strip(' '):
r = '0.0'
if not pa.strip(' '):
pa = '0.0'
if not bmaj.strip(' '):
bmaj = '0.0'
if not e.strip(' '):
e = "1.0"
if np.isnan(float(bpa)):
bpa = "0.0"
else:
bpa = bpa + 'v'
if bmaj == '0.0':
type_ = 0
bpa = "0.0"
else:
bmaj = bmaj + 'v'
type_ = 1
fn.write("{}v {}v {}v {} {} {} {} {} {}".format(
flux, r, pa, bmaj, e, bpa, type_, "0", "0\n"))
fn.close()
[id, flux, r, pa, t_mid, closest_epoch, sigma_pos])
for source in sorted(df2['source'].unique()):
if len(source_dict[source]) > 5:
epoch = source_dict[source][0][5]
source_dir = os.path.join(base_dir, source, epoch)
if not os.path.exists(source_dir):
os.makedirs(source_dir)
get_mojave_mdl_file(os.path.join(base_dir, 'asu.tsv'),
source,
epoch,
outdir=source_dir)
epoch_ = "{}_{}_{}".format(*epoch.split('-'))
download_mojave_uv_fits(source,
epochs=[epoch_],
download_dir=source_dir,
bands=['u'])
fname = mojave_uv_fits_fname(source, 'u', epoch_)
uvdata = UVData(os.path.join(source_dir, fname))
print(uvdata.stokes)
if 'RR' not in uvdata.stokes or 'LL' not in uvdata.stokes:
continue
# Refit difmap model
modelfit_difmap(fname,
"{}_{}.mdl".format(source, epoch),
"{}_{}.mdl".format(source, epoch),
niter=200,
path=source_dir,
mdl_path=source_dir,
out_path=source_dir,
data_dir = '/home/ilya/Dropbox/papers/boot/new_pics/corner'
# Done
# source = '0016+731'
# epoch = '2005_01_06'
# epoch_ = '2005-01-06'
source = '1807+698'
epoch = '2007_07_03'
epoch_ = '2007-07-03'
source_dir = os.path.join(data_dir, source)
if not os.path.exists(source_dir):
os.mkdir(source_dir)
uv_fits_fname = mojave_uv_fits_fname(source, 'u', epoch)
uv_fits_path = os.path.join(source_dir, uv_fits_fname)
if not os.path.exists(uv_fits_path):
download_mojave_uv_fits(source, [epoch], download_dir=source_dir)
names = [
'source', 'id', 'trash', 'epoch', 'flux', 'r', 'pa', 'bmaj', 'e', 'bpa'
]
df = pd.read_table(os.path.join(data_dir, 'asu.tsv'),
sep=';',
header=None,
names=names,
dtype={key: str
for key in names},
index_col=False)
# Create instance of Model and bootstrap uv-data
dfm_model_fname = 'dfmp_original_model.mdl'
dfm_model_path = os.path.join(source_dir, dfm_model_fname)
str(source): str(epoch)
for source, epoch in source_epoch_dict.items()
}
sources = sorted(source_epoch_dict.keys())
# for source in sources:
# print("Querying source {}".format(source))
# epochs = get_epochs_for_source(source, use_db='u')
# source_epoch_dict.update({source: epochs})
for source in sources:
data_dir = os.path.join(base_dir, source)
if not os.path.exists(data_dir):
os.makedirs(data_dir)
source_images = dict()
download_mojave_uv_fits(source,
epochs=[source_epoch_dict[source]],
bands=['u'],
download_dir=data_dir)
epoch = source_epoch_dict[source]
fname = mojave_uv_fits_fname(source, 'u', epoch)
stokes = 'I'
cc_fits = map_fname(source, epoch, stokes)
clean_difmap(fname,
cc_fits,
stokes,
mapsize,
path=data_dir,
path_to_script=path_to_script,
outpath=data_dir)
i_image = create_clean_image_from_fits_file(os.path.join(
data_dir, cc_fits))
# imsize = i_image.imsize
# for source in sources:
# print "Simulating source {}".format(source)
# simulate(source, source_epoch_dict[source], ['x', 'y', 'j', 'u'],
# n_sample=3, rotm_clim=[-200, 200],
# path_to_script=path_to_script, mapsize_dict=mapsize_dict,
# mapsize_common=mapsize_common, base_dir=base_dir,
# rotm_value_0=0., max_jet_flux=0.005)
for source, epoch in source_epoch_dict.items():
print "Source {}, epoch {}".format(source, epoch)
data_dir = os.path.join(base_dir, source)
if not os.path.exists(data_dir):
os.makedirs(data_dir)
# First, download lowest frequency band and find beam width
download_mojave_uv_fits(source,
epochs=[epoch],
bands=[bands[0]],
download_dir=data_dir)
uv_fits_fname = mojave_uv_fits_fname(source, bands[0], epoch)
cc_fits_fname = "{}_{}_{}_{}_naitive_cc.fits".format(
source, epoch, bands[0], 'I')
clean_difmap(uv_fits_fname,
cc_fits_fname,
'I',
mapsize_dict[bands[0]],
path=data_dir,
path_to_script=path_to_script,
outpath=data_dir)
cc_image = create_clean_image_from_fits_file(
os.path.join(data_dir, cc_fits_fname))
image = create_image_from_fits_file(
os.path.join(data_dir, cc_fits_fname))
from mojave import download_mojave_uv_fits
save_dir = '/home/ilya/fs/sshfs/frb/data'
with open('/home/ilya/Dropbox/stack/sources', 'r') as fo:
lines = fo.readlines()
lines = lines[1:]
sources = list()
for line in lines:
sources.append(line.strip('\n').split(" ")[0])
for source in sources:
download_mojave_uv_fits(source, bands=['u'], download_dir=save_dir)