def score(uv_fits_path, mdl_path, stokes='I'):
"""
Returns rms of model on given uv-data for stokes 'I'.
:param uv_fits_path:
Path to uv-fits file.
:param mdl_path:
Path to difmap model text file or FITS-file with CLEAN model.
:param stokes: (optional)
Stokes parameter string. ``I``, ``RR`` or ``LL`` currently supported.
(default: ``I``)
:return:
Per-point rms between given data and model evaluated at given data
points.
"""
if stokes not in ('I', 'RR', 'LL'):
raise Exception("Only stokes (I, RR, LL) supported!")
uvdata = UVData(uv_fits_path)
uvdata_model = UVData(uv_fits_path)
try:
model = create_model_from_fits_file(mdl_path)
except IOError:
dfm_mdl_dir, dfm_mdl_fname = os.path.split(mdl_path)
comps = import_difmap_model(dfm_mdl_fname, dfm_mdl_dir)
model = Model(stokes=stokes)
model.add_components(*comps)
uvdata_model.substitute([model])
uvdata_diff = uvdata - uvdata_model
if stokes == 'I':
i_diff = 0.5 * (uvdata_diff.uvdata_weight_masked[..., 0] +
uvdata_diff.uvdata_weight_masked[..., 1])
elif stokes == 'RR':
i_diff = uvdata_diff.uvdata_weight_masked[..., 0]
elif stokes == 'LL':
i_diff = uvdata_diff.uvdata_weight_masked[..., 1]
else:
raise Exception("Only stokes (I, RR, LL) supported!")
# 2 means that Re & Im are counted independently
factor = 2 * np.count_nonzero(i_diff)
# factor = np.count_nonzero(~uvdata_diff.uvdata_weight_masked.mask[:, :, :2])
# squared_diff = uvdata_diff.uvdata_weight_masked[:, :, :2] * \
# uvdata_diff.uvdata_weight_masked[:, :, :2].conj()
squared_diff = i_diff * i_diff.conj()
return np.sqrt(float(np.sum(squared_diff)) / factor)
components_priors.append({'flux': (sp.stats.uniform.ppf, [0, 1], {}),
'x': (sp.stats.uniform.ppf, [-2, 4], {}),
'y': (sp.stats.uniform.ppf, [-2, 4], {}),
'bmaj': (sp.stats.uniform.ppf, [0, 1], {})})
components_priors.append({'flux': (sp.stats.uniform.ppf, [0, 1], {}),
'x': (sp.stats.uniform.ppf, [-5, 10], {}),
'y': (sp.stats.uniform.ppf, [-5, 10], {}),
'bmaj': (sp.stats.uniform.ppf, [0, 2], {})})
components_priors.append({'flux': (sp.stats.uniform.ppf, [0, 1], {}),
'x': (sp.stats.uniform.ppf, [-6, 12], {}),
'y': (sp.stats.uniform.ppf, [-6, 12], {}),
'bmaj': (sp.stats.uniform.ppf, [0, 3], {})})
results = fit_model_with_nestle(uv_fits, mdl_file, components_priors,
outdir=outdir)
data_dir = '/home/ilya/code/vlbi_errors/silke'
# uv_fits = '0851+202.u.2012_11_11.uvf'
uv_fits = '0851+202.u.2004_11_05.uvf'
# mdl_fname = '2.mod.2012_11_11'
mdl_fname = '1.mod.2004_11_05'
uv_data = UVData(os.path.join(data_dir, uv_fits))
comps = import_difmap_model(mdl_fname, data_dir)
model = Model(stokes='I')
model.add_components(*comps)
fig = uv_data.uvplot(style='a&p')
uv_data.substitute([model])
uv_data.uvplot(color='r', fig=fig, phase_range=[-0.2, 0.2])
uv_fits_fnames = {
freq: mojave_uv_fits_fname(source, freq, epoch)
for freq in ('x', 'j', 'u')
}
for freq, uv_fits_fname in uv_fits_fnames.items():
uv_fits_path = os.path.join(data_dir, uv_fits_fname)
cg1 = CGComponent(2.0, 0., 0., 0.2)
cg2 = CGComponent(1.0, 0., 0.3, 0.3)
cg3 = CGComponent(0.5, 0., 1.5, 0.4)
mdl = Model(stokes='I')
mdl.add_components(cg1, cg2, cg3)
uvdata = UVData(uv_fits_path)
noise = uvdata.noise()
for i in range(1, 101):
uvdata = UVData(uv_fits_path)
uvdata.substitute([mdl])
uvdata.noise_add(noise)
art_fits_fname = 'art_{}_{}.fits'.format(freq, i)
art_fits_path = os.path.join(data_dir, art_fits_fname)
uvdata.save(art_fits_path)
# Here we should MCMC posterior
modelfit_difmap(art_fits_fname,
'initial.mdl',
'out_{}_{}.mdl'.format(freq, i),
niter=100,
path=data_dir,
mdl_path=data_dir,
out_path=data_dir)
params = list()
# Workflow for one source
source = '0945+408'
epoch = '2007_04_18'
band = 'u'
# TODO: Standard it
image_fname = 'original_cc.fits'
uv_fname_cc = '0945+408.u.2007_04_18.uvf'
uv_fname_uv = '0945+408.u.2007_04_18.uvf'
dfm_model_fname = 'dfmp_original_model.mdl'
comps = import_difmap_model(dfm_model_fname, base_path)
model_uv = Model(stokes='I')
model_uv.add_components(*comps)
uvdata = UVData(os.path.join(base_path, uv_fname_uv))
uvdata_m = UVData(os.path.join(base_path, uv_fname_uv))
uvdata_m.substitute([model_uv])
uvdata_r = uvdata - uvdata_m
# Plot uv-data
label_size = 12
matplotlib.rcParams['xtick.labelsize'] = label_size
matplotlib.rcParams['ytick.labelsize'] = label_size
uvdata.uvplot(style='re&im', freq_average=True)
matplotlib.pyplot.show()
matplotlib.pyplot.savefig('/home/ilya/sandbox/heteroboot/uvdata_original.png',
bbox_inches='tight', dpi=400)
matplotlib.pyplot.close()
# # Plot residuals in radplot
# label_size = 12
# matplotlib.rcParams['xtick.labelsize'] = label_size
train_scores = list()
for i, fname in enumerate(['1IF.fits', '12IF.fits', '123IF.fits', '1234IF.fits',
'12345IF.fits', '123456IF.fits', '1234567IF.fits']):
current_fits = os.path.join(data_dir, fname)
modelfit_difmap(current_fits,
original_model_fname, 'out_{}.mdl'.format(i),
path=data_dir, mdl_path=data_dir,
out_path=data_dir, niter=100)
comps = import_difmap_model('out_{}.mdl'.format(i), data_dir)
model = Model(stokes='I')
model.add_components(*comps)
# Calculate performance on training data
uvdata_train_model = UVData(current_fits)
uvdata_train = UVData(current_fits)
uvdata_train_model.substitute([model])
uvdata_diff_train = uvdata_train - uvdata_train_model
factor = np.count_nonzero(~uvdata_diff_train.uvdata_weight_masked.mask[:, :, :2])
squared_diff = uvdata_diff_train.uvdata_weight_masked[:, :, :2] *\
uvdata_diff_train.uvdata_weight_masked[:, :, :2].conj()
score = float(np.sum(squared_diff)) / factor
train_scores.append(score)
# Calculate performance on test data
uvdata_test_model = UVData(os.path.join(data_dir, '8IF.fits'))
uvdata_test = UVData(os.path.join(data_dir, '8IF.fits'))
uvdata_test_model.substitute([model])
uvdata_diff_test = uvdata_test - uvdata_test_model
factor = np.count_nonzero(~uvdata_diff_test.uvdata_weight_masked.mask[:, :, :2])
squared_diff = uvdata_diff_test.uvdata_weight_masked[:, :, :2] * \
def score(uv_fits_path,
mdl_path,
stokes='I',
bmaj=None,
score="l2",
use_weights=True):
"""
Returns rms of the trained model (CLEAN or difmap) on a given test UVFITS
data set.
:param uv_fits_path:
Path to uv-fits file (test data).
:param mdl_path:
Path to difmap model text file or FITS-file with CLEAN model (trained
model).
:param stokes: (optional)
Stokes parameter string. ``I``, ``RR`` or ``LL`` currently supported.
(default: ``I``)
:param bmaj: (optional)
FWHM of the circular beam to account for. If ``None`` than do not
account for the beam. (default: ``None``)
:return:
Per-point rms between given test data and trained model evaluated at a
given test data points.
"""
stokes = stokes.upper()
if stokes not in ('I', 'RR', 'LL'):
raise Exception("Only stokes I, RR or LL are supported!")
if bmaj is not None:
c = (np.pi * bmaj * mas_to_rad)**2 / (4 * np.log(2))
else:
c = 1.0
# Loading test data with its own big mask
uvdata = UVData(uv_fits_path)
uvdata_model = UVData(uv_fits_path)
# Loading trained model
# CC-model
try:
model = create_model_from_fits_file(mdl_path)
# Difmap model
except IOError:
dfm_mdl_dir, dfm_mdl_fname = os.path.split(mdl_path)
comps = import_difmap_model(dfm_mdl_fname, dfm_mdl_dir)
model = Model(stokes=stokes)
model.add_components(*comps)
# Computing difference and score
uvdata_model.substitute([model])
uvdata_diff = uvdata - uvdata_model
if stokes == 'I':
i_diff = 0.5 * (uvdata_diff.uvdata_weight_masked[..., 0] +
uvdata_diff.uvdata_weight_masked[..., 1])
weights = uvdata.weights_nw_masked[...,
0] + uvdata.weights_nw_masked[...,
1]
elif stokes == 'RR':
i_diff = uvdata_diff.uvdata_weight_masked[..., 0]
weights = uvdata.weights_nw_masked[..., 0]
elif stokes == 'LL':
i_diff = uvdata_diff.uvdata_weight_masked[..., 1]
weights = uvdata.weights_nw_masked[..., 1]
else:
raise Exception("Only stokes (I, RR, LL) supported!")
# Normalize weights
weights = weights / np.ma.sum(weights)
# Account for beam
if bmaj is not None:
u = uvdata_diff.uv[:, 0]
v = uvdata_diff.uv[:, 1]
taper = np.exp(-c * (u * u + v * v))
i_diff = i_diff * taper[:, np.newaxis]
# Number of unmasked visibilities (accounting each IF)
if stokes == "I":
# 2 means that Re & Im are counted independently
factor = 2 * np.count_nonzero(~i_diff.mask)
else:
factor = np.count_nonzero(~i_diff.mask)
print("Number of independent test data points = ", factor)
if score == "l2":
if use_weights:
result = np.sqrt(
(np.ma.sum(i_diff * i_diff.conj() * weights)).real)
else:
result = np.sqrt((np.ma.sum(i_diff * i_diff.conj())).real / factor)
elif score == "l1":
if use_weights:
result = (np.ma.sum(np.abs(i_diff) * weights)).real
else:
result = (np.ma.sum(np.abs(i_diff))).real / factor
else:
raise Exception("score must be in (l1, l2)!")
return result
# uv_file = '/home/ilya/github/bck/jetshow/uvf/0716+714_raks01xg_C_LL_0060s_uva.fits'
uv_file = '/home/ilya/github/bck/jetshow/uvf/2200+420_K_SVLBI.uvf'
uvdata_ext = UVData(uv_file)
uvdata_orig = UVData(uv_file)
# clean_difmap('2200+420_K_SVLBI.uvf', 'bllac_cc.fits', 'I', (8192, 0.0035),
# path='/home/ilya/github/bck/jetshow/uvf/',
# path_to_script='/home/ilya/github/vlbi_errors/difmap/final_clean_nw',
# show_difmap_output=True)
comps = import_difmap_model('/home/ilya/github/bck/jetshow/uvf/ell_c_ell.mdl')
ext_model = Model(stokes='I')
ext_model.add_component(comps[-1])
# cc_fits = '/home/ilya/github/vlbi_errors/vlbi_errors/bllac_cc.fits'
# fig = uvdata_ext.uvplot()
# ccmodel = create_model_from_fits_file(cc_fits)
# ccmodel.filter_components_by_r(r_max_mas=0.15)
uvdata_ext.substitute([ext_model])
uvdata_core = uvdata_orig - uvdata_ext
# uvdata_core.save('/home/ilya/github/vlbi_errors/vlbi_errors/bllac_core.uvf')
# Set up ModelImage component
image = '/home/ilya/github/bck/jetshow/cmake-build-debug/map_i.txt'
image = np.loadtxt(image)
imsize = 1734
imsize = (imsize, imsize)
mas_in_pix = 0.00253
y, z = np.meshgrid(np.arange(imsize[0]), np.arange(imsize[1]))
y = y - imsize[0] / 2. + 0.5
z = z - imsize[0] / 2. + 0.5
y_mas = y * mas_in_pix
z_mas = z * mas_in_pix
y_rad = mas_to_rad * y_mas
path_to_script=path_to_script,
show_difmap_output=True,
outpath=data_dir)
model = create_model_from_fits_file(os.path.join(data_dir, 'cc.fits'))
comps = model._components
comps = sorted(comps,
key=lambda x: np.sqrt(x._p[1]**2 + x._p[2]**2),
reverse=True)
for i, comp in enumerate(comps[350:]):
print "Substracting {} components".format((i + 350))
uvdata_ = UVData(os.path.join(data_dir, uv_fits))
mdl1 = Model(stokes='I')
mdl1.add_component(comp)
uvdata_.substitute([mdl1])
uvdata_diff = uvdata - uvdata_
uvdata_diff.save(
os.path.join(data_dir,
'without_{}_ccs.uvp'.format(str(i + 350).zfill(3))))
fits_files = sorted(glob.glob(os.path.join(data_dir, 'without*')))
for fits_file in fits_files:
fits_fname = os.path.split(fits_file)[-1]
i = fits_fname.split('_')[1]
modelfit_difmap(fits_file,
model_fname,
'without_{}_css.mdl'.format(i),
path=data_dir,
mdl_path=data_dir,
out_path=data_dir)
# cg = CGComponent(0.5, 0, 0, 0.5)
# model = Model(stokes=stokes)
# model.add_components(*new_dfm_model)
# model.add_components(ccmodel)
if stokes == "I":
use_V = True
else:
use_V = False
noise = uvdata_template.noise(use_V=use_V)
params = list()
for i in range(n_mc):
uvdata_template.substitute([ccmodel])
uvdata_template.uvdata = uvdata_template.uvdata * corrections
uvdata_template.noise_add(noise)
uvdata_template.save(os.path.join(data_dir, "artificial.uvf"),
rewrite=True,
downscale_by_freq=True)
# Self-calibrate
selfcal_difmap(
fname="artificial.uvf",
outfname="artificial.uvf",
path=data_dir,
path_to_script="/home/ilya/github/ve/difmap/auto_selfcal",
outpath=data_dir,
show_difmap_output=True)
