def score(uv_fits_path, mdl_path, stokes='I'): """ Returns rms of model on given uv-data for stokes 'I'. :param uv_fits_path: Path to uv-fits file. :param mdl_path: Path to difmap model text file or FITS-file with CLEAN model. :param stokes: (optional) Stokes parameter string. ``I``, ``RR`` or ``LL`` currently supported. (default: ``I``) :return: Per-point rms between given data and model evaluated at given data points. """ if stokes not in ('I', 'RR', 'LL'): raise Exception("Only stokes (I, RR, LL) supported!") uvdata = UVData(uv_fits_path) uvdata_model = UVData(uv_fits_path) try: model = create_model_from_fits_file(mdl_path) except IOError: dfm_mdl_dir, dfm_mdl_fname = os.path.split(mdl_path) comps = import_difmap_model(dfm_mdl_fname, dfm_mdl_dir) model = Model(stokes=stokes) model.add_components(*comps) uvdata_model.substitute([model]) uvdata_diff = uvdata - uvdata_model if stokes == 'I': i_diff = 0.5 * (uvdata_diff.uvdata_weight_masked[..., 0] + uvdata_diff.uvdata_weight_masked[..., 1]) elif stokes == 'RR': i_diff = uvdata_diff.uvdata_weight_masked[..., 0] elif stokes == 'LL': i_diff = uvdata_diff.uvdata_weight_masked[..., 1] else: raise Exception("Only stokes (I, RR, LL) supported!") # 2 means that Re & Im are counted independently factor = 2 * np.count_nonzero(i_diff) # factor = np.count_nonzero(~uvdata_diff.uvdata_weight_masked.mask[:, :, :2]) # squared_diff = uvdata_diff.uvdata_weight_masked[:, :, :2] * \ # uvdata_diff.uvdata_weight_masked[:, :, :2].conj() squared_diff = i_diff * i_diff.conj() return np.sqrt(float(np.sum(squared_diff)) / factor)
components_priors.append({'flux': (sp.stats.uniform.ppf, [0, 1], {}), 'x': (sp.stats.uniform.ppf, [-2, 4], {}), 'y': (sp.stats.uniform.ppf, [-2, 4], {}), 'bmaj': (sp.stats.uniform.ppf, [0, 1], {})}) components_priors.append({'flux': (sp.stats.uniform.ppf, [0, 1], {}), 'x': (sp.stats.uniform.ppf, [-5, 10], {}), 'y': (sp.stats.uniform.ppf, [-5, 10], {}), 'bmaj': (sp.stats.uniform.ppf, [0, 2], {})}) components_priors.append({'flux': (sp.stats.uniform.ppf, [0, 1], {}), 'x': (sp.stats.uniform.ppf, [-6, 12], {}), 'y': (sp.stats.uniform.ppf, [-6, 12], {}), 'bmaj': (sp.stats.uniform.ppf, [0, 3], {})}) results = fit_model_with_nestle(uv_fits, mdl_file, components_priors, outdir=outdir) data_dir = '/home/ilya/code/vlbi_errors/silke' # uv_fits = '0851+202.u.2012_11_11.uvf' uv_fits = '0851+202.u.2004_11_05.uvf' # mdl_fname = '2.mod.2012_11_11' mdl_fname = '1.mod.2004_11_05' uv_data = UVData(os.path.join(data_dir, uv_fits)) comps = import_difmap_model(mdl_fname, data_dir) model = Model(stokes='I') model.add_components(*comps) fig = uv_data.uvplot(style='a&p') uv_data.substitute([model]) uv_data.uvplot(color='r', fig=fig, phase_range=[-0.2, 0.2])
uv_fits_fnames = { freq: mojave_uv_fits_fname(source, freq, epoch) for freq in ('x', 'j', 'u') } for freq, uv_fits_fname in uv_fits_fnames.items(): uv_fits_path = os.path.join(data_dir, uv_fits_fname) cg1 = CGComponent(2.0, 0., 0., 0.2) cg2 = CGComponent(1.0, 0., 0.3, 0.3) cg3 = CGComponent(0.5, 0., 1.5, 0.4) mdl = Model(stokes='I') mdl.add_components(cg1, cg2, cg3) uvdata = UVData(uv_fits_path) noise = uvdata.noise() for i in range(1, 101): uvdata = UVData(uv_fits_path) uvdata.substitute([mdl]) uvdata.noise_add(noise) art_fits_fname = 'art_{}_{}.fits'.format(freq, i) art_fits_path = os.path.join(data_dir, art_fits_fname) uvdata.save(art_fits_path) # Here we should MCMC posterior modelfit_difmap(art_fits_fname, 'initial.mdl', 'out_{}_{}.mdl'.format(freq, i), niter=100, path=data_dir, mdl_path=data_dir, out_path=data_dir) params = list()
# Workflow for one source source = '0945+408' epoch = '2007_04_18' band = 'u' # TODO: Standard it image_fname = 'original_cc.fits' uv_fname_cc = '0945+408.u.2007_04_18.uvf' uv_fname_uv = '0945+408.u.2007_04_18.uvf' dfm_model_fname = 'dfmp_original_model.mdl' comps = import_difmap_model(dfm_model_fname, base_path) model_uv = Model(stokes='I') model_uv.add_components(*comps) uvdata = UVData(os.path.join(base_path, uv_fname_uv)) uvdata_m = UVData(os.path.join(base_path, uv_fname_uv)) uvdata_m.substitute([model_uv]) uvdata_r = uvdata - uvdata_m # Plot uv-data label_size = 12 matplotlib.rcParams['xtick.labelsize'] = label_size matplotlib.rcParams['ytick.labelsize'] = label_size uvdata.uvplot(style='re&im', freq_average=True) matplotlib.pyplot.show() matplotlib.pyplot.savefig('/home/ilya/sandbox/heteroboot/uvdata_original.png', bbox_inches='tight', dpi=400) matplotlib.pyplot.close() # # Plot residuals in radplot # label_size = 12 # matplotlib.rcParams['xtick.labelsize'] = label_size
train_scores = list() for i, fname in enumerate(['1IF.fits', '12IF.fits', '123IF.fits', '1234IF.fits', '12345IF.fits', '123456IF.fits', '1234567IF.fits']): current_fits = os.path.join(data_dir, fname) modelfit_difmap(current_fits, original_model_fname, 'out_{}.mdl'.format(i), path=data_dir, mdl_path=data_dir, out_path=data_dir, niter=100) comps = import_difmap_model('out_{}.mdl'.format(i), data_dir) model = Model(stokes='I') model.add_components(*comps) # Calculate performance on training data uvdata_train_model = UVData(current_fits) uvdata_train = UVData(current_fits) uvdata_train_model.substitute([model]) uvdata_diff_train = uvdata_train - uvdata_train_model factor = np.count_nonzero(~uvdata_diff_train.uvdata_weight_masked.mask[:, :, :2]) squared_diff = uvdata_diff_train.uvdata_weight_masked[:, :, :2] *\ uvdata_diff_train.uvdata_weight_masked[:, :, :2].conj() score = float(np.sum(squared_diff)) / factor train_scores.append(score) # Calculate performance on test data uvdata_test_model = UVData(os.path.join(data_dir, '8IF.fits')) uvdata_test = UVData(os.path.join(data_dir, '8IF.fits')) uvdata_test_model.substitute([model]) uvdata_diff_test = uvdata_test - uvdata_test_model factor = np.count_nonzero(~uvdata_diff_test.uvdata_weight_masked.mask[:, :, :2]) squared_diff = uvdata_diff_test.uvdata_weight_masked[:, :, :2] * \
def score(uv_fits_path, mdl_path, stokes='I', bmaj=None, score="l2", use_weights=True): """ Returns rms of the trained model (CLEAN or difmap) on a given test UVFITS data set. :param uv_fits_path: Path to uv-fits file (test data). :param mdl_path: Path to difmap model text file or FITS-file with CLEAN model (trained model). :param stokes: (optional) Stokes parameter string. ``I``, ``RR`` or ``LL`` currently supported. (default: ``I``) :param bmaj: (optional) FWHM of the circular beam to account for. If ``None`` than do not account for the beam. (default: ``None``) :return: Per-point rms between given test data and trained model evaluated at a given test data points. """ stokes = stokes.upper() if stokes not in ('I', 'RR', 'LL'): raise Exception("Only stokes I, RR or LL are supported!") if bmaj is not None: c = (np.pi * bmaj * mas_to_rad)**2 / (4 * np.log(2)) else: c = 1.0 # Loading test data with its own big mask uvdata = UVData(uv_fits_path) uvdata_model = UVData(uv_fits_path) # Loading trained model # CC-model try: model = create_model_from_fits_file(mdl_path) # Difmap model except IOError: dfm_mdl_dir, dfm_mdl_fname = os.path.split(mdl_path) comps = import_difmap_model(dfm_mdl_fname, dfm_mdl_dir) model = Model(stokes=stokes) model.add_components(*comps) # Computing difference and score uvdata_model.substitute([model]) uvdata_diff = uvdata - uvdata_model if stokes == 'I': i_diff = 0.5 * (uvdata_diff.uvdata_weight_masked[..., 0] + uvdata_diff.uvdata_weight_masked[..., 1]) weights = uvdata.weights_nw_masked[..., 0] + uvdata.weights_nw_masked[..., 1] elif stokes == 'RR': i_diff = uvdata_diff.uvdata_weight_masked[..., 0] weights = uvdata.weights_nw_masked[..., 0] elif stokes == 'LL': i_diff = uvdata_diff.uvdata_weight_masked[..., 1] weights = uvdata.weights_nw_masked[..., 1] else: raise Exception("Only stokes (I, RR, LL) supported!") # Normalize weights weights = weights / np.ma.sum(weights) # Account for beam if bmaj is not None: u = uvdata_diff.uv[:, 0] v = uvdata_diff.uv[:, 1] taper = np.exp(-c * (u * u + v * v)) i_diff = i_diff * taper[:, np.newaxis] # Number of unmasked visibilities (accounting each IF) if stokes == "I": # 2 means that Re & Im are counted independently factor = 2 * np.count_nonzero(~i_diff.mask) else: factor = np.count_nonzero(~i_diff.mask) print("Number of independent test data points = ", factor) if score == "l2": if use_weights: result = np.sqrt( (np.ma.sum(i_diff * i_diff.conj() * weights)).real) else: result = np.sqrt((np.ma.sum(i_diff * i_diff.conj())).real / factor) elif score == "l1": if use_weights: result = (np.ma.sum(np.abs(i_diff) * weights)).real else: result = (np.ma.sum(np.abs(i_diff))).real / factor else: raise Exception("score must be in (l1, l2)!") return result
# uv_file = '/home/ilya/github/bck/jetshow/uvf/0716+714_raks01xg_C_LL_0060s_uva.fits' uv_file = '/home/ilya/github/bck/jetshow/uvf/2200+420_K_SVLBI.uvf' uvdata_ext = UVData(uv_file) uvdata_orig = UVData(uv_file) # clean_difmap('2200+420_K_SVLBI.uvf', 'bllac_cc.fits', 'I', (8192, 0.0035), # path='/home/ilya/github/bck/jetshow/uvf/', # path_to_script='/home/ilya/github/vlbi_errors/difmap/final_clean_nw', # show_difmap_output=True) comps = import_difmap_model('/home/ilya/github/bck/jetshow/uvf/ell_c_ell.mdl') ext_model = Model(stokes='I') ext_model.add_component(comps[-1]) # cc_fits = '/home/ilya/github/vlbi_errors/vlbi_errors/bllac_cc.fits' # fig = uvdata_ext.uvplot() # ccmodel = create_model_from_fits_file(cc_fits) # ccmodel.filter_components_by_r(r_max_mas=0.15) uvdata_ext.substitute([ext_model]) uvdata_core = uvdata_orig - uvdata_ext # uvdata_core.save('/home/ilya/github/vlbi_errors/vlbi_errors/bllac_core.uvf') # Set up ModelImage component image = '/home/ilya/github/bck/jetshow/cmake-build-debug/map_i.txt' image = np.loadtxt(image) imsize = 1734 imsize = (imsize, imsize) mas_in_pix = 0.00253 y, z = np.meshgrid(np.arange(imsize[0]), np.arange(imsize[1])) y = y - imsize[0] / 2. + 0.5 z = z - imsize[0] / 2. + 0.5 y_mas = y * mas_in_pix z_mas = z * mas_in_pix y_rad = mas_to_rad * y_mas
path_to_script=path_to_script, show_difmap_output=True, outpath=data_dir) model = create_model_from_fits_file(os.path.join(data_dir, 'cc.fits')) comps = model._components comps = sorted(comps, key=lambda x: np.sqrt(x._p[1]**2 + x._p[2]**2), reverse=True) for i, comp in enumerate(comps[350:]): print "Substracting {} components".format((i + 350)) uvdata_ = UVData(os.path.join(data_dir, uv_fits)) mdl1 = Model(stokes='I') mdl1.add_component(comp) uvdata_.substitute([mdl1]) uvdata_diff = uvdata - uvdata_ uvdata_diff.save( os.path.join(data_dir, 'without_{}_ccs.uvp'.format(str(i + 350).zfill(3)))) fits_files = sorted(glob.glob(os.path.join(data_dir, 'without*'))) for fits_file in fits_files: fits_fname = os.path.split(fits_file)[-1] i = fits_fname.split('_')[1] modelfit_difmap(fits_file, model_fname, 'without_{}_css.mdl'.format(i), path=data_dir, mdl_path=data_dir, out_path=data_dir)
# cg = CGComponent(0.5, 0, 0, 0.5) # model = Model(stokes=stokes) # model.add_components(*new_dfm_model) # model.add_components(ccmodel) if stokes == "I": use_V = True else: use_V = False noise = uvdata_template.noise(use_V=use_V) params = list() for i in range(n_mc): uvdata_template.substitute([ccmodel]) uvdata_template.uvdata = uvdata_template.uvdata * corrections uvdata_template.noise_add(noise) uvdata_template.save(os.path.join(data_dir, "artificial.uvf"), rewrite=True, downscale_by_freq=True) # Self-calibrate selfcal_difmap( fname="artificial.uvf", outfname="artificial.uvf", path=data_dir, path_to_script="/home/ilya/github/ve/difmap/auto_selfcal", outpath=data_dir, show_difmap_output=True)