def boot_ci_bc(boot_cc_fits_paths, original_cc_fits_path, alpha=0.68): """ Calculate bootstrap CI. :param boot_cc_fits_paths: Iterable of paths to bootstrapped CC FITS-files. :param original_cc_fits_path: Path to original CC FITS-file. :return: Two numpy arrays with low and high CI borders for each pixel. """ alpha = 0.5 * (1. - alpha) original_image = create_image_from_fits_file(original_cc_fits_path) boot_images = list() for boot_cc_fits_path in boot_cc_fits_paths: print("Reading image from {}".format(boot_cc_fits_path)) image = create_image_from_fits_file(boot_cc_fits_path) boot_images.append(image.image) images_cube = np.dstack(boot_images) boot_ci_0 = np.zeros(np.shape(images_cube[:, :, 0])) boot_ci_1 = np.zeros(np.shape(images_cube[:, :, 0])) print("calculating CI intervals") for (x, y), value in np.ndenumerate(boot_ci_0): boot_ci_0[x, y] = bc_endpoint(images_cube[x, y, :], original_image.image[x, y], alpha) boot_ci_1[x, y] = bc_endpoint(images_cube[x, y, :], original_image.image[x, y], 1. - alpha) return boot_ci_0, boot_ci_1
def boot_ci_asymm(boot_cc_fits_paths, original_cc_fits_path, alpha=0.68): """ Calculate bootstrap CI. :param boot_cc_fits_paths: Iterable of paths to bootstrapped CC FITS-files. :param original_cc_fits_path: Path to original CC FITS-file. :return: Two numpy arrays with low and high CI borders for each pixel. """ original_image = create_image_from_fits_file(original_cc_fits_path) boot_images = list() for boot_cc_fits_path in boot_cc_fits_paths: print("Reading image from {}".format(boot_cc_fits_path)) image = create_image_from_fits_file(boot_cc_fits_path) boot_images.append(image.image) images_cube = np.dstack(boot_images) boot_ci = np.zeros(np.shape(images_cube[:, :, 0])) print("calculating CI intervals") for (x, y), value in np.ndenumerate(boot_ci): hdi = hdi_of_mcmc(images_cube[x, y, :], cred_mass=alpha) mean_boot = np.mean(images_cube[x, y, :]) hdi_low = original_image.image - (mean_boot - hdi[0]) hdi_high = original_image.image + hdi[1] - mean_boot return hdi_low, hdi_high
def rms_image_shifted(uv_fits_path, hovatta_factor=True, shift=(1000, 1000), tmp_name='shifted_clean_map.fits', tmp_dir=None, stokes='I', image=None, image_fits=None, mapsize_clean=None, path_to_script=None, niter=None): """ Estimate image per-pixel rms using shifted image. """ path, uv_fits_fname = os.path.split(uv_fits_path) if tmp_dir is None: tmp_dir = os.getcwd() if path_to_script is None: raise Exception("Provide location of difmap final CLEAN script!") if mapsize_clean is None and image is not None: import utils pixsize = abs(image.pixsize[0]) / utils.mas_to_rad mapsize_clean = (image.imsize[0], pixsize) if mapsize_clean is None and image_fits is not None: import from_fits image = from_fits.create_image_from_fits_file(image_fits) import utils pixsize = abs(image.pixsize[0]) / utils.mas_to_rad mapsize_clean = (image.imsize[0], pixsize) import spydiff if niter is None: spydiff.clean_difmap(uv_fits_fname, tmp_name, stokes=stokes, mapsize_clean=mapsize_clean, path=path, path_to_script=path_to_script, outpath=tmp_dir, shift=shift) else: spydiff.clean_n(uv_fits_path, tmp_name, stokes=stokes, mapsize_clean=mapsize_clean, niter=niter, path_to_script=path_to_script, outpath=tmp_dir, shift=shift) import from_fits image = from_fits.create_image_from_fits_file( os.path.join(tmp_dir, tmp_name)) return rms_image(image, hovatta_factor=hovatta_factor)
def plot_coverage_maps(coverage_map, original_uv_fits_path, original_cc_fits_path, data_dir, outname, path_to_script): i_image_cc = create_clean_image_from_fits_file(original_cc_fits_path) i_image = create_image_from_fits_file(original_cc_fits_path) rms = rms_image_shifted(original_uv_fits_path, tmp_dir=data_dir, image_fits=original_cc_fits_path, path_to_script=path_to_script) blc, trc = find_bbox(i_image.image, 2 * rms, delta=int(i_image_cc._beam.bmaj / 2)) fig = iplot(i_image.image, coverage_map - 0.68, x=i_image.x, y=i_image.y, min_abs_level=2. * rms, outdir=data_dir, beam=i_image_cc.beam, show=True, beam_place='lr', show_beam=True, cmap='viridis', blc=blc, trc=trc, color_clim=[-0.3, 0.3], colors_mask=i_image.image < 2 * rms) fig.savefig(os.path.join(data_dir, '{}.pdf'.format(outname)), bbox_inches='tight', format='pdf', dpi=600)
def create_coverage_map_classic(original_uv_fits_path, ci_type, original_cc_fits_path=None, imsize=None, outdir=None, n_boot=200, path_to_script=None, alpha=0.68, n_cov=100, n_rms=1., stokes='I', sample_cc_fits_paths=None, sample_uv_fits_paths=None): """ Conduct coverage analysis of image pixels flux CI. Find number of times when CI of `sample` values contains `true` value. :param original_uv_fits_path: Path to original FITS-file with uv-data. :param ci_type: Type of CI to test. ``boot`` or ``rms``. If ``boot`` then use residuals bootstrap CI. If ``rms`` then use Hovatta corrected image rms CI. :param original_cc_fits_path: (optional) Path to original FITS-file with CC model. If ``None`` then use ``imsize`` parameter to get `original` CC model from ``original_uv_fits_path``. (default: ``None``) :param imsize: (optional) Image parameters (image size [pix], pixel size [mas]) to use when doing first CC with ``original_cc_fits_path = None``. (default: ``None``) :param outdir: (optional) Directory to store intermediate results. If ``None`` then use CWD. (default: ``None``) :param n_boot: (optional) Number of bootstrap replications to use when calculating bootstrap CI for ``ci_type = boot`` option when ``boot_cc_fits_paths`` hasn't specified. (default: ``200``) :param path_to_script: (optional) Path to Dan Homan's script for final clean. If ``None`` then use CWD. (default: ``None``) :param alpha: (optional) Level of significance when calculating bootstrap CI for ``ci_type = boot`` case. E.g. ``0.68`` corresponds to `1 \sigma`. (default: ``0.68``) :param n_cov: (optional) Number of `samples` from infinite population to consider in coverage analysis of intervals. Here `samples` - observations of known source with different realisations of noise with known parameters. (default: ``100``) :param n_rms: (optional) Number of rms to use in ``ci_type = rms`` case. (default: ``1.``) :param stokes: (optional) Stokes parameter to use. If ``None`` then use ``I``. (default: ``None``) :param boot_cc_fits_paths: (optional) If ``ci_type = boot`` then this parameter could specify paths to cleaned bootstrapped uv-data. :param sample_uv_fits_paths: (optional) Path to FITS-files with `sample` uv-data. If ``None`` then create ``n_cov`` `sample` uv-data from noise of `original` uv-data and `original` CLEAN model. (default: ``None``) :param sample_cc_fits_paths: (optional) Path to FITS-files with CLEAN models of `sample` uv-data. If ``None`` then create ``n_cov`` `sample` uv-data from noise of `original` uv-data and `original` CLEAN model. (default: ``None``) :return: Coverage map. Each pixel contain frequency of times when CI for samples from population contain `true` value for given pixel. """ if original_cc_fits_path is None: print( "No `original` CLEAN model specified! Will CLEAN `original`" " uv-data.") if imsize is None: raise Exception("Specify ``imsize``") uv_fits_dir, uv_fits_fname = os.path.split(original_uv_fits_path) print("Cleaning `original` uv-data to" " {}".format(os.path.join(outdir, 'original_cc.fits'))) clean_difmap(uv_fits_fname, 'original_cc.fits', stokes, imsize, path=uv_fits_dir, path_to_script=path_to_script, outpath=outdir) original_cc_fits_path = os.path.join(outdir, 'original_cc.fits') # Find images parameters for cleaning if necessary if imsize is None: print( "Getting image parameters from `original`" " CLEAN FITS file {}.".format(original_cc_fits_path)) image_params = get_fits_image_info(original_cc_fits_path) imsize = (image_params['imsize'][0], abs(image_params['pixsize'][0]) / mas_to_rad) # This is `true` values. Will check how often they arise in `sample` CIs. original_image = create_image_from_fits_file(original_cc_fits_path) # If `sample` data doesn't ready - create it! if sample_uv_fits_paths is None: # Create `sample` sample_uv_fits_paths, sample_cc_fits_paths =\ create_sample(original_uv_fits_path, original_cc_fits_path=original_cc_fits_path, imsize=imsize, outdir=outdir, path_to_script=path_to_script, n_sample=n_cov, stokes=stokes) # For each pixel check how often CI of `sample` images contains `model`] # values. print("Creating coverage array") cov_array = np.zeros((imsize[0], imsize[0]), dtype=float) # Testing coverage of bootstrapped CI # For each `sample` uv-data and model generate bootstrap CI print sample_uv_fits_paths print sample_cc_fits_paths for sample_cc_fits_path, sample_uv_fits_path in zip( sample_cc_fits_paths, sample_uv_fits_paths): print("Sample : {}".format(sample_cc_fits_path)) if ci_type == 'boot': n__ = sample_uv_fits_path.split('.')[0].split('_')[-1] n_ = sample_cc_fits_path.split('.')[0].split('_')[-1] assert n_ == n__ print( "Bootstrapping sample uv-data {} with sample model {} using" " {} replications".format(sample_uv_fits_path, sample_cc_fits_path, n_boot)) print("Removing old bootstrapped files...") boot_cc_fits_paths = glob.glob( os.path.join(outdir, 'sample_cc_boot_*.fits')) for rmfile in boot_cc_fits_paths: print("Removing CC file {}".format(rmfile)) os.unlink(rmfile) boot_uv_fits_paths = \ sorted(glob.glob(os.path.join(outdir, 'sample_uv_boot_*.uvf'))) for rmfile in boot_uv_fits_paths: print("Removing UV file {}".format(rmfile)) os.unlink(rmfile) bootstrap_uv_fits(sample_uv_fits_path, [sample_cc_fits_path], n_boot, outpath=outdir, outname=['sample_uv_boot', '.uvf']) boot_uv_fits_paths =\ sorted(glob.glob(os.path.join(outdir, 'sample_uv_boot_*.uvf'))) # Clean each bootstrapped uv-data for i, uv_fits_path in enumerate(boot_uv_fits_paths): uv_fits_dir, uv_fits_fname = os.path.split(uv_fits_path) j = uv_fits_fname.split('.')[0].split('_')[-1] cc_fname = 'sample_cc_boot_{}.fits'.format(j) print("Cleaning {} bootstrapped sample" " uv-data to {}".format(uv_fits_path, os.path.join(outdir, cc_fname))) clean_difmap(uv_fits_fname, cc_fname, stokes, imsize, path=uv_fits_dir, path_to_script=path_to_script, outpath=outdir) boot_cc_fits_paths = sorted( glob.glob(os.path.join(outdir, 'sample_cc_boot_*.fits'))) # Calculate bootstrap CI for current `sample` # hdi_low, hdi_high = boot_ci_bc(boot_cc_fits_paths, # observed_cc_fits_path, alpha=alpha) hdi_low, hdi_high = boot_ci(boot_cc_fits_paths, sample_cc_fits_path, alpha=alpha) elif ci_type == 'rms': # Calculate ``n_rms`` CI print("Calculating rms...") sample_image = create_image_from_fits_file(sample_cc_fits_path) rms = rms_image_shifted(sample_uv_fits_path, image_fits=sample_cc_fits_path, path_to_script=path_to_script) # rms = rms_image(sample_image) hdi_low = sample_image.image - n_rms * rms hdi_high = sample_image.image + n_rms * rms else: raise Exception("CI intervals must be `boot` or `rms`!") # Check if `model` value falls in current `sample` CI print("Calculating hits of `true` values for" " {}".format(sample_cc_fits_path)) for (x, y), value in np.ndenumerate(cov_array): cov_array[x, y] += float( np.logical_and(hdi_low[x, y] < original_image.image[x, y], original_image.image[x, y] < hdi_high[x, y])) return cov_array / n_cov
def create_coverage_map(original_uv_fits_path, ci_type, original_cc_fits_path=None, imsize=None, outdir=None, n_boot=200, path_to_script=None, alpha=0.68, n_cov=100, n_rms=1., stokes='I', boot_cc_fits_paths=None, sample_cc_fits_paths=None): """ Conduct coverage analysis of image pixels flux CI. Find number of times when CI of `observed` value contains values of `samples`. :param original_uv_fits_path: Path to original FITS-file with uv-data. :param ci_type: Type of CI to test. ``boot`` or ``rms``. If ``boot`` then use residuals bootstrap CI. If ``rms`` then use Hovatta corrected image rms CI. :param original_cc_fits_path: (optional) Path to original FITS-file with CC model. If ``None`` then use ``imsize`` parameter to get `original` CC model from ``original_uv_fits_path``. (default: ``None``) :param imsize: (optional) Image parameters (image size [pix], pixel size [mas]) to use when doing first CC with ``original_cc_fits_path = None``. (default: ``None``) :param outdir: (optional) Directory to store intermediate results. If ``None`` then use CWD. (default: ``None``) :param n_boot: (optional) Number of bootstrap replications to use when calculating bootstrap CI for ``ci_type = boot`` option when ``boot_cc_fits_paths`` hasn't specified. (default: ``200``) :param path_to_script: (optional) Path to Dan Homan's script for final clean. If ``None`` then use CWD. (default: ``None``) :param alpha: (optional) Level of significance when calculating bootstrap CI for ``ci_type = boot`` case. E.g. ``0.68`` corresponds to `1 \sigma`. (default: ``0.68``) :param n_cov: (optional) Number of `samples` from infinite population to consider in coverage analysis of intervals. Here `samples` - observations of known source with different realisations of noise with known parameters. (default: ``100``) :param n_rms: (optional) Number of rms to use in ``ci_type = rms`` case. (default: ``1.``) :param stokes: (optional) Stokes parameter to use. If ``None`` then use ``I``. (default: ``None``) :param boot_cc_fits_paths: (optional) If ``ci_type = boot`` then this parameter could specify paths to cleaned bootstrapped uv-data. :param sample_cc_fits_paths: (optional) Path to FITS-files with CLEAN models of `sample` uv-data. If ``None`` then create ``n_cov`` `sample` uv-data from noise of `original` uv-data and `original` CLEAN model. (default: ``None``) :return: Coverage map. Each pixel contain frequency of times when samples from population hit inside CI for given pixel. """ # If not given `original` CLEAN model - get it by cleaning `original` # uv-data if original_cc_fits_path is None: print( "No `original` CLEAN model specified! Will CLEAN `original`" " uv-data.") if imsize is None: raise Exception("Specify ``imsize``") uv_fits_dir, uv_fits_fname = os.path.split(original_uv_fits_path) print("Cleaning `original` uv-data to" " {}".format(os.path.join(outdir, 'cc.fits'))) clean_difmap(uv_fits_fname, 'cc.fits', stokes, imsize, path=uv_fits_dir, path_to_script=path_to_script, outpath=outdir) original_cc_fits_path = os.path.join(outdir, 'cc.fits') original_uv_data = UVData(original_uv_fits_path) noise = original_uv_data.noise() original_model = create_model_from_fits_file(original_cc_fits_path) # Find images parameters for cleaning if necessary if imsize is None: print( "Getting image parameters from `original`" " CLEAN FITS file {}.".format(original_cc_fits_path)) image_params = get_fits_image_info(original_cc_fits_path) imsize = (image_params['imsize'][0], abs(image_params['pixsize'][0]) / mas_to_rad) # Substitute uv-data with original model and create `model` uv-data print("Substituting original uv-data with CLEAN model...") model_uv_data = copy.deepcopy(original_uv_data) model_uv_data.substitute([original_model]) # Add noise to `model` uv-data to get `observed` uv-data observed_uv_data = copy.deepcopy(model_uv_data) observed_uv_data.noise_add(noise) observed_uv_fits_path = os.path.join(outdir, 'observed_uv.uvf') if os.path.isfile(observed_uv_fits_path): os.unlink(observed_uv_fits_path) print("Adding noise to `model` uv-data to get `observed` uv-data...") observed_uv_data.save(fname=observed_uv_fits_path) observed_cc_fits_path = os.path.join(outdir, 'observed_cc.fits') if os.path.isfile(observed_cc_fits_path): os.unlink(observed_cc_fits_path) # Clean `observed` uv-data to get `observed` image and model print("Cleaning `observed` uv-data to `observed` CLEAN model...") clean_difmap('observed_uv.uvf', 'observed_cc.fits', original_model.stokes, imsize, path=outdir, path_to_script=path_to_script, outpath=outdir) # Get `observed` model and image observed_model = create_model_from_fits_file(observed_cc_fits_path) observed_image = create_image_from_fits_file(observed_cc_fits_path) # Testing coverage of bootstrapped CI if ci_type == 'boot': # Bootstrap and clean only when necessary if boot_cc_fits_paths is None: # Bootstrap `observed` uv-data with `observed` model boot = CleanBootstrap([observed_model], observed_uv_data) cwd = os.getcwd() path_to_script = path_to_script or cwd os.chdir(outdir) print("Bootstrapping uv-data with {} replications".format(n_boot)) boot.run(outname=['observed_uv_boot', '.uvf'], n=n_boot) os.chdir(cwd) boot_uv_fits_paths = sorted( glob.glob(os.path.join(outdir, 'observed_uv_boot*.uvf'))) # Clean each bootstrapped uv-data for i, uv_fits_path in enumerate(boot_uv_fits_paths): uv_fits_dir, uv_fits_fname = os.path.split(uv_fits_path) print("Cleaning {} bootstrapped observed" " uv-data to {}".format( uv_fits_path, os.path.join( outdir, 'observed_cc_boot_{}.fits'.format(i + 1)))) clean_difmap(uv_fits_fname, 'observed_cc_boot_{}.fits'.format(i + 1), original_model.stokes, imsize, path=uv_fits_dir, path_to_script=path_to_script, outpath=outdir) boot_cc_fits_paths = glob.glob( os.path.join(outdir, 'observed_cc_*.fits')) # Calculate bootstrap CI # hdi_low, hdi_high = boot_ci_bc(boot_cc_fits_paths, # observed_cc_fits_path, alpha=alpha) hdi_low, hdi_high = boot_ci(boot_cc_fits_paths, observed_cc_fits_path, alpha=alpha) elif ci_type == 'rms': # Calculate ``n_rms`` CI rms = observed_image.rms(region=(50, 50, 50, None)) rms = np.sqrt(rms**2. + (1.5 * rms**2.)**2.) hdi_low = observed_image.image - rms hdi_high = observed_image.image + rms else: raise Exception("CI intervals must be `boot` or `rms`!") # Create `sample` uv-data and clean it only when necessary if sample_cc_fits_paths is None: # Add noise to `model` uv-data ``n_cov`` times and get ``n_cov`` # `samples` from population sample_uv_fits_paths = list() for i in range(n_cov): sample_uv_data = copy.deepcopy(model_uv_data) sample_uv_data.noise_add(noise) sample_uv_fits_path = os.path.join(outdir, 'samle_uv_{}.uvf'.format(i + 1)) sample_uv_data.save(sample_uv_fits_path) sample_uv_fits_paths.append(sample_uv_fits_path) # Clean each `sample` FITS-file for i, uv_fits_path in enumerate(sample_uv_fits_paths): uv_fits_dir, uv_fits_fname = os.path.split(uv_fits_path) print("Cleaning {} sample uv-data to" " {}".format( uv_fits_path, os.path.join(outdir, 'sample_cc_{}.fits'.format(i + 1)))) clean_difmap(uv_fits_fname, 'sample_cc_{}.fits'.format(i + 1), original_model.stokes, imsize, path=uv_fits_dir, path_to_script=path_to_script, outpath=outdir) sample_cc_fits_paths = glob.glob( os.path.join(outdir, 'sample_cc_*.fits')) sample_images = list() for sample_cc_fits_path in sample_cc_fits_paths: image = create_image_from_fits_file(sample_cc_fits_path) sample_images.append(image.image) # For each pixel check how often flux in `sample` images lies in CI derived # for observed image. cov_array = np.zeros((imsize[0], imsize[0]), dtype=float) print("calculating CI intervals") for (x, y), value in np.ndenumerate(cov_array): for image in sample_images: cov_array[x, y] += float( np.logical_and(hdi_low[x, y] < image[x, y], image[x, y] < hdi_high[x, y])) return cov_array / n_cov
# original_cc_fits_path = None coverage_map =\ create_coverage_map_classic(original_uv_fits_path, ci_type=ci_type, original_cc_fits_path=original_cc_fits_path, imsize=imsize, outdir=data_dir, path_to_script=path_to_script, sample_cc_fits_paths=sample_cc_fits_paths, sample_uv_fits_paths=sample_uv_fits_paths, n_rms=n_rms, alpha=alpha, n_boot=n_boot, n_cov=n_cov, stokes=stokes) np.savetxt(os.path.join(data_dir, "{}.txt".format(outfile)), coverage_map) original_cc_fits_path = os.path.join(data_dir, 'original_cc.fits') i_image_cc = create_clean_image_from_fits_file(original_cc_fits_path) i_image = create_image_from_fits_file(original_cc_fits_path) rms = rms_image_shifted(original_uv_fits_path, tmp_dir=data_dir, image_fits=original_cc_fits_path, path_to_script=path_to_script) blc, trc = find_bbox(i_image.image, 2 * rms, delta=int(i_image_cc._beam.bmaj / 2)) # iplot(i_image.image, coverage_map, x=i_image.x, y=i_image.y, # min_abs_level=2. * rms, outfile=outfile, outdir=data_dir, # beam=i_image_cc.beam, show=True, beam_corner='lr', # show_beam=True, cmap='viridis', blc=blc, trc=trc) fig = iplot(i_image.image, coverage_map - 0.68, x=i_image.x, y=i_image.y,
# i_image_zeroed = i_image.image.copy() # i_image_zeroed[mask] = 0. # y, x = np.nonzero(i_image_zeroed) # y -= i_image.pixref[0] # x -= i_image.pixref[1] # distances = np.sqrt(x ** 2. + y ** 2.) # max_dist = int(sorted(distances)[-1]) beam = i_image.beam pixsize = abs(i_image.pixsize[0]) / mas_to_rad beam = (beam[0] / pixsize, beam[1] / pixsize, beam[2]) # cc_fits = '/home/ilya/vlbi_errors/examples/X/1226+023/I/boot/68/original_cc.fits' # cc_fits = '/home/ilya/vlbi_errors/examples/L/1038+064/rms/68/original_cc.fits' # cc_fits = '/home/ilya/vlbi_errors/examples/L/1633+382/rms/68/original_cc.fits' image = create_image_from_fits_file(os.path.join(data_dir, cc_fits)) rms = image_ops.rms_image(image) data = image.image.copy() from scipy.ndimage.filters import gaussian_filter data = gaussian_filter(data, 5) mask = data < 3. * rms data[mask] = 0 data[~mask] = 1 skel, distance = medial_axis(data, return_distance=True) dist_on_skel = distance * skel # Plot area and skeleton fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(8, 4),
clean_difmap(x_boot_uv, 'x_cc_same_{}.fits'.format(str(i+1).zfill(3)), 'I', (1024, 0.1), path_to_script=path_to_script, show_difmap_output=True, outpath=data_dir, beam_restore=ccimage_x.beam) for i, u_boot_uv in enumerate(u_boot_uvfits): clean_difmap(u_boot_uv, 'u_cc_same_{}.fits'.format(str(i+1).zfill(3)), 'I', (1024, 0.1), path_to_script=path_to_script, show_difmap_output=True, outpath=data_dir, beam_restore=ccimage_x.beam) # For mask size 0 to 80 pix find shifts for all bootstrapped images boot_sh_values = list() for i in range(1, 101): files = glob.glob(os.path.join(data_dir, '*_cc_same_{}.fits'.format(str(i).zfill(3)))) im1, im2 = files im1 = create_image_from_fits_file(im1) im2 = create_image_from_fits_file(im2) sh_values = list() for r in range(0, 80): sh = im1.cross_correlate(im2, region1=(im1.x_c, im1.y_c, r, None), region2=(im2.x_c, im2.y_c, r, None), upsample_factor=100) sh_value = math.sqrt(sh[0]**2 + sh[1]**2) sh_values.append(sh_value) boot_sh_values.append(sh_values) for i, sh_values in enumerate(boot_sh_values): plt.plot(np.arange(0, 80)/17.12, sh_values, color="#4682b4", alpha=0.25) plt.xlabel("Mask Radius, beam") plt.ylabel("Shift Value, pxl")
# Now clean and restore with circular beam for stokes in ('I', 'Q', 'U'): print "Cleaning stokes {} with circular restoring" \ " beam".format(stokes) # First restore with naitive beam to get it parameters clean_difmap(fname, map_fname(source, epoch, stokes, 'circ'), stokes, mapsize, path=data_dir, path_to_script=path_to_script, outpath=data_dir, beam_restore=circ_beam) i_image_circ =\ create_image_from_fits_file(os.path.join(data_dir, map_fname(source, epoch, 'I', 'circ'))) i_rms = i_image_circ.rms(region=(imsize[0] / 10., imsize[0] / 10., imsize[0] / 10., None)) print "I r.m.s. of circular-convolved map : {}".format(i_rms) # Calculate distance to most distant pixel with rms > 10 * rms mask = i_image_circ.image < n_rms_max * i_rms i_image_zeroed = i_image_circ.image.copy() i_image_zeroed[mask] = 0. y, x = np.nonzero(i_image_zeroed) y -= i_image.pixref[0] x -= i_image.pixref[1] distances = np.sqrt(x**2. + y**2.) max_dist = int(sorted(distances)[-1])
epoch = '2007_04_30' # Find core shift between each pair of frequencies low_band = 'c1' high_band = 'x2' im_fits_path_low = im_fits_path(source, low_band, epoch, stoke='i', base_path=base_path) im_fits_path_high = im_fits_path(source, high_band, epoch, stoke='i', base_path=base_path) image_low = create_image_from_fits_file( os.path.join(im_fits_path_low, 'cc_orig.fits')) image_high = create_image_from_fits_file( os.path.join(im_fits_path_high, 'cc_orig.fits')) shifts_orig = list() for r in range(0, 100, 5): region = (image_low.imsize[0] / 2, image_low.imsize[0] / 2, r, None) shift_orig = image_low.cross_correlate(image_high, region1=region, region2=region) shifts_orig.append(shift_orig) shifts_orig = np.vstack(shifts_orig) shifts_orig = shifts_orig[:, 0] + 1j * shifts_orig[:, 1] # Find bootstrapped distribution of shifts shifts_dict_boot = dict() for j in range(1, n_boot + 1):
import matplotlib.pyplot as plt # plt.matshow(np.log(image)) # plt.show() from model import Model from components import ImageComponent, CGComponent from from_fits import create_image_from_fits_file import os from uv_data import UVData original_uvdata = UVData( os.path.join('/home/ilya/Dropbox/0235/VLBA/', '0235+164.q1.2008_09_02.uvf_difmap')) noise = original_uvdata.noise() noise = {key: 0.25 * value for key, value in noise.items()} image_stack = create_image_from_fits_file( os.path.join('/home/ilya/Dropbox/0235/VLBA/', '0235+164.q1.stack.fits')) pixsize = abs(image_stack.pixsize[0]) imsize = image_stack.imsize model = Model(stokes='I') model.from_2darray(image, (0.01, 0.01)) # model.add_component(ImageComponent(image, x[0,:], # y[:,0])) # model.add_component(CGComponent(2.25, 0., 0., 0.01)) import copy model_uvdata = copy.deepcopy(original_uvdata) model_uvdata.substitute([model]) model_uvdata.noise_add(noise) #fig = original_uvdata.uvplot() model_uvdata.uvplot() model_uvdata.save('opening.fits', rewrite=True) # import astropy.io.fits as pf
uv_fits_path = os.path.join(data_dir, uv_fits_fname) out_fits_fname = 'cv_1000_cc_1000mas_shifter.fits' out_fits_path = os.path.join(data_dir, out_fits_fname) beam_fits_fname = 'cv_1000_cc.fits' beam_fits_path = os.path.join(data_dir, beam_fits_fname) clean_n(uv_fits_path, out_fits_fname, 'I', (512, 0.1), niter=niter, path_to_script=path_to_script, outpath=data_dir, show_difmap_output=True, shift=(1000, 0)) image = create_image_from_fits_file(out_fits_path) plt.matshow(image.image) patch = image.image[250:300, 250:300] _, _, _, C, A = variogram(patch) # Image already shifted so don't need rms_image_shifted here rms = rms_image(image) from image import plot as iplot ccimage = create_clean_image_from_fits_file(beam_fits_path) fig = iplot(image.image, image.image, x=image.x, y=image.y, show_beam=True, min_abs_level=rms, cmap='viridis',
images = Images() images.add_from_fits(wildcard=os.path.join(data_dir, "{}.*.common.*cn.fits".format(source))) # Find PPA uncertainty Hovatta's way # Uncertainty in EVPA calibration [deg] sigma_evpa = 2. # D-terms spread [dimensionless] are equal for all frequencies d_spread = 0.002 n_ant = 8 n_if = 2 n_scans = 10 # For all frequencies calculate D-term error image sigma_d_images_dict = dict() for l in ('l18', 'l20', 'l21', 'l22'): fname = "{}.{}.common.icn.fits".format(source, l) image = create_image_from_fits_file(os.path.join(data_dir, fname)) i_peak = np.max(image.image.ravel()) sigma_d_image = d_spread * np.sqrt(image.image ** 2. + (0.3 * i_peak) ** 2) / np.sqrt(n_ant * n_if * n_scans) sigma_d_images_dict.update({l: sigma_d_image}) # For all frequencies find rms for Q & U images rms_dict = dict() for l in ('l18', 'l20', 'l21', 'l22'): rms_dict[l] = dict() for stoke in ('q', 'u'): fname = "{}.{}.common.{}cn.fits".format(source, l, stoke) image = create_image_from_fits_file(os.path.join(data_dir, fname)) rms = image.rms(region=(40, 40, 40, None))
import numpy as np from itertools import combinations from sklearn import gaussian_process from from_fits import create_image_from_fits_file from simulations import simulate # First find best NCLEAN using cv_cc.py # Plot covariance matrix of the residuals (not difmap, but, probably, AIPS?) # Plot covariogramm, GP fit? if False: # Estimate correlation in image pixel values # FIXME: Better use residuals image from difmap or AIPS image_fits = '/home/ilya/code/vlbi_errors/vlbi_errors/residuals_3c273_15000.fits' image = create_image_from_fits_file(image_fits) slices = [slice(50 * i, 50 * (i + 1)) for i in range(20)] sigma2_list = list() for slice1, slice2 in list(combinations(slices, 2))[:51]: print "slices {} {}".format(slice1, slice2) data = image.image[slice1, slice2] X = list() y = list() for (i, j), val in np.ndenumerate(data): X.append([i, j]) y.append(val) Y = np.array(y).reshape(2500, 1) gp = gaussian_process.GaussianProcess(thetaL=(0.01, 0.01),
from my_utils import find_image_std, find_bbox data_dir = "/home/ilya/data/deep_clean" uvfits = "2200+420.u.2019_01_19.uvf" deep_clean_difmap(fname=uvfits, outfname="cc.fits", stokes="I", path=data_dir, outpath=data_dir, mapsize_clean=(1024, 0.1), path_to_script="/home/ilya/github/ve/difmap/final_clean_rms", show_difmap_output=False) ccimage = create_clean_image_from_fits_file(os.path.join(data_dir, "cc.fits")) dimage = create_image_from_fits_file(os.path.join(data_dir, "dmap_cc.fits")) beam = ccimage.beam npixels_beam = np.pi * beam[0] * beam[1] / 0.1**2 std = find_image_std(ccimage.image, beam_npixels=npixels_beam) # blc, trc = find_bbox(ccimage.image, level=4*std, min_maxintensity_mjyperbeam=4*std, # min_area_pix=2*npixels_beam, delta=10) fig = iplot( ccimage.image, dimage.image, x=ccimage.x, y=ccimage.y, abs_levels=[3 * std], colors_mask=None, color_clim=None, #blc=blc, trc=trc, beam=beam, close=False,
def get_stacked_map(source, mojave_dir=None, out_dir=None, imsize=(512, 0.1), path_to_script=None, epochs_slice=None): """ Functions that returns stacked image of given source using MOJAVE 15 GHz data downloading it directly from MOJAVE DB or getting it from user-specified directory. :param source: Source name [B1950]. :param mojave_dir: (optional) Path to directory with MOJAVE 15 GHz data. If ``None`` then download from MOJAVE DB. (default: ``None``) :param out_dir: (optional) Directory where to store files. If ``None`` then use CWD. (default: ``None``) :param imsize: (optional) Tuple of image size [pix], pixel size [mas] to put to difmap. (default: ``(512, 0.1)``) :param path_to_script: (optional) Path to difmap CLEANing script. If ``None`` then use CWD. (default: ``None``) :param epochs_slice: (optional) Slice of epochs sorted to process. If ``None`` then use all available epochs. (default: ``None``) :return: Numpy 2D array with stacked image. """ if path_to_script is None: path_to_script = os.getcwd() epochs = get_epochs_for_source(source) if epochs_slice is not None: epochs = epochs[::-1][epochs_slice] if out_dir is None: out_dir = os.getcwd() elif not os.path.exists(out_dir): os.makedirs(out_dir) if mojave_dir is None: download_mojave_uv_fits(source, epochs, bands=['u'], download_dir=out_dir) else: out_dir = mojave_dir beams_dict = dict() print "Output directory : {}".format(out_dir) # First clean and restore with native beam epoch_stokes_dict = dict() for epoch in sorted(epochs): print "Cleaning epoch {} with naitive restoring beam".format(epoch) uv_fits_fname = mojave_uv_fits_fname(source, 'u', epoch) uvdata = UVData(os.path.join(out_dir, uv_fits_fname)) uv_stokes = uvdata.stokes if 'RR' in uv_stokes and 'LL' in uv_stokes: stokes = 'I' elif 'RR' in uv_stokes and 'LL' not in uv_stokes: stokes = 'RR' elif 'LL' in uv_stokes and 'RR' not in uv_stokes: stokes = 'LL' else: continue epoch_stokes_dict.update({epoch: stokes}) im_fits_fname = "{}_{}_{}_{}.fits".format(source, 'U', epoch, stokes) print "Difmap params: " print "uv_fits_fname : {}".format(uv_fits_fname) print "im_fits_fname : {}".format(im_fits_fname) print "path : {}".format(out_dir) print "outpath: {}".format(out_dir) clean_difmap(uv_fits_fname, im_fits_fname, stokes, imsize, path=out_dir, path_to_script=path_to_script, outpath=out_dir) ccimage = create_clean_image_from_fits_file(os.path.join(out_dir, im_fits_fname)) beam = ccimage.beam print "Beam for epoch {} : {} [mas, mas, deg]".format(epoch, beam) beams_dict.update({epoch: (beam[0], beam[0], 0)}) circ_beam = np.mean([beam[0] for beam in beams_dict.values()]) # Now clean and restore with circular beam images = list() for epoch in sorted(epochs): stokes = epoch_stokes_dict[epoch] uv_fits_fname = mojave_uv_fits_fname(source, 'u', epoch) im_fits_fname = "{}_{}_{}_{}_circ.fits".format(source, 'U', epoch, stokes) print "Difmap params: " print "uv_fits_fname : {}".format(uv_fits_fname) print "im_fits_fname : {}".format(im_fits_fname) print "path : {}".format(out_dir) print "outpath: {}".format(out_dir) clean_difmap(uv_fits_fname, im_fits_fname, stokes, imsize, path=out_dir, path_to_script=path_to_script, outpath=out_dir, beam_restore=(circ_beam, circ_beam, 0)) image = create_image_from_fits_file(os.path.join(out_dir, im_fits_fname)) images.append(image.image.copy()) images = np.dstack(images) return np.mean(images, axis=2)
def analyze_source(uv_fits_paths, n_boot, imsizes=None, common_imsize=None, common_beam=None, find_shifts=False, outdir=None, path_to_script=None, clear_difmap_logs=True, rotm_slices=None): """ Function that uses multifrequency self-calibration data for in-depth analysis. :param uv_fits_paths: Iterable of paths to self-calibrated uv-data FITS-files. :param n_boot: Number of bootstrap replications to use in analysis. :param imsizes: (optional) Iterable of image parameters (imsize, pixsize) that should be used for CLEANing of uv-data if no CLEAN-images are supplied. Should be sorted in increasing frequency order. If ``None`` then specify parameters by CLEAN images. (default: ``None``) :param common_imsize: (optional) Image parameters that will be used in making common size images for multifrequency analysis. If ``None`` then use physical image size of lowest frequency and pixel size of highest frequency. (default: ``None``) :param outdir: (optional) Output directory. This directory will be used for saving picture, data, etc. If ``None`` then use CWD. (default: ``None``) :param path_to_script: (optional) Path ot difmap CLEAN script. If ``None`` then use CWD. (default: ``None``) :notes: Workflow: 1) Чистка с родным разрешением всех N диапазонов и получение родных моделей I, Q, U. 2) Выбор общей ДН из N возможных 3) (Опционально) Выбор uv-tapering 4) Чистка uv-данных для всех диапазонов с (опционально применяемым uv-tapering) общей ДН 5) Оценка сдвига ядра 6) Создание B наборов из N многочастотных симулированных данных используя родные модели 7) (Опционально) Чистка B наборов из N многочастотных симданных с родным разрешением для получения карт ошибок I для каждой из N частот 8) Чистка B наборов из N многочастотных симданных для всех диапазонов с (опционально применяемым uv-tapering) общей ДН 9) Оценка ошибки определения сдвига ядра 10) Оценка RM и ее ошибки 11) Оценка alpha и ее ошибки """ # Fail early if imsizes is None: raise Exception("Provide imsizes argument!") if common_imsize is not None: print("Using common image size {}".format(common_imsize)) else: raise Exception("Provide common_imsize argument!") # Setting up the output directory if outdir is None: outdir = os.getcwd() print("Using output directory {}".format(outdir)) os.chdir(outdir) # Assume input self-calibrated uv-data FITS files have different frequencies n_freq = len(uv_fits_paths) print("Using {} frequencies".format(n_freq)) # Assuming full multifrequency analysis stokes = ('I', 'Q', 'U') # Container for original self-calibrated uv-data uv_data_dict = dict() # Container for original self-calibrated uv-data FITS-file paths uv_fits_dict = dict() for uv_fits_path in uv_fits_paths: uvdata = UVData(uv_fits_path) # Mark frequencies by total band center [Hz] for consistency with image. uv_data_dict.update({uvdata.band_center: uvdata}) uv_fits_dict.update({uvdata.band_center: uv_fits_path}) # Lowest frequency goes first freqs = sorted(uv_fits_dict.keys()) print("Frequencies are: {}".format(freqs)) # Assert we have original map parameters for all frequencies assert len(imsizes) == n_freq # Container for original CLEAN-images of self-calibrated uv-data cc_image_dict = dict() # Container for paths to FITS-files with original CLEAN-images of # self-calibrated uv-data cc_fits_dict = dict() # Container for original CLEAN-image's beam parameters cc_beam_dict = dict() for freq in freqs: cc_image_dict.update({freq: dict()}) cc_fits_dict.update({freq: dict()}) cc_beam_dict.update({freq: dict()}) # 1. # Clean original uv-data with specified map parameters print("1. Clean original uv-data with specified map parameters...") imsizes_dict = dict() for i, freq in enumerate(freqs): imsizes_dict.update({freq: imsizes[i]}) for freq in freqs: uv_fits_path = uv_fits_dict[freq] uv_dir, uv_fname = os.path.split(uv_fits_path) for stoke in stokes: outfname = '{}_{}_cc.fits'.format(freq, stoke) outpath = os.path.join(outdir, outfname) clean_difmap(uv_fname, outfname, stoke, imsizes_dict[freq], path=uv_dir, path_to_script=path_to_script, outpath=outdir) cc_fits_dict[freq].update({stoke: os.path.join(outdir, outfname)}) image = create_clean_image_from_fits_file(outpath) cc_image_dict[freq].update({stoke: image}) if stoke == 'I': cc_beam_dict.update({freq: image.beam}) # Containers for images and paths to FITS files with common size images cc_cs_image_dict = dict() cc_cs_fits_dict = dict() # 2. # Choose common beam size print("2. Choosing common beam size...") if common_beam is None: common_beam = cc_beam_dict[freqs[0]] print("Using common beam [mas, mas, deg] : {}".format(common_beam)) # 3. # Optionally uv-tapering uv-data print("3. Optionally uv-tapering uv-data...") print("skipping...") # 4. # Clean original uv-data with common map parameters print("4. Clean original uv-data with common map parameters...") for freq in freqs: cc_cs_image_dict.update({freq: dict()}) cc_cs_fits_dict.update({freq: dict()}) uv_fits_path = uv_fits_dict[freq] uv_dir, uv_fname = os.path.split(uv_fits_path) for stoke in stokes: outfname = 'cs_{}_{}_cc.fits'.format(freq, stoke) outpath = os.path.join(outdir, outfname) # clean_difmap(uv_fname_cc, outfname, stoke, common_imsize, # path=uv_dir, path_to_script=path_to_script, # outpath=outdir, show_difmap_output=False) cc_cs_fits_dict[freq].update( {stoke: os.path.join(outdir, outfname)}) image = create_image_from_fits_file(outpath) cc_cs_image_dict[freq].update({stoke: image}) # 5. # Optionally find shifts between original CLEAN-images print("5. Optionally find shifts between original CLEAN-images...") if find_shifts: print("Determining images shift...") shift_dict = dict() freq_1 = freqs[0] image_1 = cc_image_dict[freq_1]['I'] for freq_2 in freqs[1:]: image_2 = cc_image_dict[freq_2]['I'] # Coarse grid of possible shifts shift = find_shift(image_1, image_2, 100, 5, max_mask_r=200, mask_step=5) # More accurate grid of possible shifts print("Using fine grid for accurate estimate") coarse_grid = range(0, 100, 5) idx = coarse_grid.index(shift) if idx > 0: min_shift = coarse_grid[idx - 1] else: min_shift = 0 shift = find_shift(image_1, image_2, coarse_grid[idx + 1], 1, min_shift=min_shift, max_mask_r=200, mask_step=5) shift_dict.update({str(( freq_1, freq_2, )): shift}) # Dumping shifts to json file in target directory with open(os.path.join(outdir, "shifts_original.json"), 'w') as fp: json.dump(shift_dict, fp) else: print("skipping...") # 6. # Bootstrap self-calibrated uv-data with CLEAN-models print("6. Bootstrap self-calibrated uv-data with CLEAN-models...") uv_boot_fits_dict = dict() for freq, uv_fits_path in uv_fits_dict.items(): # cc_fits_paths = [cc_fits_dict[freq][stoke] for stoke in stokes] # bootstrap_uv_fits(uv_fits_path, cc_fits_paths, n_boot, outpath=outdir, # outname=('boot_{}'.format(freq), '_uv.fits')) files = glob.glob(os.path.join(outdir, 'boot_{}*.fits'.format(freq))) uv_boot_fits_dict.update({freq: sorted(files)}) # 7. # Optionally clean bootstrap replications with original restoring beams and # map sizes to get error estimates for original resolution maps of I, PPOL, # FPOL, ... print( "7. Optionally clean bootstrap replications with original restoring" " beams and map sizes...") print("skipping...") # 8. # Optionally clean bootstrap replications with common restoring beams and # map sizes print( "8. Optionally clean bootstrap replications with common restoring" " beams and map sizes...") cc_boot_fits_dict = dict() for freq in freqs: cc_boot_fits_dict.update({freq: dict()}) uv_fits_paths = uv_boot_fits_dict[freq] for stoke in stokes: for i, uv_fits_path in enumerate(uv_fits_paths): uv_dir, uv_fname = os.path.split(uv_fits_path) outfname = 'boot_{}_{}_cc_{}.fits'.format( freq, stoke, str(i + 1).zfill(3)) # clean_difmap(uv_fname_cc, outfname, stoke, common_imsize, # path=uv_dir, path_to_script=path_to_script, # outpath=outdir, show_difmap_output=False) files = sorted( glob.glob( os.path.join(outdir, 'boot_{}_{}_cc_*.fits'.format(freq, stoke)))) cc_boot_fits_dict[freq].update({stoke: files}) # 9. Optionally estimate RM map and it's error print("9. Optionally estimate RM map and it's error...") original_cs_images = Images() for freq in freqs: for stoke in stokes: original_cs_images.add_images(cc_cs_image_dict[freq][stoke]) # Find rough mask for creating bootstrap images of RM, alpha, ... print("Finding rough mask for creating bootstrap images of RM, alpha, ...") cs_mask = pol_mask( {stoke: cc_cs_image_dict[freqs[-1]][stoke] for stoke in stokes}, n_sigma=3.) rotm_image, _ = original_cs_images.create_rotm_image(mask=cs_mask) boot_images = Images() fnames = sorted(glob.glob(os.path.join(data_dir, "boot_*_*_cc_*.fits"))) for freq in freqs: for stoke in stokes: boot_images.add_from_fits(cc_boot_fits_dict[freq][stoke]) boot_rotm_images = boot_images.create_rotm_images(mask=cs_mask) s_rotm_image = boot_rotm_images.create_error_image(cred_mass=0.95) if rotm_slices is not None: fnames = [ 'rotm_slice_spread_{}.png'.format(i + 1) for i in range(len(rotm_slices)) ] for rotm_slice, fname in zip(rotm_slices, fnames): analyze_rotm_slice(rotm_slice, rotm_image, boot_rotm_images, outdir=outdir, outfname=fname) # # Calculate simulataneous confidence bands # # Bootstrap slices # slices = list() # for image in rotm_images_sym.images: # slice_ = image.slice((216, 276), (296, 276)) # slices.append(slice_[~np.isnan(slice_)]) # # Find means # obs_slice = rotm_image_sym.slice((216, 276), (296, 276)) # x = np.arange(216, 296, 1) # x = x[~np.isnan(obs_slice)] # obs_slice = obs_slice[~np.isnan(obs_slice)] # # Find sigmas # slices_ = [arr.reshape((1, len(obs_slice))) for arr in slices] # sigmas = hdi_of_arrays(slices_).squeeze() # means = np.mean(np.vstack(slices), axis=0) # diff = obs_slice - means # # Move bootstrap curves to original simulated centers # slices_ = [slice_ + diff for slice_ in slices] # # Find low and upper confidence band # low, up = create_sim_conf_band(slices_, obs_slice, sigmas, # alpha=conf_band_alpha) # # Plot confidence bands and model values # fig = plt.figure() # ax = fig.add_subplot(1, 1, 1) # ax.plot(x, low[::-1], 'g') # ax.plot(x, up[::-1], 'g') # [ax.plot(x, slice_[::-1], 'r', lw=0.15) for slice_ in slices_] # ax.plot(x, obs_slice[::-1], '.k') # # Plot ROTM model # ax.plot(np.arange(216, 296, 1), # rotm_grad_value * (np.arange(216, 296, 1) - 256.)[::-1] + # rotm_value_0) # fig.savefig(os.path.join(data_dir, 'rotm_slice_spread.png'), # bbox_inches='tight', dpi=200) # plt.close() if clear_difmap_logs: print("Removing difmap log-files...") difmap_logs = glob.glob(os.path.join(outdir, "difmap.log*")) for difmpa_log in difmap_logs: os.unlink(difmpa_log)