else: raise if len(fn2) == 1: fn2 = fn2[0] elif len(fn2) == 0: continue else: raise try: with warnings.catch_warnings(): warnings.filterwarnings('ignore') ax1, ax2, ax3, fig, diffstats = make_comparison_image( fn2, fn1, title1='February 2021', title2='May 2021', writediff=True, allow_zero_diff=True) if not os.path.exists(f"{outpath}"): os.mkdir(f"{outpath}") fig.savefig( f"{outpath}/{field}_B{band}_{config}_{imtype}_FvM_comparison.png", bbox_inches='tight') except IndexError: raise except Exception as ex: log.error(f"Failure for {path1} {path2}") log.error((field, band, config, imtype, ex)) raise ex #continue
fig = pl.figure(1, figsize=(14,6)) if fig.get_figheight() != 6: fig.set_figheight(6) if fig.get_figwidth() != 14: fig.set_figwidth(14) bsens = '_bsens' if 'bsens' in postselfcal_name else '' try: with warnings.catch_warnings(): warnings.filterwarnings('ignore') ax1, ax2, ax3, fig, diffstats = make_comparison_image(feb_postselfcal, may_postselfcal, title1='February2021', title2='May2021', writediff=False) fig.savefig(f"{figpath}/{field}_B{band}_{config}{bsens}_selfcal{last_selfcal}_MayVsFeb2021_comparison.png", bbox_inches='tight') except IndexError: raise except Exception as ex: log.error(f"Failure for pre={preselfcal_name} post={postselfcal_name}") log.error((field, band, config, imtype, ex)) raise ex #continue matchrow = ((tbl['region'] == field) & (tbl['band'] == f'B{band}') & (tbl['array'] == ('12Monly' if config == '12M' else config)) & (tbl['robust'] == 'r0.0') &
fig = pl.figure(1, figsize=(14, 6)) if fig.get_figheight() != 6: fig.set_figheight(6) if fig.get_figwidth() != 14: fig.set_figwidth(14) bsens = '_bsens' if 'bsens' in postselfcal_name else '' nobright = '_nobright' if 'noco' in postselfcal_name or 'non2hp' in postselfcal_name else '' try: with warnings.catch_warnings(): warnings.filterwarnings('ignore') ax1, ax2, ax3, fig, diffstats = make_comparison_image( preselfcal_name, postselfcal_name, title1='Preselfcal', title2='Postselfcal', writediff=True) if not os.path.exists( f"{basepath}/{field}/B{band}/comparisons/"): os.mkdir( f"{basepath}/{field}/B{band}/comparisons/") if not os.path.exists( f"{sharepath}/comparison_images/"): os.mkdir(f"{sharepath}/comparison_images/") pl.suptitle(f"{field} B{band}") fig.savefig( f"{basepath}/{field}/B{band}/comparisons/{field}_B{band}_{config}{bsens}{nobright}_selfcal{last_selfcal}_comparison.png", bbox_inches='tight') shutil.copy( f"{basepath}/{field}/B{band}/comparisons/{field}_B{band}_{config}{bsens}{nobright}_selfcal{last_selfcal}_comparison.png",
print(ex) continue fig = pl.figure(1, figsize=(14, 6)) if fig.get_figheight() != 6: fig.set_figheight(6) if fig.get_figwidth() != 14: fig.set_figwidth(14) try: with warnings.catch_warnings(): warnings.filterwarnings('ignore') ax1, ax2, ax3, fig, diffstats = make_comparison_image( filename1=cleanest, filename2=f7m12m, title1='cleanest', title2='7m12m', allow_reproj=True) except IndexError: raise except Exception as ex: log.error(f"Failure for 7m12m={f7m12m} cleanest={cleanest}") log.error((field, band, config, ex)) raise continue if not os.path.exists(f"{basepath}/{field}/B{band}/comparisons/"): os.mkdir(f"{basepath}/{field}/B{band}/comparisons/") pl.savefig( f"{basepath}/{field}/B{band}/comparisons/{field}_B{band}_{config}_7m12m_vs_cleanest_comparison.png", bbox_inches='tight',
continue fig = pl.figure(1, figsize=(14,6)) if fig.get_figheight() != 6: fig.set_figheight(6) if fig.get_figwidth() != 14: fig.set_figwidth(14) try: with warnings.catch_warnings(): warnings.filterwarnings('ignore') ax1, ax2, ax3, fig, diffstats = make_comparison_image(filename1=cleanest, filename2=bsens, title1='cleanest', title2='bsens', allow_reproj=allow_reproj, ) except IndexError: raise except Exception as ex: log.error(f"Failure for bsens={bsens} cleanest={cleanest}") log.error((field, band, config, ex)) raise continue if not os.path.exists(f"{basepath}/{field}/B{band}/comparisons/"): os.mkdir(f"{basepath}/{field}/B{band}/comparisons/") pl.savefig(f"{basepath}/{field}/B{band}/comparisons/{field}_B{band}_{config}_bsens_vs_cleanest_comparison.png", bbox_inches='tight', dpi=200)
continue fig = pl.figure(1, figsize=(14, 6)) if fig.get_figheight() != 6: fig.set_figheight(6) if fig.get_figwidth() != 14: fig.set_figwidth(14) try: with warnings.catch_warnings(): warnings.filterwarnings('ignore') ax1, ax2, ax3, fig, diffstats = make_comparison_image( filename1=cleanest, filename2=bsens, title1='cleanest', title2='bsens', allow_reproj=allow_reproj, writediff=True, diff_suffix='.bsens-cleanest') except IndexError: raise except Exception as ex: log.error(f"Failure for bsens={bsens} cleanest={cleanest}") log.error((field, band, config, ex)) raise continue if not os.path.exists( f"{basepath}/{field}/B{band}/comparisons/"): os.mkdir(f"{basepath}/{field}/B{band}/comparisons/") pl.savefig(
#field = fn.split("_uid")[0].split("/")[-1] filepath = fn.split("bsens")[0] bsens_fh = fits.open(bsens) try: clean_fh = fits.open(cleanest) except Exception as ex: log.error(f"Failed to open 'cleanest' image {cleanest}") print(ex) continue try: with warnings.catch_warnings(): warnings.filterwarnings('ignore') ax1, ax2, ax3, fig, diffstats = make_comparison_image( bsens, cleanest, title1='cleanest', title2='bsens') except IndexError: raise except Exception as ex: log.error(f"Failure for bsens={bsens} cleanest={cleanest}") log.error((field, band, config, ex)) raise continue if not os.path.exists(f"{basepath}/{field}/B{band}/comparisons/"): os.mkdir(f"{basepath}/{field}/B{band}/comparisons/") pl.savefig( f"{basepath}/{field}/B{band}/comparisons/{field}_B{band}_{config}_bsens_vs_cleanest_comparison.png", bbox_inches='tight', dpi=200)
def bsens_comparison(release="June2021Release"): cwd = os.getcwd() basepath = f'/orange/adamginsburg/web/secure/ALMA-IMF/{release}' basepath = f'/orange/adamginsburg/ALMA_IMF/2017.1.01355.L/{release}/' os.chdir(basepath) sharepath = f'/orange/adamginsburg/web/secure/ALMA-IMF/{release}/' if not os.path.exists(f"{sharepath}/bsens_comparison_images/"): os.mkdir(f"{sharepath}/bsens_comparison_images/") import imstats #tbl = imstats.savestats(basepath=basepath) #tbl = Table.read('/orange/adamginsburg/web/secure/ALMA-IMF/June2021Release/tables/metadata.ecsv') tbl = Table.read( f'/orange/adamginsburg/ALMA_IMF/2017.1.01355.L/{release}/tables/metadata_image.tt0.ecsv' ) tbl.add_column( Column(name='casaversion_bsens', data=[' '] * len(tbl))) tbl.add_column( Column(name='casaversion_cleanest', data=[' '] * len(tbl))) tbl.add_column(Column(name='bsens_fn', data=[' ' * 200] * len(tbl))) tbl.add_column(Column(name='cleanest_fn', data=[' ' * 200] * len(tbl))) tbl.add_column(Column(name='contselMaxDiff', data=[np.nan] * len(tbl))) tbl.add_column(Column(name='contselMinDiff', data=[np.nan] * len(tbl))) tbl.add_column(Column(name='contselMADDiff', data=[np.nan] * len(tbl))) tbl.add_column(Column(name='contselMeanDiff', data=[np.nan] * len(tbl))) tbl.add_column(Column(name='contselMedianDiff', data=[np.nan] * len(tbl))) tbl.add_column(Column(name='contselSumDiff', data=[np.nan] * len(tbl))) tbl.add_column(Column(name='shape', data=[np.nan] * len(tbl))) tbl.add_column(Column(name='ppbeam', data=[np.nan] * len(tbl))) tbl.add_column(Column(name='dr_bsens', data=[np.nan] * len(tbl))) tbl.add_column(Column(name='dr_cleanest', data=[np.nan] * len(tbl))) tbl.add_column(Column(name='masksum_bsens', data=[np.nan] * len(tbl))) tbl.add_column(Column(name='masksum_cleanest', data=[np.nan] * len(tbl))) tbl.add_column(Column(name='sum_bsens', data=[np.nan] * len(tbl))) tbl.add_column(Column(name='sum_cleanest', data=[np.nan] * len(tbl))) tbl.add_column(Column(name='sum_diff', data=[np.nan] * len(tbl))) tbl.add_column(Column(name='min_bsens', data=[np.nan] * len(tbl))) tbl.add_column(Column(name='min_cleanest', data=[np.nan] * len(tbl))) tbl.add_column(Column(name='max_bsens', data=[np.nan] * len(tbl))) tbl.add_column(Column(name='max_cleanest', data=[np.nan] * len(tbl))) tbl.add_column(Column(name='mad_bsens', data=[np.nan] * len(tbl))) tbl.add_column(Column(name='mad_cleanest', data=[np.nan] * len(tbl))) tbl.add_column(Column(name='mad_sample_bsens', data=[np.nan] * len(tbl))) tbl.add_column(Column(name='mad_sample_cleanest', data=[np.nan] * len(tbl))) tbl.add_column(Column(name='std_sample_bsens', data=[np.nan] * len(tbl))) tbl.add_column(Column(name='std_sample_cleanest', data=[np.nan] * len(tbl))) tbl.add_column( Column(name='dr_improvement_bsens', data=[np.nan] * len(tbl))) for field in "G008.67 G337.92 W43-MM3 G328.25 G351.77 G012.80 G327.29 W43-MM1 G010.62 W51-IRS2 W43-MM2 G333.60 G338.93 W51-E G353.41".split( ): for band in (3, 6): for config in ("12M", ): # "7M12M"): for suffix in ('image.tt0.fits', 'image.tt0.pbcor.fits'): fns = glob.glob( f"{basepath}/{field}/B{band}/bsens/*_{config}_robust0_*finaliter*.{suffix}" ) if len(fns) > 1: raise ValueError("Too many matches!") elif len(fns) == 0: log.error( f"No matches to field={field} band={band} config={config}" ) continue raise ValueError("No matches!") fn = fns[0] pl.clf() bsens = fn cleanest = fn.replace("_bsens", "").replace("/bsens/", "/cleanest/") #print(os.path.exists(bsens), os.path.exists(clean)) #field = fn.split("_uid")[0].split("/")[-1] filepath = fn.split("bsens")[0] try: bsens_cube = SpectralCube.read( bsens, format='fits' if 'fits' in bsens else 'casa_image') except Exception as ex: log.error(f"Failed to open 'bsens' image {bsens}") raise ex if not os.path.exists(cleanest): # hackaround for mismatched UID names, which shouldn't happen but did ind = cleanest.find('uid') cleanest_glob = cleanest[:ind] + "*" + cleanest[ind + 21:] cleanest_fl = glob.glob(cleanest_glob) if len(cleanest_fl) > 0: cleanest = cleanest_fl[0] if len(cleanest_fl) > 1: log.warn( "WARNING: found multiple 'cleanest' matches {0}" .format(cleanest_fl)) log.warn( f"Replaced 'cleanest' with {cleanest} to match {bsens}" ) allow_reproj = True else: allow_reproj = False if not os.path.exists(cleanest): # hackaround for mismatched number of selfcal iterations cfns = glob.glob( f"{basepath}/{field}/B{band}/cleanest/*_{config}_robust0_*finaliter*.{suffix}" ) if len(cfns) == 1: log.info( f"Replaced original cleanest {cleanest} with {cfns[0]}" ) allow_reproj = False cleanest = cfns[0] elif len(cfns) == 0: log.warn( f"Did not find any cleanest matches to {fn}") else: log.warn(f"Found too many matches: {cfns}") try: clean_cube = SpectralCube.read( cleanest, format='fits' if 'fits' in cleanest else 'casa_image') except Exception as ex: log.error( f"Failed to open 'cleanest' image {cleanest} (check for a bsens-cleanest mismatch)" ) print(ex) #raise continue fig = pl.figure(1, figsize=(14, 6)) if fig.get_figheight() != 6: fig.set_figheight(6) if fig.get_figwidth() != 14: fig.set_figwidth(14) try: with warnings.catch_warnings(): warnings.filterwarnings('ignore') ax1, ax2, ax3, fig, diffstats = make_comparison_image( filename1=cleanest, filename2=bsens, title1='cleanest', title2='bsens', allow_reproj=allow_reproj, writediff=True, diff_suffix='.bsens-cleanest') except IndexError: raise except Exception as ex: log.error( f"Failure for bsens={bsens} cleanest={cleanest}") log.error((field, band, config, ex)) raise continue if not os.path.exists( f"{basepath}/{field}/B{band}/comparisons/"): os.mkdir(f"{basepath}/{field}/B{band}/comparisons/") pl.suptitle(f"{field} B{band}") pl.savefig( f"{basepath}/{field}/B{band}/comparisons/{field}_B{band}_{config}_bsens_vs_cleanest_comparison.png", bbox_inches='tight', dpi=200) pl.savefig( f"{basepath}/{field}/B{band}/comparisons/{field}_B{band}_{config}_bsens_vs_cleanest_comparison.pdf", bbox_inches='tight', dpi=200) shutil.copy( f"{basepath}/{field}/B{band}/comparisons/{field}_B{band}_{config}_bsens_vs_cleanest_comparison.png", f"{sharepath}/bsens_comparison_images/") shutil.copy( f"{basepath}/{field}/B{band}/comparisons/{field}_B{band}_{config}_bsens_vs_cleanest_comparison.pdf", f"{sharepath}/bsens_comparison_images/") matchrow = ( (tbl['region'] == field) & (tbl['band'] == f'B{band}') & (tbl['array'] == ('12Monly' if config == '12M' else config)) & (tbl['pbcor'] if 'pbcor' in suffix else ~tbl['pbcor']) & (tbl['robust'] == 'r0.0')) tbl['contselMaxDiff'][matchrow] = diffstats['max'] tbl['contselMinDiff'][matchrow] = diffstats['min'] tbl['contselMADDiff'][matchrow] = diffstats['mad'] tbl['contselMeanDiff'][matchrow] = diffstats['mean'] tbl['contselMedianDiff'][matchrow] = diffstats['median'] tbl['contselSumDiff'][matchrow] = diffstats['sum'] tbl['bsens_fn'][matchrow] = os.path.basename(bsens) tbl['cleanest_fn'][matchrow] = os.path.basename(cleanest) tbl['dr_bsens'][matchrow] = diffstats['dr_post'] tbl['dr_cleanest'][matchrow] = diffstats['dr_pre'] tbl['min_bsens'][matchrow] = diffstats['min_post'] tbl['min_cleanest'][matchrow] = diffstats['min_pre'] tbl['max_bsens'][matchrow] = diffstats['max_post'] tbl['max_cleanest'][matchrow] = diffstats['max_pre'] tbl['sum_bsens'][matchrow] = diffstats['sum_post'] tbl['sum_cleanest'][matchrow] = diffstats['sum_pre'] tbl['masksum_bsens'][matchrow] = diffstats['masksum_post'] tbl['masksum_cleanest'][matchrow] = diffstats[ 'masksum_pre'] tbl['shape'][matchrow] = diffstats['shape'] tbl['ppbeam'][matchrow] = diffstats['ppbeam'] tbl['mad_bsens'][matchrow] = diffstats['mad_post'] tbl['mad_cleanest'][matchrow] = diffstats['mad_pre'] tbl['mad_sample_bsens'][matchrow] = diffstats[ 'mad_sample_post'] tbl['mad_sample_cleanest'][matchrow] = diffstats[ 'mad_sample_pre'] tbl['std_sample_bsens'][matchrow] = diffstats[ 'std_sample_post'] tbl['std_sample_cleanest'][matchrow] = diffstats[ 'std_sample_pre'] tbl['dr_improvement_bsens'][ matchrow] = diffstats['dr_post'] / diffstats['dr_pre'] tbl['casaversion_bsens'][matchrow] = fits.getheader( bsens)['ORIGIN'] tbl['casaversion_cleanest'][matchrow] = fits.getheader( cleanest)['ORIGIN'] print(fns) print(f"{field}_B{band}: matched {matchrow.sum()} rows") print() formats = { 'dr_improvement_bsens': lambda x: '{0:0.2f}'.format(x), 'contselMaxDiff': lambda x: f'{x:0.6g}', 'BeamVsReq': lambda x: f'{x:0.2f}', } for bp in (f'/orange/adamginsburg/web/secure/ALMA-IMF/{release}', f'/orange/adamginsburg/ALMA_IMF/2017.1.01355.L/{release}'): tbl.write(f'{bp}/tables/metadata_bsens_cleanest.ecsv', overwrite=True) tbl.write(f'{bp}/tables/metadata_bsens_cleanest.html', formats=formats, format='ascii.html', overwrite=True) tbl.write(f'{bp}/tables/metadata_bsens_cleanest.tex', formats=formats, overwrite=True) tbl.write( f'{bp}/tables/metadata_bsens_cleanest.js.html', #formats=formats, format='jsviewer') os.chdir(cwd) # paper figure config = '12M' band = 6 field = 'G351.77' cleanest = f'{basepath}/{field}/B{band}/cleanest/{field}_B{band}_uid___A001_X1296_X201_continuum_merged_12M_robust0_preselfcal_finalmodel.image.tt0' bsens = f'{basepath}/{field}/B{band}/bsens/{field}_B{band}_uid___A001_X1296_X201_continuum_merged_bsens_12M_robust0_selfcal4_finaliter.image.tt0' ax1, ax2, ax3, fig, diffstats = make_comparison_image( filename1=cleanest, filename2=bsens, title1='cleanest', title2='bsens', writediff=True, diff_suffix='.bsens-cleanest') pl.savefig( f"{basepath}/{field}/B{band}/comparisons/{field}_B{band}_{config}_bsens_vs_cleanest_comparison.png", bbox_inches='tight', dpi=200) pl.savefig( f"{basepath}/{field}/B{band}/comparisons/{field}_B{band}_{config}_bsens_vs_cleanest_comparison.pdf", bbox_inches='tight', dpi=200)
if len(glb) == 1: autoname = glb[0] else: print(f"Skipping {basename} because there's no match.") continue field = basename.split("_uid")[0].split("/")[-1] fh1 = fits.open(fn) fh2 = fits.open(autoname) if (fh1[0].data.shape == fh2[0].data.shape): try: make_comparison_image(fn, autoname, title1='Delivered', title2='Auto', allow_reproj=True) except ValueError as ex: if 'WCS' in str(ex): print(str(ex)) continue else: raise ex pl.savefig( f"{basepath}/comparisons/auto_vs_not_{basename.replace('.fits','')}.png", bbox_inches='tight', dpi=200) else: print(f"Skipping {fn} because there was a shape mismatch.")
imresults = '/orange/adamginsburg/ALMA_IMF/2017.1.01355.L/imaging_results' for fn in filelist: pl.clf() basename = os.path.split(fn)[-1] if os.path.exists(f'{imresults}/{basename}'): autoname = f'{imresults}/{basename}' else: pfx = basename.split("uid")[0] sfx = basename.split("continuum_merged")[1] glb = glob.glob(f"{imresults}/{pfx}*{sfx}") if len(glb) == 1: autoname = glb[0] else: print(f"Skipping {basename} because there's no match.") continue field = basename.split("_uid")[0].split("/")[-1] fh1 = fits.open(fn) fh2 = fits.open(autoname) if (fh1[0].data.shape == fh2[0].data.shape): make_comparison_image(fn, autoname, title1='Delivered', title2='Auto') pl.savefig( f"{basepath}/comparisons/auto_vs_not_{basename.replace('.fits','')}.png", bbox_inches='tight', dpi=200) else: print(f"Skipping {fn} because there was a shape mismatch.")