else:
raise
if len(fn2) == 1:
fn2 = fn2[0]
elif len(fn2) == 0:
continue
else:
raise
try:
with warnings.catch_warnings():
warnings.filterwarnings('ignore')
ax1, ax2, ax3, fig, diffstats = make_comparison_image(
fn2,
fn1,
title1='February 2021',
title2='May 2021',
writediff=True,
allow_zero_diff=True)
if not os.path.exists(f"{outpath}"):
os.mkdir(f"{outpath}")
fig.savefig(
f"{outpath}/{field}_B{band}_{config}_{imtype}_FvM_comparison.png",
bbox_inches='tight')
except IndexError:
raise
except Exception as ex:
log.error(f"Failure for {path1} {path2}")
log.error((field, band, config, imtype, ex))
raise ex
#continue
fig = pl.figure(1, figsize=(14,6))
if fig.get_figheight() != 6:
fig.set_figheight(6)
if fig.get_figwidth() != 14:
fig.set_figwidth(14)
bsens = '_bsens' if 'bsens' in postselfcal_name else ''
try:
with warnings.catch_warnings():
warnings.filterwarnings('ignore')
ax1, ax2, ax3, fig, diffstats = make_comparison_image(feb_postselfcal,
may_postselfcal,
title1='February2021',
title2='May2021',
writediff=False)
fig.savefig(f"{figpath}/{field}_B{band}_{config}{bsens}_selfcal{last_selfcal}_MayVsFeb2021_comparison.png", bbox_inches='tight')
except IndexError:
raise
except Exception as ex:
log.error(f"Failure for pre={preselfcal_name} post={postselfcal_name}")
log.error((field, band, config, imtype, ex))
raise ex
#continue
matchrow = ((tbl['region'] == field) &
(tbl['band'] == f'B{band}') &
(tbl['array'] == ('12Monly' if config == '12M' else config)) &
(tbl['robust'] == 'r0.0') &
fig = pl.figure(1, figsize=(14, 6))
if fig.get_figheight() != 6:
fig.set_figheight(6)
if fig.get_figwidth() != 14:
fig.set_figwidth(14)
bsens = '_bsens' if 'bsens' in postselfcal_name else ''
nobright = '_nobright' if 'noco' in postselfcal_name or 'non2hp' in postselfcal_name else ''
try:
with warnings.catch_warnings():
warnings.filterwarnings('ignore')
ax1, ax2, ax3, fig, diffstats = make_comparison_image(
preselfcal_name,
postselfcal_name,
title1='Preselfcal',
title2='Postselfcal',
writediff=True)
if not os.path.exists(
f"{basepath}/{field}/B{band}/comparisons/"):
os.mkdir(
f"{basepath}/{field}/B{band}/comparisons/")
if not os.path.exists(
f"{sharepath}/comparison_images/"):
os.mkdir(f"{sharepath}/comparison_images/")
pl.suptitle(f"{field} B{band}")
fig.savefig(
f"{basepath}/{field}/B{band}/comparisons/{field}_B{band}_{config}{bsens}{nobright}_selfcal{last_selfcal}_comparison.png",
bbox_inches='tight')
shutil.copy(
f"{basepath}/{field}/B{band}/comparisons/{field}_B{band}_{config}{bsens}{nobright}_selfcal{last_selfcal}_comparison.png",
print(ex)
continue
fig = pl.figure(1, figsize=(14, 6))
if fig.get_figheight() != 6:
fig.set_figheight(6)
if fig.get_figwidth() != 14:
fig.set_figwidth(14)
try:
with warnings.catch_warnings():
warnings.filterwarnings('ignore')
ax1, ax2, ax3, fig, diffstats = make_comparison_image(
filename1=cleanest,
filename2=f7m12m,
title1='cleanest',
title2='7m12m',
allow_reproj=True)
except IndexError:
raise
except Exception as ex:
log.error(f"Failure for 7m12m={f7m12m} cleanest={cleanest}")
log.error((field, band, config, ex))
raise
continue
if not os.path.exists(f"{basepath}/{field}/B{band}/comparisons/"):
os.mkdir(f"{basepath}/{field}/B{band}/comparisons/")
pl.savefig(
f"{basepath}/{field}/B{band}/comparisons/{field}_B{band}_{config}_7m12m_vs_cleanest_comparison.png",
bbox_inches='tight',
continue
fig = pl.figure(1, figsize=(14,6))
if fig.get_figheight() != 6:
fig.set_figheight(6)
if fig.get_figwidth() != 14:
fig.set_figwidth(14)
try:
with warnings.catch_warnings():
warnings.filterwarnings('ignore')
ax1, ax2, ax3, fig, diffstats = make_comparison_image(filename1=cleanest,
filename2=bsens,
title1='cleanest',
title2='bsens',
allow_reproj=allow_reproj,
)
except IndexError:
raise
except Exception as ex:
log.error(f"Failure for bsens={bsens} cleanest={cleanest}")
log.error((field, band, config, ex))
raise
continue
if not os.path.exists(f"{basepath}/{field}/B{band}/comparisons/"):
os.mkdir(f"{basepath}/{field}/B{band}/comparisons/")
pl.savefig(f"{basepath}/{field}/B{band}/comparisons/{field}_B{band}_{config}_bsens_vs_cleanest_comparison.png",
bbox_inches='tight', dpi=200)
continue
fig = pl.figure(1, figsize=(14, 6))
if fig.get_figheight() != 6:
fig.set_figheight(6)
if fig.get_figwidth() != 14:
fig.set_figwidth(14)
try:
with warnings.catch_warnings():
warnings.filterwarnings('ignore')
ax1, ax2, ax3, fig, diffstats = make_comparison_image(
filename1=cleanest,
filename2=bsens,
title1='cleanest',
title2='bsens',
allow_reproj=allow_reproj,
writediff=True,
diff_suffix='.bsens-cleanest')
except IndexError:
raise
except Exception as ex:
log.error(f"Failure for bsens={bsens} cleanest={cleanest}")
log.error((field, band, config, ex))
raise
continue
if not os.path.exists(
f"{basepath}/{field}/B{band}/comparisons/"):
os.mkdir(f"{basepath}/{field}/B{band}/comparisons/")
pl.savefig(
#field = fn.split("_uid")[0].split("/")[-1]
filepath = fn.split("bsens")[0]
bsens_fh = fits.open(bsens)
try:
clean_fh = fits.open(cleanest)
except Exception as ex:
log.error(f"Failed to open 'cleanest' image {cleanest}")
print(ex)
continue
try:
with warnings.catch_warnings():
warnings.filterwarnings('ignore')
ax1, ax2, ax3, fig, diffstats = make_comparison_image(
bsens, cleanest, title1='cleanest', title2='bsens')
except IndexError:
raise
except Exception as ex:
log.error(f"Failure for bsens={bsens} cleanest={cleanest}")
log.error((field, band, config, ex))
raise
continue
if not os.path.exists(f"{basepath}/{field}/B{band}/comparisons/"):
os.mkdir(f"{basepath}/{field}/B{band}/comparisons/")
pl.savefig(
f"{basepath}/{field}/B{band}/comparisons/{field}_B{band}_{config}_bsens_vs_cleanest_comparison.png",
bbox_inches='tight',
dpi=200)
def bsens_comparison(release="June2021Release"):
cwd = os.getcwd()
basepath = f'/orange/adamginsburg/web/secure/ALMA-IMF/{release}'
basepath = f'/orange/adamginsburg/ALMA_IMF/2017.1.01355.L/{release}/'
os.chdir(basepath)
sharepath = f'/orange/adamginsburg/web/secure/ALMA-IMF/{release}/'
if not os.path.exists(f"{sharepath}/bsens_comparison_images/"):
os.mkdir(f"{sharepath}/bsens_comparison_images/")
import imstats
#tbl = imstats.savestats(basepath=basepath)
#tbl = Table.read('/orange/adamginsburg/web/secure/ALMA-IMF/June2021Release/tables/metadata.ecsv')
tbl = Table.read(
f'/orange/adamginsburg/ALMA_IMF/2017.1.01355.L/{release}/tables/metadata_image.tt0.ecsv'
)
tbl.add_column(
Column(name='casaversion_bsens', data=[' '] * len(tbl)))
tbl.add_column(
Column(name='casaversion_cleanest', data=[' '] * len(tbl)))
tbl.add_column(Column(name='bsens_fn', data=[' ' * 200] * len(tbl)))
tbl.add_column(Column(name='cleanest_fn', data=[' ' * 200] * len(tbl)))
tbl.add_column(Column(name='contselMaxDiff', data=[np.nan] * len(tbl)))
tbl.add_column(Column(name='contselMinDiff', data=[np.nan] * len(tbl)))
tbl.add_column(Column(name='contselMADDiff', data=[np.nan] * len(tbl)))
tbl.add_column(Column(name='contselMeanDiff', data=[np.nan] * len(tbl)))
tbl.add_column(Column(name='contselMedianDiff', data=[np.nan] * len(tbl)))
tbl.add_column(Column(name='contselSumDiff', data=[np.nan] * len(tbl)))
tbl.add_column(Column(name='shape', data=[np.nan] * len(tbl)))
tbl.add_column(Column(name='ppbeam', data=[np.nan] * len(tbl)))
tbl.add_column(Column(name='dr_bsens', data=[np.nan] * len(tbl)))
tbl.add_column(Column(name='dr_cleanest', data=[np.nan] * len(tbl)))
tbl.add_column(Column(name='masksum_bsens', data=[np.nan] * len(tbl)))
tbl.add_column(Column(name='masksum_cleanest', data=[np.nan] * len(tbl)))
tbl.add_column(Column(name='sum_bsens', data=[np.nan] * len(tbl)))
tbl.add_column(Column(name='sum_cleanest', data=[np.nan] * len(tbl)))
tbl.add_column(Column(name='sum_diff', data=[np.nan] * len(tbl)))
tbl.add_column(Column(name='min_bsens', data=[np.nan] * len(tbl)))
tbl.add_column(Column(name='min_cleanest', data=[np.nan] * len(tbl)))
tbl.add_column(Column(name='max_bsens', data=[np.nan] * len(tbl)))
tbl.add_column(Column(name='max_cleanest', data=[np.nan] * len(tbl)))
tbl.add_column(Column(name='mad_bsens', data=[np.nan] * len(tbl)))
tbl.add_column(Column(name='mad_cleanest', data=[np.nan] * len(tbl)))
tbl.add_column(Column(name='mad_sample_bsens', data=[np.nan] * len(tbl)))
tbl.add_column(Column(name='mad_sample_cleanest',
data=[np.nan] * len(tbl)))
tbl.add_column(Column(name='std_sample_bsens', data=[np.nan] * len(tbl)))
tbl.add_column(Column(name='std_sample_cleanest',
data=[np.nan] * len(tbl)))
tbl.add_column(
Column(name='dr_improvement_bsens', data=[np.nan] * len(tbl)))
for field in "G008.67 G337.92 W43-MM3 G328.25 G351.77 G012.80 G327.29 W43-MM1 G010.62 W51-IRS2 W43-MM2 G333.60 G338.93 W51-E G353.41".split(
):
for band in (3, 6):
for config in ("12M", ): # "7M12M"):
for suffix in ('image.tt0.fits', 'image.tt0.pbcor.fits'):
fns = glob.glob(
f"{basepath}/{field}/B{band}/bsens/*_{config}_robust0_*finaliter*.{suffix}"
)
if len(fns) > 1:
raise ValueError("Too many matches!")
elif len(fns) == 0:
log.error(
f"No matches to field={field} band={band} config={config}"
)
continue
raise ValueError("No matches!")
fn = fns[0]
pl.clf()
bsens = fn
cleanest = fn.replace("_bsens",
"").replace("/bsens/", "/cleanest/")
#print(os.path.exists(bsens), os.path.exists(clean))
#field = fn.split("_uid")[0].split("/")[-1]
filepath = fn.split("bsens")[0]
try:
bsens_cube = SpectralCube.read(
bsens,
format='fits' if 'fits' in bsens else 'casa_image')
except Exception as ex:
log.error(f"Failed to open 'bsens' image {bsens}")
raise ex
if not os.path.exists(cleanest):
# hackaround for mismatched UID names, which shouldn't happen but did
ind = cleanest.find('uid')
cleanest_glob = cleanest[:ind] + "*" + cleanest[ind +
21:]
cleanest_fl = glob.glob(cleanest_glob)
if len(cleanest_fl) > 0:
cleanest = cleanest_fl[0]
if len(cleanest_fl) > 1:
log.warn(
"WARNING: found multiple 'cleanest' matches {0}"
.format(cleanest_fl))
log.warn(
f"Replaced 'cleanest' with {cleanest} to match {bsens}"
)
allow_reproj = True
else:
allow_reproj = False
if not os.path.exists(cleanest):
# hackaround for mismatched number of selfcal iterations
cfns = glob.glob(
f"{basepath}/{field}/B{band}/cleanest/*_{config}_robust0_*finaliter*.{suffix}"
)
if len(cfns) == 1:
log.info(
f"Replaced original cleanest {cleanest} with {cfns[0]}"
)
allow_reproj = False
cleanest = cfns[0]
elif len(cfns) == 0:
log.warn(
f"Did not find any cleanest matches to {fn}")
else:
log.warn(f"Found too many matches: {cfns}")
try:
clean_cube = SpectralCube.read(
cleanest,
format='fits'
if 'fits' in cleanest else 'casa_image')
except Exception as ex:
log.error(
f"Failed to open 'cleanest' image {cleanest} (check for a bsens-cleanest mismatch)"
)
print(ex)
#raise
continue
fig = pl.figure(1, figsize=(14, 6))
if fig.get_figheight() != 6:
fig.set_figheight(6)
if fig.get_figwidth() != 14:
fig.set_figwidth(14)
try:
with warnings.catch_warnings():
warnings.filterwarnings('ignore')
ax1, ax2, ax3, fig, diffstats = make_comparison_image(
filename1=cleanest,
filename2=bsens,
title1='cleanest',
title2='bsens',
allow_reproj=allow_reproj,
writediff=True,
diff_suffix='.bsens-cleanest')
except IndexError:
raise
except Exception as ex:
log.error(
f"Failure for bsens={bsens} cleanest={cleanest}")
log.error((field, band, config, ex))
raise
continue
if not os.path.exists(
f"{basepath}/{field}/B{band}/comparisons/"):
os.mkdir(f"{basepath}/{field}/B{band}/comparisons/")
pl.suptitle(f"{field} B{band}")
pl.savefig(
f"{basepath}/{field}/B{band}/comparisons/{field}_B{band}_{config}_bsens_vs_cleanest_comparison.png",
bbox_inches='tight',
dpi=200)
pl.savefig(
f"{basepath}/{field}/B{band}/comparisons/{field}_B{band}_{config}_bsens_vs_cleanest_comparison.pdf",
bbox_inches='tight',
dpi=200)
shutil.copy(
f"{basepath}/{field}/B{band}/comparisons/{field}_B{band}_{config}_bsens_vs_cleanest_comparison.png",
f"{sharepath}/bsens_comparison_images/")
shutil.copy(
f"{basepath}/{field}/B{band}/comparisons/{field}_B{band}_{config}_bsens_vs_cleanest_comparison.pdf",
f"{sharepath}/bsens_comparison_images/")
matchrow = (
(tbl['region'] == field) & (tbl['band'] == f'B{band}')
& (tbl['array']
== ('12Monly' if config == '12M' else config)) &
(tbl['pbcor'] if 'pbcor' in suffix else ~tbl['pbcor'])
& (tbl['robust'] == 'r0.0'))
tbl['contselMaxDiff'][matchrow] = diffstats['max']
tbl['contselMinDiff'][matchrow] = diffstats['min']
tbl['contselMADDiff'][matchrow] = diffstats['mad']
tbl['contselMeanDiff'][matchrow] = diffstats['mean']
tbl['contselMedianDiff'][matchrow] = diffstats['median']
tbl['contselSumDiff'][matchrow] = diffstats['sum']
tbl['bsens_fn'][matchrow] = os.path.basename(bsens)
tbl['cleanest_fn'][matchrow] = os.path.basename(cleanest)
tbl['dr_bsens'][matchrow] = diffstats['dr_post']
tbl['dr_cleanest'][matchrow] = diffstats['dr_pre']
tbl['min_bsens'][matchrow] = diffstats['min_post']
tbl['min_cleanest'][matchrow] = diffstats['min_pre']
tbl['max_bsens'][matchrow] = diffstats['max_post']
tbl['max_cleanest'][matchrow] = diffstats['max_pre']
tbl['sum_bsens'][matchrow] = diffstats['sum_post']
tbl['sum_cleanest'][matchrow] = diffstats['sum_pre']
tbl['masksum_bsens'][matchrow] = diffstats['masksum_post']
tbl['masksum_cleanest'][matchrow] = diffstats[
'masksum_pre']
tbl['shape'][matchrow] = diffstats['shape']
tbl['ppbeam'][matchrow] = diffstats['ppbeam']
tbl['mad_bsens'][matchrow] = diffstats['mad_post']
tbl['mad_cleanest'][matchrow] = diffstats['mad_pre']
tbl['mad_sample_bsens'][matchrow] = diffstats[
'mad_sample_post']
tbl['mad_sample_cleanest'][matchrow] = diffstats[
'mad_sample_pre']
tbl['std_sample_bsens'][matchrow] = diffstats[
'std_sample_post']
tbl['std_sample_cleanest'][matchrow] = diffstats[
'std_sample_pre']
tbl['dr_improvement_bsens'][
matchrow] = diffstats['dr_post'] / diffstats['dr_pre']
tbl['casaversion_bsens'][matchrow] = fits.getheader(
bsens)['ORIGIN']
tbl['casaversion_cleanest'][matchrow] = fits.getheader(
cleanest)['ORIGIN']
print(fns)
print(f"{field}_B{band}: matched {matchrow.sum()} rows")
print()
formats = {
'dr_improvement_bsens': lambda x: '{0:0.2f}'.format(x),
'contselMaxDiff': lambda x: f'{x:0.6g}',
'BeamVsReq': lambda x: f'{x:0.2f}',
}
for bp in (f'/orange/adamginsburg/web/secure/ALMA-IMF/{release}',
f'/orange/adamginsburg/ALMA_IMF/2017.1.01355.L/{release}'):
tbl.write(f'{bp}/tables/metadata_bsens_cleanest.ecsv', overwrite=True)
tbl.write(f'{bp}/tables/metadata_bsens_cleanest.html',
formats=formats,
format='ascii.html',
overwrite=True)
tbl.write(f'{bp}/tables/metadata_bsens_cleanest.tex',
formats=formats,
overwrite=True)
tbl.write(
f'{bp}/tables/metadata_bsens_cleanest.js.html',
#formats=formats,
format='jsviewer')
os.chdir(cwd)
# paper figure
config = '12M'
band = 6
field = 'G351.77'
cleanest = f'{basepath}/{field}/B{band}/cleanest/{field}_B{band}_uid___A001_X1296_X201_continuum_merged_12M_robust0_preselfcal_finalmodel.image.tt0'
bsens = f'{basepath}/{field}/B{band}/bsens/{field}_B{band}_uid___A001_X1296_X201_continuum_merged_bsens_12M_robust0_selfcal4_finaliter.image.tt0'
ax1, ax2, ax3, fig, diffstats = make_comparison_image(
filename1=cleanest,
filename2=bsens,
title1='cleanest',
title2='bsens',
writediff=True,
diff_suffix='.bsens-cleanest')
pl.savefig(
f"{basepath}/{field}/B{band}/comparisons/{field}_B{band}_{config}_bsens_vs_cleanest_comparison.png",
bbox_inches='tight',
dpi=200)
pl.savefig(
f"{basepath}/{field}/B{band}/comparisons/{field}_B{band}_{config}_bsens_vs_cleanest_comparison.pdf",
bbox_inches='tight',
dpi=200)
if len(glb) == 1:
autoname = glb[0]
else:
print(f"Skipping {basename} because there's no match.")
continue
field = basename.split("_uid")[0].split("/")[-1]
fh1 = fits.open(fn)
fh2 = fits.open(autoname)
if (fh1[0].data.shape == fh2[0].data.shape):
try:
make_comparison_image(fn,
autoname,
title1='Delivered',
title2='Auto',
allow_reproj=True)
except ValueError as ex:
if 'WCS' in str(ex):
print(str(ex))
continue
else:
raise ex
pl.savefig(
f"{basepath}/comparisons/auto_vs_not_{basename.replace('.fits','')}.png",
bbox_inches='tight',
dpi=200)
else:
print(f"Skipping {fn} because there was a shape mismatch.")
imresults = '/orange/adamginsburg/ALMA_IMF/2017.1.01355.L/imaging_results'
for fn in filelist:
pl.clf()
basename = os.path.split(fn)[-1]
if os.path.exists(f'{imresults}/{basename}'):
autoname = f'{imresults}/{basename}'
else:
pfx = basename.split("uid")[0]
sfx = basename.split("continuum_merged")[1]
glb = glob.glob(f"{imresults}/{pfx}*{sfx}")
if len(glb) == 1:
autoname = glb[0]
else:
print(f"Skipping {basename} because there's no match.")
continue
field = basename.split("_uid")[0].split("/")[-1]
fh1 = fits.open(fn)
fh2 = fits.open(autoname)
if (fh1[0].data.shape == fh2[0].data.shape):
make_comparison_image(fn, autoname, title1='Delivered', title2='Auto')
pl.savefig(
f"{basepath}/comparisons/auto_vs_not_{basename.replace('.fits','')}.png",
bbox_inches='tight',
dpi=200)
else:
print(f"Skipping {fn} because there was a shape mismatch.")
