def test_nrs_msa_spec2(self):
"""
Regression test of calwebb_spec2 pipeline performed on NIRSpec MSA data.
"""
input = 'f170lp-g235m_mos_observation-6-c0e0_001_dn_nrs1_mod.fits'
input_file = self.get_data(self.test_dir, input)
self.get_data(self.test_dir, 'jw95065006001_0_short_msa.fits')
# define step for use in test
step = Spec2Pipeline()
step.save_results = True
step.save_bsub = False
step.output_use_model = True
step.resample_spec.save_results = True
step.extract_1d.save_results = True
step.extract_1d.smoothing_length = 0
step.extract_1d.bkg_order = 0
step.run(input_file)
outputs = [
('f170lp-g235m_mos_observation-6-c0e0_001_dn_nrs1_mod_cal.fits',
'f170lp-g235m_mos_observation-6-c0e0_001_dn_nrs1_mod_cal_ref.fits'
),
('f170lp-g235m_mos_observation-6-c0e0_001_dn_nrs1_mod_s2d.fits',
'f170lp-g235m_mos_observation-6-c0e0_001_dn_nrs1_mod_s2d_ref.fits'
),
('f170lp-g235m_mos_observation-6-c0e0_001_dn_nrs1_mod_x1d.fits',
'f170lp-g235m_mos_observation-6-c0e0_001_dn_nrs1_mod_x1d_ref.fits'
)
]
self.compare_outputs(outputs)
def test_mrs_spec2(self):
"""
Regression test of calwebb_spec2 pipeline performed on MIRI MRS data.
"""
self.rtol = 0.000001
input_file = self.get_data(self.test_dir,
'jw10001001001_01101_00001_mirifushort_rate.fits')
step = Spec2Pipeline()
step.save_bsub=True,
step.save_results=True
step.resample_spec.save_results = True
step.cube_build.save_results = True
step.extract_1d.save_results = True
step.run(input_file)
outputs = [('jw10001001001_01101_00001_mirifushort_cal.fits',
'jw10001001001_01101_00001_mirifushort_cal_ref.fits',
['primary','sci','err','dq','relsens2d']),
('jw10001001001_01101_00001_mirifushort_s3d.fits',
'jw10001001001_01101_00001_mirifushort_s3d_ref.fits',
['primary','sci','err','dq','wmap']),
('jw10001001001_01101_00001_mirifushort_x1d.fits',
'jw10001001001_01101_00001_mirifushort_x1d_ref.fits',
['primary','extract1d'])
]
self.compare_outputs(outputs)
def test_miri_lrs_slit_1(self):
"""
Regression test of calwebb_spec2 pipeline performed on a single
MIRI LRS fixed-slit exposure.
"""
input_file = self.get_data(self.test_dir,
'jw00035001001_01101_00001_MIRIMAGE_rate.fits')
step = Spec2Pipeline()
step.save_bsub=True,
step.save_results=True
step.resample_spec.save_results = True
step.cube_build.save_results = True
step.extract_1d.save_results = True
step.run(input_file)
outputs = [('jw00035001001_01101_00001_MIRIMAGE_cal.fits',
'jw00035001001_01101_00001_MIRIMAGE_cal_ref.fits',
['primary','sci','err','dq','relsens']),
('jw00035001001_01101_00001_MIRIMAGE_x1d.fits',
'jw00035001001_01101_00001_MIRIMAGE_x1d_ref.fits',
['primary','extract1d'])
]
self.compare_outputs(outputs)
def test_mrs2pipeline1(self):
"""
Regression test of calwebb_spec2 pipeline performed on MIRI MRS data.
"""
test_dir = 'test_mrs2pipeline'
self.ref_loc = ['test_mrs2pipeline', 'truth']
input_file = self.get_data(test_dir,
'jw80500018001_02101_00002_MIRIFUSHORT_rate.fits')
step = Spec2Pipeline()
step.save_bsub=True,
step.save_results=True
step.resample_spec.save_results = True
step.cube_build.save_results = True
step.extract_1d.save_results = True
step.run(input_file)
outputs = [('jw80500018001_02101_00002_MIRIFUSHORT_cal.fits',
'jw80500018001_02101_00002_MIRIFUSHORT_cal_ref.fits',
['primary','sci','err','dq']),
('jw80500018001_02101_00002_MIRIFUSHORT_s3d.fits',
'jw80500018001_02101_00002_MIRIFUSHORT_s3d_ref.fits',
['primary','sci','err','dq','wmap']),
('jw80500018001_02101_00002_MIRIFUSHORT_x1d.fits',
'jw80500018001_02101_00002_MIRIFUSHORT_x1d_ref.fits',
['primary','extract1d'])
]
self.compare_outputs(outputs)
def test_nrs2_nodata_api(envopt, _jail):
"""
Regression test of handling NRS2 detector that has no data.\
"""
# Only need to ensure that assing_wcs is run.
# This still will fail and should cause the pipeline to halt.
step = Spec2Pipeline()
step.assign_wcs.skip = False
with pytest.raises(NoDataOnDetectorError):
step.run(
get_bigdata('jwst-pipeline', envopt, 'nirspec', 'test_assignwcs',
'jw84700006001_02101_00001_nrs2_rate.fits'))
def test_nrs_fs_multi_spec2(self, input, outputs):
"""
Regression test of calwebb_spec2 pipeline performed on NIRSpec data.
"""
input_file = self.get_data(self.test_dir, input)
step = Spec2Pipeline()
step.save_bsub = True
step.save_results = True
step.resample_spec.save_results = True
step.cube_build.save_results = True
step.extract_1d.save_results = True
step.run(input_file)
self.compare_outputs(outputs)
def test_nirspec_nrs2_nodata_api(_jail, rtdata, fitsdiff_default_kwargs):
"""
Test of calwebb_spec2 pipeline performed on NIRSpec IFU exposure
that has a filter/grating combination that produces no data on
the NRS2 detector. Pipeline should raise an exception.
"""
# Get the input file
rtdata.get_data('nirspec/ifu/jw84700006001_02101_00001_nrs2_rate.fits')
# Call the Spec2Pipeline
step = Spec2Pipeline()
step.assign_wcs.skip = False
with pytest.raises(NoDataOnDetectorError):
step.run(rtdata.input)
def test_nrs_fs_single_spec2(self):
"""
Regression test of calwebb_spec2 pipeline performed on NIRSpec fixed-slit data
that uses a single-slit subarray (S200B1).
"""
input_file = self.get_data(self.test_dir,
'jw84600002001_02101_00001_nrs2_rate.fits')
step = Spec2Pipeline()
step.save_bsub = True
step.save_results = True
step.resample_spec.save_results = True
step.cube_build.save_results = True
step.extract_1d.save_results = True
step.run(input_file)
outputs = [('jw84600002001_02101_00001_nrs2_cal.fits',
'jw84600002001_02101_00001_nrs2_cal_ref.fits'),
('jw84600002001_02101_00001_nrs2_s2d.fits',
'jw84600002001_02101_00001_nrs2_s2d_ref.fits'),
('jw84600002001_02101_00001_nrs2_x1d.fits',
'jw84600002001_02101_00001_nrs2_x1d_ref.fits')]
self.compare_outputs(outputs)
def test_miri_lrs_bkgnod(self):
"""
Regression test of calwebb_spec2 pipeline performed on an association
of nodded MIRI LRS fixed-slit exposures.
"""
asn_file = self.get_data(self.test_dir,
'lrs_bkgnod_asn.json')
for file in raw_from_asn(asn_file):
self.get_data(self.test_dir, file)
step = Spec2Pipeline()
step.save_bsub=True,
step.save_results=True
step.resample_spec.save_results = True
step.cube_build.save_results = True
step.extract_1d.save_results = True
step.run(asn_file)
outputs = [('test_lrs1_bsub.fits', 'test_lrs1_bsub_ref.fits',
['primary','sci','err','dq']),
('test_lrs2_bsub.fits','test_lrs2_bsub_ref.fits',
['primary','sci','err','dq']),
('test_lrs3_bsub.fits','test_lrs3_bsub_ref.fits',
['primary','sci','err','dq']),
('test_lrs4_bsub.fits','test_lrs4_bsub_ref.fits',
['primary','sci','err','dq']),
('test_lrs1_cal.fits', 'test_lrs1_cal_ref.fits',
['primary','sci','err','dq','relsens']),
('test_lrs2_cal.fits', 'test_lrs2_cal_ref.fits',
['primary','sci','err','dq','relsens']),
('test_lrs3_cal.fits', 'test_lrs3_cal_ref.fits',
['primary','sci','err','dq','relsens']),
('test_lrs4_cal.fits', 'test_lrs4_cal_ref.fits',
['primary','sci','err','dq','relsens'])
]
self.compare_outputs(outputs)
def test_miri_lrs_slit_1b(self):
"""
Regression test of calwebb_spec2 pipeline performed on a single
MIRI LRS fixed-slit exposure with multiple integrations. Compare _calints.
"""
input_file = self.get_data(self.test_dir,
'jw00035001001_01101_00001_MIRIMAGE_rateints.fits')
step = Spec2Pipeline()
step.save_bsub=True,
step.save_results=True
step.extract_1d.save_results = True
step.run(input_file)
outputs = [('jw00035001001_01101_00001_MIRIMAGE_calints.fits',
'jw00035001001_01101_00001_MIRIMAGE_calints_ref.fits',
['primary','sci','err','dq','relsens']),
('jw00035001001_01101_00001_MIRIMAGE_x1dints.fits',
'jw00035001001_01101_00001_MIRIMAGE_x1dints_ref.fits',
['primary', ('extract1d', 1), ('extract1d', 2), ('extract1d', 3), ('extract1d', 4)]
)
]
self.compare_outputs(outputs)
def test_nrs_msa_spec2b(self):
"""
Regression test of calwebb_spec2 pipeline performed on NIRSpec MSA data,
including barshadow correction.
"""
input = 'jw95065_nrs_msaspec_barshadow.fits'
input_file = self.get_data(self.test_dir, input)
self.get_data(self.test_dir, 'jwst_nirspec_shutters_barshadow.fits')
step = Spec2Pipeline()
step.output_file = 'jw95065_nrs_msaspec_barshadow_cal.fits'
step.save_bsub = False
step.save_results = True
step.resample_spec.save_results = True
step.extract_1d.save_results = True
step.run(input_file)
outputs = [('jw95065_nrs_msaspec_barshadow_cal.fits',
'jw95065_nrs_msaspec_barshadow_cal_ref.fits'),
('jw95065_nrs_msaspec_barshadow_s2d.fits',
'jw95065_nrs_msaspec_barshadow_s2d_ref.fits'),
('jw95065_nrs_msaspec_barshadow_x1d.fits',
'jw95065_nrs_msaspec_barshadow_x1d_ref.fits')]
self.compare_outputs(outputs)
def calibrate(self, configdir=None, outdir=None, **kwargs):
"""
Pipeline process the file
Parameters
----------
configdir: str
The directory containing the configuration files
outdir: str
The directory to put the calibrated files into
"""
# Get config directory
if configdir is None:
configdir = resource_filename('specialsoss', 'files')
# Get output directory
if outdir is None:
outdir = os.path.dirname(self.file)
# Get basename
basename = os.path.basename(self.file)
file = os.path.join(outdir, basename)
# Dict for new files
new_files = {}
if self.ext == 'uncal':
# Create Detector1Pipeline instance
cfg1_file = os.path.join(configdir, 'calwebb_tso1.cfg')
from jwst.pipeline import Detector1Pipeline
tso1 = Detector1Pipeline.call(self.file,
save_results=True,
config_file=cfg1_file,
output_dir=outdir)
# Calibrated files
new_files['ramp'] = os.path.join(
outdir, file.replace('_uncal.fits', '_ramp.fits'))
new_files['rate'] = os.path.join(
outdir, file.replace('_uncal.fits', '_rate.fits'))
new_files['rateints'] = os.path.join(
outdir, file.replace('_uncal.fits', '_rateints.fits'))
if self.ext in ['rate', 'rateints']:
# SPEC2 Pipeline
cfg2_file = os.path.join(configdir, 'calwebb_tso-spec2.cfg')
from jwst.pipeline import Spec2Pipeline
tso2 = Spec2Pipeline(save_results=True,
config_file=cfg2_file,
output_dir=outdir)
# Configure steps
tso2.cube_build.skip = True
tso2.extract_2d.skip = True
tso2.bkg_subtract.skip = True
tso2.msa_flagging.skip = True
tso2.barshadow.skip = True
tso2.extract_1d.save_results = True
# Run the pipeline
tso2.run(self.file)
# Calibrated files
new_files['calints'] = os.path.join(
outdir, file.replace('_rateints.fits', '_calints.fits'))
new_files['x1dints'] = os.path.join(
outdir, file.replace('_rateints.fits', '_x1dints.fits'))
else:
print("Not sure how to calibrate a '{}' file right now.".format(
self.ext))
return new_files
def pipeline_mirisim(input_dir):
"""
"""
# set up logging
log_file = os.path.join(os.path.abspath(input_dir),'pipeline_MIRISim.log')
# check if the pipeline log file exists
if os.path.isfile(log_file): os.remove(log_file)
else: pass
testing_logger = logging.getLogger(__name__)
testing_logger.setLevel(logging.DEBUG)
handler = logging.FileHandler(log_file)
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
testing_logger.addHandler(handler)
# check if the simulation folder exists
if os.path.isdir(input_dir):
pass
else:
print("Simulation folder not found")
sys.exit(0)
# go through individual simulations and run through pipeline
simulations = glob.glob(os.path.join(input_dir,'IMA*','20*'))
simulations.extend(glob.glob(os.path.join(input_dir,'MRS*','20*')))
simulations.extend(glob.glob(os.path.join(input_dir,'LRS*','20*')))
#for simulation in simulations: print(simulation)
# get the full path of the cwd
cwd = os.path.abspath(os.getcwd())
# set the output figure directory
out_fig_dir = os.path.join(cwd,input_dir,'pipeline_plots')
try: shutil.rmtree(out_fig_dir)
except: pass
os.mkdir(out_fig_dir)
# cycle through the simulations
for simulation in simulations:
os.chdir(os.path.join(simulation,'det_images'))
level1b_files = glob.glob('*.fits')
# isolate the simulation name for logging
sim_name = simulation.split(os.sep)[1]
# check which MIRI mode
with datamodels.open(level1b_files[0]) as level1b_dm:
miri_mode = level1b_dm.meta.exposure.type
# IMAGER --------------------------------------------
if miri_mode == 'MIR_IMAGE':
# run level 1 and 2 imager pipelines
for f in level1b_files:
with datamodels.open(f) as level1b_dm:
try:
level2a_dm = Detector1Pipeline.call(level1b_dm, output_use_model=True, save_results=True,
steps={'ipc': {'skip': True}})
level2a_dm.meta.wcsinfo.wcsaxes = 2
Image2Pipeline.call(level2a_dm, save_results=True, output_use_model=True)
# log pass
testing_logger.info('%s levels 1 and 2 passed' % sim_name)
levels12_check = True
except Exception as e:
testing_logger.warning('%s failed' % sim_name)
testing_logger.warning(' %s: %s' % (e.__class__.__name__, str(e)))
levels12_check = False
# run level 3 pipeline
if len(level1b_files) > 1:
try:
level2B_files = glob.glob(os.path.join('*_cal.fits'))
call(["asn_from_list", "-o", "IMA_asn.json"] + level2B_files + ["--product-name", "dither"])
dm_3_container = datamodels.ModelContainer("IMA_asn.json")
Image3Pipeline.call(dm_3_container, save_results=True,
steps={'tweakreg': {'skip': True}})
# log pass
testing_logger.info('%s level 3 passed' % sim_name)
level3_check = True
except Exception as e:
testing_logger.warning('%s failed' % sim_name)
testing_logger.warning(' %s: %s' % (e.__class__.__name__, str(e)))
level3_check = False
if len(level1b_files) == 1 and levels12_check == True:
level2A_file = glob.glob(os.path.join('*_rate.fits'))[0]
level2B_file = glob.glob(os.path.join('*_cal.fits'))[0]
# set up output plots
fig, axs = plt.subplots(1, 3)
fig.set_figwidth(15.0)
fig.set_figheight(5.0)
axs = axs.ravel()
# plot level 1b, 2a, 2b
with datamodels.open(level1b_files[0]) as level1b_dm:
with datamodels.open(level2A_file) as level2a_dm:
with datamodels.open(level2B_file) as level2b_dm:
axs[0].imshow(level1b_dm.data[0][-1], cmap='jet', interpolation='nearest', norm=LogNorm(), origin='lower')
axs[0].annotate('level 1B', xy=(0.7, 0.95), xycoords='axes fraction', fontsize=10, fontweight='bold',
color='w')
axs[1].imshow(level2a_dm.data, cmap='jet', interpolation='nearest', norm=LogNorm(), origin='lower')
axs[1].annotate('level 2A', xy=(0.7, 0.95), xycoords='axes fraction', fontsize=10, fontweight='bold',
color='w')
axs[2].imshow(level2b_dm.data, cmap='jet', interpolation='nearest', norm=LogNorm(), origin='lower')
axs[2].annotate('level 2B', xy=(0.7, 0.95), xycoords='axes fraction', fontsize=10, fontweight='bold',
color='w')
# save the pipeline plot
out_fig_name = sim_name + '.pdf'
fig.savefig(os.path.join(out_fig_dir, out_fig_name), dpi=200)
del fig
elif len(level1b_files) > 1 and level3_check == True:
driz_dm = datamodels.open('dither_i2d.fits')
# set up output plots
fig, axs = plt.subplots(1, 1)
fig.set_figwidth(8.0)
fig.set_figheight(8.0)
# plot drizzled image
axs.imshow(driz_dm.data, cmap='jet', interpolation='nearest', origin='lower',
norm=LogNorm(vmin=1, vmax=1000))
axs.annotate('Drizzled image', xy=(0.0, 1.02), xycoords='axes fraction', fontsize=12, fontweight='bold',
color='k')
axs.set_facecolor('black')
# save the pipeline plot
out_fig_name = sim_name + '.pdf'
fig.savefig(os.path.join(out_fig_dir, out_fig_name), dpi=200)
del fig
# MRS --------------------------------------------
elif miri_mode == 'MIR_MRS':
# run level 1 and 2 pipelines
for f in level1b_files:
with datamodels.open(f) as level1b_dm:
try:
level2a_dm = Detector1Pipeline.call(level1b_dm, output_use_model=True, save_results=True,
steps={'ipc': {'skip': True}})
Spec2Pipeline.call(level2a_dm, save_results=True, steps={'straylight':{'skip':True},
'extract_1d':{'save_results':True}})
# log pass
testing_logger.info('%s levels 1 and 2 passed' % sim_name)
levels12_check = True
except Exception as e:
testing_logger.warning('%s failed' % sim_name)
testing_logger.warning(' %s: %s' % (e.__class__.__name__, str(e)))
levels12_check = False
# run level 3 pipeline
if len(level1b_files) > 1:
try:
level2B_files = glob.glob(os.path.join('*_cal.fits'))
call(["asn_from_list", "-o", "MRS_asn.json"] + level2B_files + ["--product-name",
"dither"])
dm_3_container = datamodels.ModelContainer("MRS_asn.json")
Spec3Pipeline.call(dm_3_container, save_results=True,
steps={'outlier_detection': {'skip': True},
'cube_build': {'save_results': True},
'extract_1d': {'save_results': True}})
# log pass
testing_logger.info('%s level 3 passed' % sim_name)
level3_check = True
except Exception as e:
testing_logger.warning('%s failed' % sim_name)
testing_logger.warning(' %s: %s' % (e.__class__.__name__, str(e)))
level3_check = False
if len(level1b_files) == 1 and levels12_check == True:
level2A_file = glob.glob('*_rate.fits')[0]
level2B_file = glob.glob('*_cal.fits')[0]
spec_file = glob.glob('*x1d.fits')[0]
# set up output plots
fig, axs = plt.subplots(2, 2)
fig.set_figwidth(15.0)
fig.set_figheight(15.0)
axs = axs.ravel()
with datamodels.open(level1b_files[0]) as level1b_dm:
with datamodels.open(level2A_file) as level2a_dm:
with datamodels.open(level2B_file) as level2b_dm:
with datamodels.open(spec_file) as spec_dm:
axs[0].imshow(level1b_dm.data[0][-1], cmap='jet', interpolation='nearest', norm=LogNorm(), origin='lower')
axs[0].annotate('level 1B', xy=(0.7, 0.95), xycoords='axes fraction', fontsize=10, fontweight='bold',
color='w')
axs[1].imshow(level2a_dm.data, cmap='jet', interpolation='nearest', norm=LogNorm(), origin='lower')
axs[1].annotate('level 2A', xy=(0.7, 0.95), xycoords='axes fraction', fontsize=10, fontweight='bold',
color='w')
axs[2].imshow(level2b_dm.data, cmap='jet', interpolation='nearest', norm=LogNorm(), origin='lower')
axs[2].annotate('level 2B', xy=(0.7, 0.95), xycoords='axes fraction', fontsize=10, fontweight='bold',
color='w')
# plot the spectrum
axs[3].plot(spec_dm.spec[0].spec_table['WAVELENGTH'], spec_dm.spec[0].spec_table['FLUX'], c='b',
marker='.', markersize=3, linestyle='-', linewidth=2)
axs[3].set_ylabel(r'Flux ($\mu$Jy)')
axs[3].set_xlabel(r'Wavelength ($\mu$m)')
axs[3].set_xlim(4.0, 28.0)
# axs[0].set_ylim(0,6000)
# save the pipeline plot
out_fig_name = sim_name + '.pdf'
fig.savefig(os.path.join(out_fig_dir, out_fig_name), dpi=200)
del fig
elif len(level1b_files) > 1 and level3_check == True:
# cube
cube_file = glob.glob("dither*s3d.fits")[0]
cube_dm = datamodels.open(cube_file)
# spec
spec_file = glob.glob( "dither*1d.fits")[0]
dm = datamodels.open(spec_file)
fig, axs = plt.subplots(1,2, figsize=(12, 8))
axs[0].imshow(np.sum(cube_dm.data, axis=0), cmap='jet', interpolation='nearest',
origin='lower', norm=LogNorm(vmin=100, vmax=5e5))
axs[0].annotate('Collapsed cube', xy=(0.0, 1.02), xycoords='axes fraction', fontsize=12,
fontweight='bold', color='k')
axs[0].set_facecolor('black')
# plot the spectrum
axs[1].plot(dm.spec[0].spec_table['WAVELENGTH'], dm.spec[0].spec_table['FLUX'], c='b', marker='.',
markersize=3, linestyle='-', linewidth=2)
axs[1].set_ylabel(r'Flux ($\mu$Jy)')
axs[1].set_xlabel(r'Wavelength ($\mu$m)')
axs[1].set_xlim(4.0, 28.0)
# axs[0].set_ylim(0,6000)
axs[1].annotate('Spectrum)', xy=(0.0, 1.02), xycoords='axes fraction', fontsize=12, fontweight='bold',
color='k')
# save the pipeline plot
out_fig_name = sim_name + '.pdf'
fig.savefig(os.path.join(out_fig_dir, out_fig_name), dpi=200)
del fig
# LRS-FIXEDSLIT --------------------------------------------
elif miri_mode == 'MIR_LRS-FIXEDSLIT':
# run level 1 and 2 pipelines
for f in level1b_files:
with datamodels.open(f) as level1b_dm:
try:
level2a_dm = Detector1Pipeline.call(level1b_dm, output_use_model=True, save_results=True,
steps={'ipc': {'skip': True}})
Spec2Pipeline.call(level2a_dm, save_results=True, steps={'extract_1d': {'save_results': True}})
# log pass
testing_logger.info('%s levels 1 and 2 passed' % sim_name)
levels12_check = True
except Exception as e:
testing_logger.warning('%s failed' % sim_name)
testing_logger.warning(' %s: %s' % (e.__class__.__name__, str(e)))
levels12_check = False
if len(level1b_files) == 1 and levels12_check == True:
level2A_file = glob.glob('*_rate.fits')[0]
level2B_file = glob.glob('*_cal.fits')[0]
spec_file = glob.glob('*x1d.fits')[0]
# set up output plots
fig, axs = plt.subplots(2, 2)
fig.set_figwidth(15.0)
fig.set_figheight(15.0)
axs = axs.ravel()
with datamodels.open(level1b_files[0]) as level1b_dm:
with datamodels.open(level2A_file) as level2a_dm:
with datamodels.open(level2B_file) as level2b_dm:
with datamodels.open(spec_file) as spec_dm:
axs[0].imshow(level1b_dm.data[0][-1], cmap='jet', interpolation='nearest', norm=LogNorm(),
origin='lower')
axs[0].annotate('level 1B', xy=(0.7, 0.95), xycoords='axes fraction', fontsize=10,
fontweight='bold', color='w')
axs[1].imshow(level2a_dm.data, cmap='jet', interpolation='nearest', norm=LogNorm(),
origin='lower')
axs[1].annotate('level 2A', xy=(0.7, 0.95), xycoords='axes fraction', fontsize=10,
fontweight='bold',
color='w')
axs[2].imshow(level2b_dm.data, cmap='jet', interpolation='nearest', norm=LogNorm(),
origin='lower')
axs[2].annotate('level 2B', xy=(0.7, 0.95), xycoords='axes fraction', fontsize=10,
fontweight='bold',color='w')
# plot the spectrum
axs[3].plot(spec_dm.spec[0].spec_table['WAVELENGTH'][1:-1], spec_dm.spec[0].spec_table['FLUX'][1:-1], c='b', marker='.',
markersize=3, linestyle='-', linewidth=2)
axs[3].set_ylabel(r'Flux ($\mu$Jy)')
axs[3].set_xlabel(r'Wavelength ($\mu$m)')
axs[3].set_xlim(3.0, 15.0)
# axs[0].set_ylim(0,6000)
axs[3].annotate('Spectrum)', xy=(0.0, 1.02), xycoords='axes fraction', fontsize=12, fontweight='bold',
color='k')
axs[3].set_facecolor('white')
# save the pipeline plot
out_fig_name = sim_name + '.pdf'
fig.savefig(os.path.join(out_fig_dir, out_fig_name), dpi=200)
del fig
# LRS-SLITLESS --------------------------------------------
elif miri_mode == 'MIR_LRS-SLITLESS':
level1b_dm = datamodels.open(level1b_files[0])
# correct for the 'SLITLESSPRISM', 'SUBPRISM' conflict
#if level1b_dm.meta.exposure.type == 'MIR_LRS-SLITLESS':
# level1b_dm.meta.subarray.name = 'SUBPRISM'
try:
# fix subarray name problem
level1b_dm.meta.subarray.name = 'SUBPRISM'
Detector1Pipeline.call(level1b_dm, output_use_model=True, save_results=True,
steps={'ipc': {'skip': True}, 'lastframe': {'skip': True}})
level2a_ints = glob.glob('*rateints.fits')[0]
Spec2Pipeline.call(level2a_ints, save_results=True, steps={'extract_1d': {'save_results': True}})
level2b_ints = glob.glob('*calints.fits')[0]
#Tso3Pipeline.call(level2b_ints)
# log pass
testing_logger.info('%s passed' % sim_name)
levels12_check = True
except Exception as e:
testing_logger.warning('%s failed' % sim_name)
testing_logger.warning(' %s: %s' % (e.__class__.__name__, str(e)))
levels12_check = False
try:
level2B_files = glob.glob('*_calints.fits')
call(["asn_from_list", "-o", "LRS-SLITLESS_asn.json"] + level2B_files + ["--product-name",
"exposures"])
Tso3Pipeline.call('LRS-SLITLESS_asn.json') # , steps={'white_light':{'skip':True}})
# log pass
testing_logger.info('%s level 3 passed' % sim_name)
level3_check = True
except Exception as e:
testing_logger.warning('%s failed' % sim_name)
testing_logger.warning(' %s: %s' % (e.__class__.__name__, str(e)))
level3_check = False
if len(level1b_files) == 1 and levels12_check == True:
spec_file = glob.glob('*x1dints.fits')[0]
# set up output plots
fig, axs = plt.subplots(3, 3, sharey=True, sharex=True)
fig.set_figwidth(15.0)
fig.set_figheight(15.0)
axs = axs.ravel()
with datamodels.open(spec_file) as spec_dm:
for n in range(9):
# plot the spectrum
axs[n].plot(spec_dm.spec[n].spec_table['WAVELENGTH'][1:-1], spec_dm.spec[n].spec_table['FLUX'][1:-1],
c='b', marker='.', markersize=0, linestyle='-', linewidth=2)
if n in [0, 3, 6]: axs[n].set_ylabel(r'Flux ($\mu$Jy)')
if n in [6, 7, 8]: axs[n].set_xlabel(r'Wavelength ($\mu$m)')
# save the pipeline plot
out_fig_name = sim_name + '.pdf'
plt.tight_layout(pad=0.0)
fig.savefig(os.path.join(out_fig_dir, out_fig_name), dpi=200)
del fig
if level3_check == True:
my_lightcurve = glob.glob('*.ecsv')[0]
lightcurve_data = Table.read(my_lightcurve, format='ascii.ecsv')
fig, axs = plt.subplots(1, 1, figsize=(10, 8))
# plot input and output ramps of the first integration
axs.plot(lightcurve_data[0][:], lightcurve_data[1][:], c='b', marker='o', markersize=3,
linestyle='-', linewidth=2, label='white light curve')
axs.set_title('White light curve', fontsize=15)
axs.set_ylabel('white light flux', fontsize=15)
axs.set_xlabel('MJD', fontsize=15)
plt.tight_layout(h_pad=0)
# save the pipeline plot
out_fig_name = sim_name + '_whitelight.pdf'
plt.tight_layout(pad=0.0)
fig.savefig(os.path.join(out_fig_dir, out_fig_name), dpi=200)
del fig
os.chdir(cwd)
def test_extract_2D(cases, direct_image_file):
'''Test extract 2D.'''
save_figs = True
# load direct image
direct_image = fits.getdata(direct_image_file, 1)
# get file names for catalog and dispersed image from association
with open(cases) as json_data:
d = json.load(json_data)
members = d['products'][0]['members']
for row in np.arange(0, len(members)):
if members[row]['exptype'] == "sourcecat":
catalog_file = members[row]['expname']
elif members[row]['exptype'] == "science":
dispersed_file = members[row]['expname']
# get the catalog and dispersed image
catalog_in = ascii.read(catalog_file)
short_catalog = Table(
{
'RA': catalog_in['icrs_centroid'].ra,
'Dec': catalog_in['icrs_centroid'].dec,
'PixelX': catalog_in['xcentroid'],
'PixelY': catalog_in['ycentroid'],
'Mag': catalog_in['abmag']
},
names=['RA', 'Dec', 'PixelX', 'PixelY', 'Mag'])
dispersed_image = fits.getdata(dispersed_file)
# run the pipeline
spec2 = Spec2Pipeline.call(cases, config_file="calwebb_spec2.cfg")
extract_2D_output = glob(dispersed_file[:-10] + "*_cal.fits")[0]
# save the plots for checking by eye (for now)
if save_figs == True:
# plot of direct image with sources in catalog circled
fig, ax = plt.subplots(1, 1, figsize=(20, 20))
plt.ylabel('y pixels', fontsize=22)
plt.xlabel('x pixels', fontsize=22)
plt.imshow(direct_image,
vmin=-0.02,
vmax=0.08,
cmap=plt.cm.gray,
origin='lower')
ax.scatter(short_catalog['PixelX'][0],
short_catalog['PixelY'][0],
s=5000,
marker='o',
facecolors='none',
edgecolors='red',
lw=3,
label='Source 1')
ax.scatter(short_catalog['PixelX'][1],
short_catalog['PixelY'][1],
s=1000,
marker='o',
facecolors='none',
edgecolors='limegreen',
lw=3,
label='Source 2')
ax.scatter(short_catalog['PixelX'][2],
short_catalog['PixelY'][2],
s=1000,
marker='o',
facecolors='none',
edgecolors='lightblue',
lw=3,
label='Source 3')
ax.text(short_catalog['PixelX'][0] - 110,
short_catalog['PixelY'][0] + 130,
"Source 1",
fontsize=22,
color='red')
ax.text(short_catalog['PixelX'][1] - 110,
short_catalog['PixelY'][1] + 90,
"Source 2",
fontsize=22,
color='limegreen')
ax.text(short_catalog['PixelX'][2] - 110,
short_catalog['PixelY'][2] + 90,
"Source 3",
fontsize=22,
color='lightblue')
ax.set_title("Direct image and catalog sources", fontsize=22)
plt.colorbar(orientation='horizontal', pad=0.05)
plt.savefig(dispersed_file[:-5] + '_direct_img_sources.png')
# source data and headers
source1_order1 = fits.getdata(extract_2D_output, 1)
src1_hdr1 = fits.getheader(extract_2D_output, 1)
source1_order2 = fits.getdata(extract_2D_output, 4)
src1_hdr2 = fits.getheader(extract_2D_output, 4)
source2_order1 = fits.getdata(extract_2D_output, 7)
src2_hdr1 = fits.getheader(extract_2D_output, 7)
source2_order2 = fits.getdata(extract_2D_output, 10)
src2_hdr2 = fits.getheader(extract_2D_output, 10)
source3_order1 = fits.getdata(extract_2D_output, 13)
src3_hdr1 = fits.getheader(extract_2D_output, 13)
source3_order2 = fits.getdata(extract_2D_output, 16)
src3_hdr2 = fits.getheader(extract_2D_output, 16)
# plot showing dispersed image and extract 2D regions highlighted
fig, ax = plt.subplots(1, 1, figsize=(20, 20))
plt.ylabel('y pixels', fontsize=22)
plt.xlabel('x pixels', fontsize=22)
plt.imshow(dispersed_image,
vmin=0.2,
vmax=1.5,
cmap=plt.cm.gray,
origin='lower')
ax.scatter(src1_hdr1['SRCXPOS'],
src1_hdr1['SRCYPOS'],
s=1000,
marker='o',
facecolors='none',
edgecolors='red',
lw=3,
label='Source 1')
ax.scatter(src2_hdr1['SRCXPOS'],
src2_hdr1['SRCYPOS'],
s=1000,
marker='o',
facecolors='none',
edgecolors='limegreen',
lw=3,
label='Source 2')
ax.scatter(src3_hdr1['SRCXPOS'],
src3_hdr1['SRCYPOS'],
s=1000,
marker='o',
facecolors='none',
edgecolors='lightblue',
lw=3,
label='Source 3')
rect = patches.Rectangle(
(src1_hdr1['SLTSTRT1'], src1_hdr1['SLTSTRT2']),
src1_hdr1['SLTSIZE1'],
src1_hdr1['SLTSIZE2'],
linewidth=1,
edgecolor='r',
lw=3,
facecolor='none')
ax.add_patch(rect)
rect = patches.Rectangle(
(src1_hdr2['SLTSTRT1'], src1_hdr2['SLTSTRT2']),
src1_hdr2['SLTSIZE1'],
src1_hdr2['SLTSIZE2'],
linewidth=1,
edgecolor='r',
lw=3,
facecolor='none')
ax.add_patch(rect)
rect = patches.Rectangle(
(src2_hdr1['SLTSTRT1'], src2_hdr1['SLTSTRT2']),
src2_hdr1['SLTSIZE1'],
src2_hdr1['SLTSIZE2'],
linewidth=1,
edgecolor='limegreen',
lw=3,
facecolor='none')
ax.add_patch(rect)
rect = patches.Rectangle(
(src2_hdr2['SLTSTRT1'], src2_hdr2['SLTSTRT2']),
src2_hdr2['SLTSIZE1'],
src2_hdr2['SLTSIZE2'],
linewidth=1,
edgecolor='limegreen',
lw=3,
facecolor='none')
ax.add_patch(rect)
rect = patches.Rectangle(
(src3_hdr1['SLTSTRT1'], src3_hdr1['SLTSTRT2']),
src3_hdr1['SLTSIZE1'],
src3_hdr1['SLTSIZE2'],
linewidth=1,
edgecolor='lightblue',
lw=3,
facecolor='none')
ax.add_patch(rect)
rect = patches.Rectangle(
(src3_hdr2['SLTSTRT1'], src3_hdr2['SLTSTRT2']),
src3_hdr2['SLTSIZE1'],
src3_hdr2['SLTSIZE2'],
linewidth=1,
edgecolor='lightblue',
lw=3,
facecolor='none')
ax.add_patch(rect)
ax.text(src1_hdr1['SLTSTRT1'] + 10,
src1_hdr1['SLTSTRT2'] + 10,
"order 1",
fontsize=16,
color='red')
ax.text(src1_hdr2['SLTSTRT1'] + 10,
src1_hdr2['SLTSTRT2'] + 10,
"order 2",
fontsize=16,
color='red')
ax.text(src2_hdr1['SLTSTRT1'] + 10,
src2_hdr1['SLTSTRT2'] + 10,
"order 1",
fontsize=16,
color='limegreen')
ax.text(src2_hdr2['SLTSTRT1'] + 10,
src2_hdr2['SLTSTRT2'] + 10,
"order 2",
fontsize=16,
color='limegreen')
ax.text(src3_hdr1['SLTSTRT1'] + 10,
src3_hdr1['SLTSTRT2'] + 10,
"order 1",
fontsize=16,
color='lightblue')
ax.text(src3_hdr2['SLTSTRT1'] + 10,
src3_hdr2['SLTSTRT2'] + 10,
"order 2",
fontsize=16,
color='lightblue')
ax.set_title(
"Dispersed image showing source locations and the cutout bounding boxes",
fontsize=22)
leg = ax.legend(fontsize=15, facecolor='black', loc=2)
for text in leg.get_texts():
plt.setp(text, color='w')
plt.colorbar(orientation='horizontal', pad=0.05)
plt.savefig(dispersed_file[:-5] + '_dispersion_cutout_regions.png')
# extraction for source 1 order 1
fig, ax = plt.subplots(1, 1, figsize=(20, 20))
plt.ylabel('y pixels', fontsize=18)
plt.xlabel('x pixels', fontsize=18)
plt.imshow(source1_order1,
vmin=0.2,
vmax=1.7,
cmap=plt.cm.gray,
origin='lower')
ax.set_title("Source 1 (order 1) -- red left box above", fontsize=18)
# extraction for source 1 order 2
fig, ax = plt.subplots(1, 1, figsize=(20, 20))
plt.ylabel('y pixels', fontsize=18)
plt.xlabel('x pixels', fontsize=18)
plt.imshow(source1_order2,
vmin=0.2,
vmax=1.7,
cmap=plt.cm.gray,
origin='lower')
ax.set_title("Source 1 (order 2) -- red right box above", fontsize=18)
plt.savefig(dispersed_file[:-5] + '_src1order2.png')
# extraction for source 2 order 1
fig, ax = plt.subplots(1, 1, figsize=(20, 20))
plt.ylabel('y pixels', fontsize=18)
plt.xlabel('x pixels', fontsize=18)
plt.imshow(source2_order1,
vmin=0.2,
vmax=1.7,
cmap=plt.cm.gray,
origin='lower')
ax.set_title("Source 2 (order 1) -- green left box above", fontsize=18)
plt.savefig(dispersed_file[:-5] + '_src2order1.png')
# extraction for source 2 order 2
fig, ax = plt.subplots(1, 1, figsize=(20, 20))
plt.ylabel('y pixels', fontsize=18)
plt.xlabel('x pixels', fontsize=18)
plt.imshow(source2_order2,
vmin=0.2,
vmax=1.7,
cmap=plt.cm.gray,
origin='lower')
ax.set_title("Source 2 (order 2) -- green right box above",
fontsize=18)
plt.savefig(dispersed_file[:-5] + '_src2order2.png')
# extraction for source 3 order 1
fig, ax = plt.subplots(1, 1, figsize=(20, 20))
plt.ylabel('y pixels', fontsize=18)
plt.xlabel('x pixels', fontsize=18)
plt.imshow(source3_order1,
vmin=0.2,
vmax=1.7,
cmap=plt.cm.gray,
origin='lower')
ax.set_title("Source 3 (order 1) -- blue left box above", fontsize=18)
plt.savefig(dispersed_file[:-5] + '_src3order1.png')
# extraction for source 3 order 2
fig, ax = plt.subplots(1, 1, figsize=(20, 20))
plt.ylabel('y pixels', fontsize=18)
plt.xlabel('x pixels', fontsize=18)
plt.imshow(source3_order2,
vmin=0.2,
vmax=1.7,
cmap=plt.cm.gray,
origin='lower')
ax.set_title("Source 3 (order 2) -- blue right box above", fontsize=18)
plt.savefig(dispersed_file[:-5] + '_src3order2.png')
