def _convert_inputs(self):
"""Convert input into datamodel required for processing.
This method converts `self.inputs` into a version of
`self.input_models` suitable for processing by the class.
This base class works on imaging data, and relies on use of the
ModelContainer class as the format needed for processing. However,
the input may not always be a ModelContainer object, so this method
will convert the input to a ModelContainer object for processing.
Additionally, sub-classes may redefine this to set up the input as
whatever format the sub-class needs for processing.
"""
bits = self.outlierpars['good_bits']
if isinstance(self.inputs, datamodels.ModelContainer):
self.input_models = self.inputs
self.converted = False
else:
self.input_models = datamodels.ModelContainer()
num_inputs = self.inputs.data.shape[0]
log.debug("Converting CubeModel to ModelContainer with {} images".
format(num_inputs))
for i in range(self.inputs.data.shape[0]):
image = datamodels.ImageModel(data=self.inputs.data[i],
err=self.inputs.err[i],
dq=self.inputs.dq[i])
image.meta = self.inputs.meta
image.wht = build_driz_weight(image,
weight_type='ivm',
good_bits=bits)
self.input_models.append(image)
self.converted = True
def test_split_container(tmp_path):
path1 = tmp_path / "foo.fits"
path2 = tmp_path / "bar.fits"
im1 = datamodels.ImageModel()
im1.save(path1)
im2 = datamodels.ImageModel()
im2.save(path2)
container = datamodels.ModelContainer([im1, im2])
container.meta.asn_table = {
"products": [{
"members": [
{
"expname": f"{path1}",
"exptype": "science"
},
{
"expname": f"{path2}",
"exptype": "background"
},
]
}]
}
sci, bkg = split_container(container)
assert sci[0].meta.filename == "foo.fits"
assert bkg[0].meta.filename == "bar.fits"
assert len(sci) == 1
assert len(bkg) == 1
def blot_median(self, median_model):
"""Blot resampled median image back to the detector images."""
interp = self.outlierpars.get('interp', 'linear')
sinscl = self.outlierpars.get('sinscl', 1.0)
# Initialize container for output blot images
blot_models = datamodels.ModelContainer()
log.info("Blotting median...")
for model in self.input_models:
blotted_median = model.copy()
blot_root = '_'.join(model.meta.filename.replace(
'.fits', '').split('_')[:-1])
blotted_median.meta.filename = '{}_blot.fits'.format(blot_root)
# clean out extra data not related to blot result
blotted_median.err = None
blotted_median.dq = None
# apply blot to re-create model.data from median image
blotted_median.data = gwcs_blot(median_model, model, interp=interp,
sinscl=sinscl)
blot_models.append(blotted_median)
return blot_models
def test_outlier_step_image_weak_CR_nodither(exptype, tsovisit):
"""Test whole step with an outlier for TSO & coronagraphic modes"""
bkg = 1.5
sig = 0.02
container = datamodels.ModelContainer(
we_many_sci(background=bkg,
sigma=sig,
signal=7.0,
exptype=exptype,
tsovisit=tsovisit))
# Drop a weak CR on the science array
# no noise so it should always be above the default threshold of 5
container[0].data[12, 12] = bkg + sig * 10
result = OutlierDetectionStep.call(container)
# Make sure nothing changed in SCI array
for image, corrected in zip(container, result):
np.testing.assert_allclose(image.data, corrected.data)
# Verify source is not flagged
for r in result:
assert r.dq[7, 7] == datamodels.dqflags.pixel["GOOD"]
# Verify CR is flagged
assert result[0].dq[12, 12] == OUTLIER_DO_NOT_USE
def test_imatch_background_reset(_jail, miri_dither_ch12):
""" Test if background polynomail is already determined - reset it"""
all_models = datamodels.ModelContainer(miri_dither_ch12)
# added a background and test is reset background
# removes the background
new_container = []
degree = (1, 1, 1,)
center = (5, 5, 5,)
poly = np.ndarray(9)
poly[:] = 1.3
channel = '2'
for m in all_models:
m.meta.background.polynomial_info.append(
{
'degree': degree,
'refpoint': center,
'coefficients': poly.ravel().tolist(),
'channel': channel
}
)
new_container.append(m)
# test if reset backgound - removes background
step = MRSIMatchStep()
step._reset_background(new_container)
for i in range(len(new_container)):
m = new_container[i]
test = len(m.meta.background.polynomial_info)
assert test == 0
def test_imatch_background_subtracted(_jail, miri_dither_ch12):
""" Test if data is already background subtracted - raise error"""
all_models = datamodels.ModelContainer(miri_dither_ch12)
# modify the data set backgroud subtracted
new_container = []
for m in all_models:
m.meta.background.subtracted = True
new_container.append(m)
# test if background subtracted - raise error
with pytest.raises(ValueError):
step = MRSIMatchStep()
step.run(new_container)
def test_outlier_step_no_outliers(we_three_sci):
"""Test whole step, no outliers"""
container = datamodels.ModelContainer(list(we_three_sci))
pristine = container.copy()
result = OutlierDetectionStep.call(container)
# Make sure nothing changed in SCI and DQ arrays
for image, uncorrected in zip(pristine, container):
np.testing.assert_allclose(image.data, uncorrected.data)
np.testing.assert_allclose(image.dq, uncorrected.dq)
# Make sure nothing changed in SCI and DQ arrays
for image, corrected in zip(container, result):
np.testing.assert_allclose(image.data, corrected.data)
np.testing.assert_allclose(image.dq, corrected.dq)
def test_outlier_step(we_three_sci):
"""Test whole step with an outlier"""
container = datamodels.ModelContainer(list(we_three_sci))
# Drop a CR on the science array
container[0].data[12, 12] += 1e3
result = OutlierDetectionStep.call(container)
# Make sure nothing changed in SCI array
for image, corrected in zip(container, result):
np.testing.assert_allclose(image.data, corrected.data)
# Verify source is not flagged
for r in result:
assert r.dq[7, 7] == datamodels.dqflags.pixel["GOOD"]
# Verify CR is flagged
assert result[0].dq[12, 12] == OUTLIER_DO_NOT_USE
def test_outlier_step_square_source_no_outliers(we_three_sci):
"""Test whole step with square source with sharp edges, no outliers"""
container = datamodels.ModelContainer(list(we_three_sci))
# put a square source in all three exposures
for ccont in container:
ccont.data[5:15, 5:15] += 1e3
pristine = container.copy()
result = OutlierDetectionStep.call(container)
# Make sure nothing changed in SCI and DQ arrays
for image, uncorrected in zip(pristine, container):
np.testing.assert_allclose(image.data, uncorrected.data)
np.testing.assert_allclose(image.dq, uncorrected.dq)
# Make sure nothing changed in SCI and DQ arrays
for image, corrected in zip(container, result):
np.testing.assert_allclose(image.data, corrected.data)
np.testing.assert_allclose(image.dq, corrected.dq)
def _generate_data():
"""Generate data of each type of input for master background step"""
image = datamodels.ImageModel((10, 10))
image.meta.instrument.name = 'MIRI'
image.meta.instrument.detector = 'MIRIMAGE'
image.meta.exposure.type = 'MIR_LRS-FIXEDSLIT'
image.meta.observation.date = '2018-01-01'
image.meta.observation.time = '00:00:00'
image.meta.subarray.xstart = 1
image.meta.subarray.ystart = 1
image.meta.wcsinfo.v2_ref = 0
image.meta.wcsinfo.v3_ref = 0
image.meta.wcsinfo.roll_ref = 0
image.meta.wcsinfo.ra_ref = 0
image.meta.wcsinfo.dec_ref = 0
image = AssignWcsStep.call(image)
# hdulist = create_nirspec_fs_file(grating="G140M", filter="F100LP")
# hdulist[1].data = np.zeros((2048, 2048))
# fs = datamodels.ImageModel(hdulist)
# fs = AssignWcsStep.call(fs)
# multislit = Extract2dStep.call(fs)
# hdulist = create_nirspec_ifu_file("OPAQUE", "G140M")
# hdulist[1].data = np.zeros((2048, 2048))
# ifu_image = datamodels.IFUImageModel(hdulist)
# ifu_image = AssignWcsStep.call(ifu_image)
container = datamodels.ModelContainer([image])
cube = datamodels.CubeModel((2, 10, 10))
# Return the data and a status dependent on whether the step can process it
return [
(image, 'COMPLETE'),
# (multislit, 'COMPLETE'),
# (ifu_image, 'COMPLETE'),
(container, 'COMPLETE'),
(cube, 'SKIPPED'),
]
def test_imatch_degree(_jail, miri_dither_ch12):
""" Test if polynomial degree is not a 3 element tuple or integer then raise error """
all_models = datamodels.ModelContainer(miri_dither_ch12)
new_container = []
degree = (
1,
1,
1,
)
center = (
5,
5,
5,
)
poly = np.ndarray(9)
poly[:] = 1.3
channel = '2'
for m in all_models:
m.meta.background.polynomial_info.append({
'degree':
degree,
'refpoint':
center,
'coefficients':
poly.ravel().tolist(),
'channel':
channel
})
new_container.append(m)
# test catches degree set incorrectly - raise error
# check that degree must be a 3 element tuple
with pytest.raises(ValueError):
step = MRSIMatchStep()
step.bkg_degree = (
1,
1,
)
step.run(new_container)
def do_detection(self):
"""Flag outlier pixels in DQ of input images."""
self._convert_inputs()
self.build_suffix(**self.outlierpars)
pars = self.outlierpars
save_intermediate_results = pars['save_intermediate_results']
if pars['resample_data']:
# Start by creating resampled/mosaic images for
# each group of exposures
resamp = resample.ResampleData(self.input_models, single=True,
blendheaders=False, **pars)
drizzled_models = resamp.do_drizzle()
if save_intermediate_results:
for model in drizzled_models:
log.info("Writing out resampled exposures...")
model_output_path = self.make_output_path(
basepath=model.meta.filename,
suffix='outlier_i2d')
model.save(model_output_path)
else:
# for non-dithered data, the resampled image is just the original image
drizzled_models = self.input_models
for i in range(len(self.input_models)):
drizzled_models[i].wht = build_driz_weight(
self.input_models[i],
weight_type='ivm',
good_bits=pars['good_bits'])
# Initialize intermediate products used in the outlier detection
median_model = datamodels.ImageModel(drizzled_models[0].data.shape)
median_model.update(drizzled_models[0])
median_model.meta.wcs = drizzled_models[0].meta.wcs
# Perform median combination on set of drizzled mosaics
median_model.data = self.create_median(drizzled_models)
if save_intermediate_results:
median_output_path = self.make_output_path(
basepath=self.input_models[0].meta.filename,
suffix='median'
)
log.info("Writing out MEDIAN image to: {}".format(
median_output_path
))
median_model.save(median_output_path)
if pars['resample_data']:
# Blot the median image back to recreate each input image specified
# in the original input list/ASN/ModelContainer
blot_models = self.blot_median(median_model)
if save_intermediate_results:
log.info("Writing out BLOT images...")
for model in blot_models:
model_path = self.make_output_path(
basename=model.meta.filename,
suffix='blot'
)
model.save(model_path)
else:
# Median image will serve as blot image
blot_models = datamodels.ModelContainer()
for i in range(len(self.input_models)):
blot_models.append(median_model)
# Perform outlier detection using statistical comparisons between
# each original input image and its blotted version of the median image
self.detect_outliers(blot_models)
# clean-up (just to be explicit about being finished with
# these results)
del median_model, blot_models
def test_apply_background_2d(_jail, miri_dither_ch12):
""" Test if background polynomial is set it is subtracted correctly"""
all_models = datamodels.ModelContainer(miri_dither_ch12)
# test if given a background polynomial - apply_background_2d gives the correct answer
# Polynomial information was created by running a faked full array data set to determine
# what the polynomial values should be
new_container = []
degree = (
1,
1,
1,
)
center = (9.99999408089876, -4.668612949274985e-06, 4.889999912818894)
poly = np.ndarray(9)
# run AssignWcsStep on first file and set wcs of other files to this
channel = '1'
for im, m in enumerate(all_models):
if im == 0:
m = AssignWcsStep.call(m)
wcs1 = m.meta.wcs
poly = [
-2.50000000e+00, -6.66518331e-15, 5.67845589e-12,
-1.23549218e-11, 3.26209108e-12, -5.43180357e-12,
-2.54903452e-09, 6.21614553e-09
]
elif im == 1:
m.meta.wcs = wcs1
poly = [
-2.50000000e+00, -9.29031986e-15, 1.74437380e-12,
-3.94894956e-12, 4.71729481e-13, 5.42031845e-13,
-9.96151554e-10, 2.78281950e-09
]
elif im == 2:
m.meta.wcs = wcs1
poly = [
7.50000000e+00, 2.22921836e-14, -6.97131279e-12,
1.58336906e-11, -5.43704212e-12, 6.79109678e-12,
2.63515372e-09, -8.00226976e-09
]
else:
m.meta.wcs = wcs1
poly = [
-2.50000000e+00, -6.33668043e-15, -4.51516905e-13,
4.70180795e-13, 1.70322157e-12, -1.90132506e-12,
9.10032350e-10, -9.96695272e-10
]
poly = np.asarray(poly)
m.meta.background.polynomial_info.append({
'degree':
degree,
'refpoint':
center,
'coefficients':
poly.ravel().tolist(),
'channel':
channel
})
# input 10 output 12.5 for all images
apply_background_2d(m, channel, subtract=True)
new_container.append(m)
# all the data should now be the same for the science pixels
# the edge pixels are located in non-science region and no
# background subtraction is done on these pixels
data1 = new_container[0].data[:, 16:]
data2 = new_container[1].data[:, 16:]
data3 = new_container[2].data[:, 16:]
data4 = new_container[3].data[:, 16:]
assert (np.allclose(data1, data2, rtol=1e-6))
assert (np.allclose(data2, data3, rtol=1e-6))
assert (np.allclose(data3, data4, rtol=1e-6))
assert (np.allclose(data1, data4, rtol=1e-6))
def pipeline_mirisim(input_dir):
"""
"""
# set up logging
log_file = os.path.join(os.path.abspath(input_dir),'pipeline_MIRISim.log')
# check if the pipeline log file exists
if os.path.isfile(log_file): os.remove(log_file)
else: pass
testing_logger = logging.getLogger(__name__)
testing_logger.setLevel(logging.DEBUG)
handler = logging.FileHandler(log_file)
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
testing_logger.addHandler(handler)
# check if the simulation folder exists
if os.path.isdir(input_dir):
pass
else:
print("Simulation folder not found")
sys.exit(0)
# go through individual simulations and run through pipeline
simulations = glob.glob(os.path.join(input_dir,'IMA*','20*'))
simulations.extend(glob.glob(os.path.join(input_dir,'MRS*','20*')))
simulations.extend(glob.glob(os.path.join(input_dir,'LRS*','20*')))
#for simulation in simulations: print(simulation)
# get the full path of the cwd
cwd = os.path.abspath(os.getcwd())
# set the output figure directory
out_fig_dir = os.path.join(cwd,input_dir,'pipeline_plots')
try: shutil.rmtree(out_fig_dir)
except: pass
os.mkdir(out_fig_dir)
# cycle through the simulations
for simulation in simulations:
os.chdir(os.path.join(simulation,'det_images'))
level1b_files = glob.glob('*.fits')
# isolate the simulation name for logging
sim_name = simulation.split(os.sep)[1]
# check which MIRI mode
with datamodels.open(level1b_files[0]) as level1b_dm:
miri_mode = level1b_dm.meta.exposure.type
# IMAGER --------------------------------------------
if miri_mode == 'MIR_IMAGE':
# run level 1 and 2 imager pipelines
for f in level1b_files:
with datamodels.open(f) as level1b_dm:
try:
level2a_dm = Detector1Pipeline.call(level1b_dm, output_use_model=True, save_results=True,
steps={'ipc': {'skip': True}})
level2a_dm.meta.wcsinfo.wcsaxes = 2
Image2Pipeline.call(level2a_dm, save_results=True, output_use_model=True)
# log pass
testing_logger.info('%s levels 1 and 2 passed' % sim_name)
levels12_check = True
except Exception as e:
testing_logger.warning('%s failed' % sim_name)
testing_logger.warning(' %s: %s' % (e.__class__.__name__, str(e)))
levels12_check = False
# run level 3 pipeline
if len(level1b_files) > 1:
try:
level2B_files = glob.glob(os.path.join('*_cal.fits'))
call(["asn_from_list", "-o", "IMA_asn.json"] + level2B_files + ["--product-name", "dither"])
dm_3_container = datamodels.ModelContainer("IMA_asn.json")
Image3Pipeline.call(dm_3_container, save_results=True,
steps={'tweakreg': {'skip': True}})
# log pass
testing_logger.info('%s level 3 passed' % sim_name)
level3_check = True
except Exception as e:
testing_logger.warning('%s failed' % sim_name)
testing_logger.warning(' %s: %s' % (e.__class__.__name__, str(e)))
level3_check = False
if len(level1b_files) == 1 and levels12_check == True:
level2A_file = glob.glob(os.path.join('*_rate.fits'))[0]
level2B_file = glob.glob(os.path.join('*_cal.fits'))[0]
# set up output plots
fig, axs = plt.subplots(1, 3)
fig.set_figwidth(15.0)
fig.set_figheight(5.0)
axs = axs.ravel()
# plot level 1b, 2a, 2b
with datamodels.open(level1b_files[0]) as level1b_dm:
with datamodels.open(level2A_file) as level2a_dm:
with datamodels.open(level2B_file) as level2b_dm:
axs[0].imshow(level1b_dm.data[0][-1], cmap='jet', interpolation='nearest', norm=LogNorm(), origin='lower')
axs[0].annotate('level 1B', xy=(0.7, 0.95), xycoords='axes fraction', fontsize=10, fontweight='bold',
color='w')
axs[1].imshow(level2a_dm.data, cmap='jet', interpolation='nearest', norm=LogNorm(), origin='lower')
axs[1].annotate('level 2A', xy=(0.7, 0.95), xycoords='axes fraction', fontsize=10, fontweight='bold',
color='w')
axs[2].imshow(level2b_dm.data, cmap='jet', interpolation='nearest', norm=LogNorm(), origin='lower')
axs[2].annotate('level 2B', xy=(0.7, 0.95), xycoords='axes fraction', fontsize=10, fontweight='bold',
color='w')
# save the pipeline plot
out_fig_name = sim_name + '.pdf'
fig.savefig(os.path.join(out_fig_dir, out_fig_name), dpi=200)
del fig
elif len(level1b_files) > 1 and level3_check == True:
driz_dm = datamodels.open('dither_i2d.fits')
# set up output plots
fig, axs = plt.subplots(1, 1)
fig.set_figwidth(8.0)
fig.set_figheight(8.0)
# plot drizzled image
axs.imshow(driz_dm.data, cmap='jet', interpolation='nearest', origin='lower',
norm=LogNorm(vmin=1, vmax=1000))
axs.annotate('Drizzled image', xy=(0.0, 1.02), xycoords='axes fraction', fontsize=12, fontweight='bold',
color='k')
axs.set_facecolor('black')
# save the pipeline plot
out_fig_name = sim_name + '.pdf'
fig.savefig(os.path.join(out_fig_dir, out_fig_name), dpi=200)
del fig
# MRS --------------------------------------------
elif miri_mode == 'MIR_MRS':
# run level 1 and 2 pipelines
for f in level1b_files:
with datamodels.open(f) as level1b_dm:
try:
level2a_dm = Detector1Pipeline.call(level1b_dm, output_use_model=True, save_results=True,
steps={'ipc': {'skip': True}})
Spec2Pipeline.call(level2a_dm, save_results=True, steps={'straylight':{'skip':True},
'extract_1d':{'save_results':True}})
# log pass
testing_logger.info('%s levels 1 and 2 passed' % sim_name)
levels12_check = True
except Exception as e:
testing_logger.warning('%s failed' % sim_name)
testing_logger.warning(' %s: %s' % (e.__class__.__name__, str(e)))
levels12_check = False
# run level 3 pipeline
if len(level1b_files) > 1:
try:
level2B_files = glob.glob(os.path.join('*_cal.fits'))
call(["asn_from_list", "-o", "MRS_asn.json"] + level2B_files + ["--product-name",
"dither"])
dm_3_container = datamodels.ModelContainer("MRS_asn.json")
Spec3Pipeline.call(dm_3_container, save_results=True,
steps={'outlier_detection': {'skip': True},
'cube_build': {'save_results': True},
'extract_1d': {'save_results': True}})
# log pass
testing_logger.info('%s level 3 passed' % sim_name)
level3_check = True
except Exception as e:
testing_logger.warning('%s failed' % sim_name)
testing_logger.warning(' %s: %s' % (e.__class__.__name__, str(e)))
level3_check = False
if len(level1b_files) == 1 and levels12_check == True:
level2A_file = glob.glob('*_rate.fits')[0]
level2B_file = glob.glob('*_cal.fits')[0]
spec_file = glob.glob('*x1d.fits')[0]
# set up output plots
fig, axs = plt.subplots(2, 2)
fig.set_figwidth(15.0)
fig.set_figheight(15.0)
axs = axs.ravel()
with datamodels.open(level1b_files[0]) as level1b_dm:
with datamodels.open(level2A_file) as level2a_dm:
with datamodels.open(level2B_file) as level2b_dm:
with datamodels.open(spec_file) as spec_dm:
axs[0].imshow(level1b_dm.data[0][-1], cmap='jet', interpolation='nearest', norm=LogNorm(), origin='lower')
axs[0].annotate('level 1B', xy=(0.7, 0.95), xycoords='axes fraction', fontsize=10, fontweight='bold',
color='w')
axs[1].imshow(level2a_dm.data, cmap='jet', interpolation='nearest', norm=LogNorm(), origin='lower')
axs[1].annotate('level 2A', xy=(0.7, 0.95), xycoords='axes fraction', fontsize=10, fontweight='bold',
color='w')
axs[2].imshow(level2b_dm.data, cmap='jet', interpolation='nearest', norm=LogNorm(), origin='lower')
axs[2].annotate('level 2B', xy=(0.7, 0.95), xycoords='axes fraction', fontsize=10, fontweight='bold',
color='w')
# plot the spectrum
axs[3].plot(spec_dm.spec[0].spec_table['WAVELENGTH'], spec_dm.spec[0].spec_table['FLUX'], c='b',
marker='.', markersize=3, linestyle='-', linewidth=2)
axs[3].set_ylabel(r'Flux ($\mu$Jy)')
axs[3].set_xlabel(r'Wavelength ($\mu$m)')
axs[3].set_xlim(4.0, 28.0)
# axs[0].set_ylim(0,6000)
# save the pipeline plot
out_fig_name = sim_name + '.pdf'
fig.savefig(os.path.join(out_fig_dir, out_fig_name), dpi=200)
del fig
elif len(level1b_files) > 1 and level3_check == True:
# cube
cube_file = glob.glob("dither*s3d.fits")[0]
cube_dm = datamodels.open(cube_file)
# spec
spec_file = glob.glob( "dither*1d.fits")[0]
dm = datamodels.open(spec_file)
fig, axs = plt.subplots(1,2, figsize=(12, 8))
axs[0].imshow(np.sum(cube_dm.data, axis=0), cmap='jet', interpolation='nearest',
origin='lower', norm=LogNorm(vmin=100, vmax=5e5))
axs[0].annotate('Collapsed cube', xy=(0.0, 1.02), xycoords='axes fraction', fontsize=12,
fontweight='bold', color='k')
axs[0].set_facecolor('black')
# plot the spectrum
axs[1].plot(dm.spec[0].spec_table['WAVELENGTH'], dm.spec[0].spec_table['FLUX'], c='b', marker='.',
markersize=3, linestyle='-', linewidth=2)
axs[1].set_ylabel(r'Flux ($\mu$Jy)')
axs[1].set_xlabel(r'Wavelength ($\mu$m)')
axs[1].set_xlim(4.0, 28.0)
# axs[0].set_ylim(0,6000)
axs[1].annotate('Spectrum)', xy=(0.0, 1.02), xycoords='axes fraction', fontsize=12, fontweight='bold',
color='k')
# save the pipeline plot
out_fig_name = sim_name + '.pdf'
fig.savefig(os.path.join(out_fig_dir, out_fig_name), dpi=200)
del fig
# LRS-FIXEDSLIT --------------------------------------------
elif miri_mode == 'MIR_LRS-FIXEDSLIT':
# run level 1 and 2 pipelines
for f in level1b_files:
with datamodels.open(f) as level1b_dm:
try:
level2a_dm = Detector1Pipeline.call(level1b_dm, output_use_model=True, save_results=True,
steps={'ipc': {'skip': True}})
Spec2Pipeline.call(level2a_dm, save_results=True, steps={'extract_1d': {'save_results': True}})
# log pass
testing_logger.info('%s levels 1 and 2 passed' % sim_name)
levels12_check = True
except Exception as e:
testing_logger.warning('%s failed' % sim_name)
testing_logger.warning(' %s: %s' % (e.__class__.__name__, str(e)))
levels12_check = False
if len(level1b_files) == 1 and levels12_check == True:
level2A_file = glob.glob('*_rate.fits')[0]
level2B_file = glob.glob('*_cal.fits')[0]
spec_file = glob.glob('*x1d.fits')[0]
# set up output plots
fig, axs = plt.subplots(2, 2)
fig.set_figwidth(15.0)
fig.set_figheight(15.0)
axs = axs.ravel()
with datamodels.open(level1b_files[0]) as level1b_dm:
with datamodels.open(level2A_file) as level2a_dm:
with datamodels.open(level2B_file) as level2b_dm:
with datamodels.open(spec_file) as spec_dm:
axs[0].imshow(level1b_dm.data[0][-1], cmap='jet', interpolation='nearest', norm=LogNorm(),
origin='lower')
axs[0].annotate('level 1B', xy=(0.7, 0.95), xycoords='axes fraction', fontsize=10,
fontweight='bold', color='w')
axs[1].imshow(level2a_dm.data, cmap='jet', interpolation='nearest', norm=LogNorm(),
origin='lower')
axs[1].annotate('level 2A', xy=(0.7, 0.95), xycoords='axes fraction', fontsize=10,
fontweight='bold',
color='w')
axs[2].imshow(level2b_dm.data, cmap='jet', interpolation='nearest', norm=LogNorm(),
origin='lower')
axs[2].annotate('level 2B', xy=(0.7, 0.95), xycoords='axes fraction', fontsize=10,
fontweight='bold',color='w')
# plot the spectrum
axs[3].plot(spec_dm.spec[0].spec_table['WAVELENGTH'][1:-1], spec_dm.spec[0].spec_table['FLUX'][1:-1], c='b', marker='.',
markersize=3, linestyle='-', linewidth=2)
axs[3].set_ylabel(r'Flux ($\mu$Jy)')
axs[3].set_xlabel(r'Wavelength ($\mu$m)')
axs[3].set_xlim(3.0, 15.0)
# axs[0].set_ylim(0,6000)
axs[3].annotate('Spectrum)', xy=(0.0, 1.02), xycoords='axes fraction', fontsize=12, fontweight='bold',
color='k')
axs[3].set_facecolor('white')
# save the pipeline plot
out_fig_name = sim_name + '.pdf'
fig.savefig(os.path.join(out_fig_dir, out_fig_name), dpi=200)
del fig
# LRS-SLITLESS --------------------------------------------
elif miri_mode == 'MIR_LRS-SLITLESS':
level1b_dm = datamodels.open(level1b_files[0])
# correct for the 'SLITLESSPRISM', 'SUBPRISM' conflict
#if level1b_dm.meta.exposure.type == 'MIR_LRS-SLITLESS':
# level1b_dm.meta.subarray.name = 'SUBPRISM'
try:
# fix subarray name problem
level1b_dm.meta.subarray.name = 'SUBPRISM'
Detector1Pipeline.call(level1b_dm, output_use_model=True, save_results=True,
steps={'ipc': {'skip': True}, 'lastframe': {'skip': True}})
level2a_ints = glob.glob('*rateints.fits')[0]
Spec2Pipeline.call(level2a_ints, save_results=True, steps={'extract_1d': {'save_results': True}})
level2b_ints = glob.glob('*calints.fits')[0]
#Tso3Pipeline.call(level2b_ints)
# log pass
testing_logger.info('%s passed' % sim_name)
levels12_check = True
except Exception as e:
testing_logger.warning('%s failed' % sim_name)
testing_logger.warning(' %s: %s' % (e.__class__.__name__, str(e)))
levels12_check = False
try:
level2B_files = glob.glob('*_calints.fits')
call(["asn_from_list", "-o", "LRS-SLITLESS_asn.json"] + level2B_files + ["--product-name",
"exposures"])
Tso3Pipeline.call('LRS-SLITLESS_asn.json') # , steps={'white_light':{'skip':True}})
# log pass
testing_logger.info('%s level 3 passed' % sim_name)
level3_check = True
except Exception as e:
testing_logger.warning('%s failed' % sim_name)
testing_logger.warning(' %s: %s' % (e.__class__.__name__, str(e)))
level3_check = False
if len(level1b_files) == 1 and levels12_check == True:
spec_file = glob.glob('*x1dints.fits')[0]
# set up output plots
fig, axs = plt.subplots(3, 3, sharey=True, sharex=True)
fig.set_figwidth(15.0)
fig.set_figheight(15.0)
axs = axs.ravel()
with datamodels.open(spec_file) as spec_dm:
for n in range(9):
# plot the spectrum
axs[n].plot(spec_dm.spec[n].spec_table['WAVELENGTH'][1:-1], spec_dm.spec[n].spec_table['FLUX'][1:-1],
c='b', marker='.', markersize=0, linestyle='-', linewidth=2)
if n in [0, 3, 6]: axs[n].set_ylabel(r'Flux ($\mu$Jy)')
if n in [6, 7, 8]: axs[n].set_xlabel(r'Wavelength ($\mu$m)')
# save the pipeline plot
out_fig_name = sim_name + '.pdf'
plt.tight_layout(pad=0.0)
fig.savefig(os.path.join(out_fig_dir, out_fig_name), dpi=200)
del fig
if level3_check == True:
my_lightcurve = glob.glob('*.ecsv')[0]
lightcurve_data = Table.read(my_lightcurve, format='ascii.ecsv')
fig, axs = plt.subplots(1, 1, figsize=(10, 8))
# plot input and output ramps of the first integration
axs.plot(lightcurve_data[0][:], lightcurve_data[1][:], c='b', marker='o', markersize=3,
linestyle='-', linewidth=2, label='white light curve')
axs.set_title('White light curve', fontsize=15)
axs.set_ylabel('white light flux', fontsize=15)
axs.set_xlabel('MJD', fontsize=15)
plt.tight_layout(h_pad=0)
# save the pipeline plot
out_fig_name = sim_name + '_whitelight.pdf'
plt.tight_layout(pad=0.0)
fig.savefig(os.path.join(out_fig_dir, out_fig_name), dpi=200)
del fig
os.chdir(cwd)
