def _convert_inputs(self): """Convert input into datamodel required for processing. This method converts `self.inputs` into a version of `self.input_models` suitable for processing by the class. This base class works on imaging data, and relies on use of the ModelContainer class as the format needed for processing. However, the input may not always be a ModelContainer object, so this method will convert the input to a ModelContainer object for processing. Additionally, sub-classes may redefine this to set up the input as whatever format the sub-class needs for processing. """ bits = self.outlierpars['good_bits'] if isinstance(self.inputs, datamodels.ModelContainer): self.input_models = self.inputs self.converted = False else: self.input_models = datamodels.ModelContainer() num_inputs = self.inputs.data.shape[0] log.debug("Converting CubeModel to ModelContainer with {} images". format(num_inputs)) for i in range(self.inputs.data.shape[0]): image = datamodels.ImageModel(data=self.inputs.data[i], err=self.inputs.err[i], dq=self.inputs.dq[i]) image.meta = self.inputs.meta image.wht = build_driz_weight(image, weight_type='ivm', good_bits=bits) self.input_models.append(image) self.converted = True
def test_split_container(tmp_path): path1 = tmp_path / "foo.fits" path2 = tmp_path / "bar.fits" im1 = datamodels.ImageModel() im1.save(path1) im2 = datamodels.ImageModel() im2.save(path2) container = datamodels.ModelContainer([im1, im2]) container.meta.asn_table = { "products": [{ "members": [ { "expname": f"{path1}", "exptype": "science" }, { "expname": f"{path2}", "exptype": "background" }, ] }] } sci, bkg = split_container(container) assert sci[0].meta.filename == "foo.fits" assert bkg[0].meta.filename == "bar.fits" assert len(sci) == 1 assert len(bkg) == 1
def blot_median(self, median_model): """Blot resampled median image back to the detector images.""" interp = self.outlierpars.get('interp', 'linear') sinscl = self.outlierpars.get('sinscl', 1.0) # Initialize container for output blot images blot_models = datamodels.ModelContainer() log.info("Blotting median...") for model in self.input_models: blotted_median = model.copy() blot_root = '_'.join(model.meta.filename.replace( '.fits', '').split('_')[:-1]) blotted_median.meta.filename = '{}_blot.fits'.format(blot_root) # clean out extra data not related to blot result blotted_median.err = None blotted_median.dq = None # apply blot to re-create model.data from median image blotted_median.data = gwcs_blot(median_model, model, interp=interp, sinscl=sinscl) blot_models.append(blotted_median) return blot_models
def test_outlier_step_image_weak_CR_nodither(exptype, tsovisit): """Test whole step with an outlier for TSO & coronagraphic modes""" bkg = 1.5 sig = 0.02 container = datamodels.ModelContainer( we_many_sci(background=bkg, sigma=sig, signal=7.0, exptype=exptype, tsovisit=tsovisit)) # Drop a weak CR on the science array # no noise so it should always be above the default threshold of 5 container[0].data[12, 12] = bkg + sig * 10 result = OutlierDetectionStep.call(container) # Make sure nothing changed in SCI array for image, corrected in zip(container, result): np.testing.assert_allclose(image.data, corrected.data) # Verify source is not flagged for r in result: assert r.dq[7, 7] == datamodels.dqflags.pixel["GOOD"] # Verify CR is flagged assert result[0].dq[12, 12] == OUTLIER_DO_NOT_USE
def test_imatch_background_reset(_jail, miri_dither_ch12): """ Test if background polynomail is already determined - reset it""" all_models = datamodels.ModelContainer(miri_dither_ch12) # added a background and test is reset background # removes the background new_container = [] degree = (1, 1, 1,) center = (5, 5, 5,) poly = np.ndarray(9) poly[:] = 1.3 channel = '2' for m in all_models: m.meta.background.polynomial_info.append( { 'degree': degree, 'refpoint': center, 'coefficients': poly.ravel().tolist(), 'channel': channel } ) new_container.append(m) # test if reset backgound - removes background step = MRSIMatchStep() step._reset_background(new_container) for i in range(len(new_container)): m = new_container[i] test = len(m.meta.background.polynomial_info) assert test == 0
def test_imatch_background_subtracted(_jail, miri_dither_ch12): """ Test if data is already background subtracted - raise error""" all_models = datamodels.ModelContainer(miri_dither_ch12) # modify the data set backgroud subtracted new_container = [] for m in all_models: m.meta.background.subtracted = True new_container.append(m) # test if background subtracted - raise error with pytest.raises(ValueError): step = MRSIMatchStep() step.run(new_container)
def test_outlier_step_no_outliers(we_three_sci): """Test whole step, no outliers""" container = datamodels.ModelContainer(list(we_three_sci)) pristine = container.copy() result = OutlierDetectionStep.call(container) # Make sure nothing changed in SCI and DQ arrays for image, uncorrected in zip(pristine, container): np.testing.assert_allclose(image.data, uncorrected.data) np.testing.assert_allclose(image.dq, uncorrected.dq) # Make sure nothing changed in SCI and DQ arrays for image, corrected in zip(container, result): np.testing.assert_allclose(image.data, corrected.data) np.testing.assert_allclose(image.dq, corrected.dq)
def test_outlier_step(we_three_sci): """Test whole step with an outlier""" container = datamodels.ModelContainer(list(we_three_sci)) # Drop a CR on the science array container[0].data[12, 12] += 1e3 result = OutlierDetectionStep.call(container) # Make sure nothing changed in SCI array for image, corrected in zip(container, result): np.testing.assert_allclose(image.data, corrected.data) # Verify source is not flagged for r in result: assert r.dq[7, 7] == datamodels.dqflags.pixel["GOOD"] # Verify CR is flagged assert result[0].dq[12, 12] == OUTLIER_DO_NOT_USE
def test_outlier_step_square_source_no_outliers(we_three_sci): """Test whole step with square source with sharp edges, no outliers""" container = datamodels.ModelContainer(list(we_three_sci)) # put a square source in all three exposures for ccont in container: ccont.data[5:15, 5:15] += 1e3 pristine = container.copy() result = OutlierDetectionStep.call(container) # Make sure nothing changed in SCI and DQ arrays for image, uncorrected in zip(pristine, container): np.testing.assert_allclose(image.data, uncorrected.data) np.testing.assert_allclose(image.dq, uncorrected.dq) # Make sure nothing changed in SCI and DQ arrays for image, corrected in zip(container, result): np.testing.assert_allclose(image.data, corrected.data) np.testing.assert_allclose(image.dq, corrected.dq)
def _generate_data(): """Generate data of each type of input for master background step""" image = datamodels.ImageModel((10, 10)) image.meta.instrument.name = 'MIRI' image.meta.instrument.detector = 'MIRIMAGE' image.meta.exposure.type = 'MIR_LRS-FIXEDSLIT' image.meta.observation.date = '2018-01-01' image.meta.observation.time = '00:00:00' image.meta.subarray.xstart = 1 image.meta.subarray.ystart = 1 image.meta.wcsinfo.v2_ref = 0 image.meta.wcsinfo.v3_ref = 0 image.meta.wcsinfo.roll_ref = 0 image.meta.wcsinfo.ra_ref = 0 image.meta.wcsinfo.dec_ref = 0 image = AssignWcsStep.call(image) # hdulist = create_nirspec_fs_file(grating="G140M", filter="F100LP") # hdulist[1].data = np.zeros((2048, 2048)) # fs = datamodels.ImageModel(hdulist) # fs = AssignWcsStep.call(fs) # multislit = Extract2dStep.call(fs) # hdulist = create_nirspec_ifu_file("OPAQUE", "G140M") # hdulist[1].data = np.zeros((2048, 2048)) # ifu_image = datamodels.IFUImageModel(hdulist) # ifu_image = AssignWcsStep.call(ifu_image) container = datamodels.ModelContainer([image]) cube = datamodels.CubeModel((2, 10, 10)) # Return the data and a status dependent on whether the step can process it return [ (image, 'COMPLETE'), # (multislit, 'COMPLETE'), # (ifu_image, 'COMPLETE'), (container, 'COMPLETE'), (cube, 'SKIPPED'), ]
def test_imatch_degree(_jail, miri_dither_ch12): """ Test if polynomial degree is not a 3 element tuple or integer then raise error """ all_models = datamodels.ModelContainer(miri_dither_ch12) new_container = [] degree = ( 1, 1, 1, ) center = ( 5, 5, 5, ) poly = np.ndarray(9) poly[:] = 1.3 channel = '2' for m in all_models: m.meta.background.polynomial_info.append({ 'degree': degree, 'refpoint': center, 'coefficients': poly.ravel().tolist(), 'channel': channel }) new_container.append(m) # test catches degree set incorrectly - raise error # check that degree must be a 3 element tuple with pytest.raises(ValueError): step = MRSIMatchStep() step.bkg_degree = ( 1, 1, ) step.run(new_container)
def do_detection(self): """Flag outlier pixels in DQ of input images.""" self._convert_inputs() self.build_suffix(**self.outlierpars) pars = self.outlierpars save_intermediate_results = pars['save_intermediate_results'] if pars['resample_data']: # Start by creating resampled/mosaic images for # each group of exposures resamp = resample.ResampleData(self.input_models, single=True, blendheaders=False, **pars) drizzled_models = resamp.do_drizzle() if save_intermediate_results: for model in drizzled_models: log.info("Writing out resampled exposures...") model_output_path = self.make_output_path( basepath=model.meta.filename, suffix='outlier_i2d') model.save(model_output_path) else: # for non-dithered data, the resampled image is just the original image drizzled_models = self.input_models for i in range(len(self.input_models)): drizzled_models[i].wht = build_driz_weight( self.input_models[i], weight_type='ivm', good_bits=pars['good_bits']) # Initialize intermediate products used in the outlier detection median_model = datamodels.ImageModel(drizzled_models[0].data.shape) median_model.update(drizzled_models[0]) median_model.meta.wcs = drizzled_models[0].meta.wcs # Perform median combination on set of drizzled mosaics median_model.data = self.create_median(drizzled_models) if save_intermediate_results: median_output_path = self.make_output_path( basepath=self.input_models[0].meta.filename, suffix='median' ) log.info("Writing out MEDIAN image to: {}".format( median_output_path )) median_model.save(median_output_path) if pars['resample_data']: # Blot the median image back to recreate each input image specified # in the original input list/ASN/ModelContainer blot_models = self.blot_median(median_model) if save_intermediate_results: log.info("Writing out BLOT images...") for model in blot_models: model_path = self.make_output_path( basename=model.meta.filename, suffix='blot' ) model.save(model_path) else: # Median image will serve as blot image blot_models = datamodels.ModelContainer() for i in range(len(self.input_models)): blot_models.append(median_model) # Perform outlier detection using statistical comparisons between # each original input image and its blotted version of the median image self.detect_outliers(blot_models) # clean-up (just to be explicit about being finished with # these results) del median_model, blot_models
def test_apply_background_2d(_jail, miri_dither_ch12): """ Test if background polynomial is set it is subtracted correctly""" all_models = datamodels.ModelContainer(miri_dither_ch12) # test if given a background polynomial - apply_background_2d gives the correct answer # Polynomial information was created by running a faked full array data set to determine # what the polynomial values should be new_container = [] degree = ( 1, 1, 1, ) center = (9.99999408089876, -4.668612949274985e-06, 4.889999912818894) poly = np.ndarray(9) # run AssignWcsStep on first file and set wcs of other files to this channel = '1' for im, m in enumerate(all_models): if im == 0: m = AssignWcsStep.call(m) wcs1 = m.meta.wcs poly = [ -2.50000000e+00, -6.66518331e-15, 5.67845589e-12, -1.23549218e-11, 3.26209108e-12, -5.43180357e-12, -2.54903452e-09, 6.21614553e-09 ] elif im == 1: m.meta.wcs = wcs1 poly = [ -2.50000000e+00, -9.29031986e-15, 1.74437380e-12, -3.94894956e-12, 4.71729481e-13, 5.42031845e-13, -9.96151554e-10, 2.78281950e-09 ] elif im == 2: m.meta.wcs = wcs1 poly = [ 7.50000000e+00, 2.22921836e-14, -6.97131279e-12, 1.58336906e-11, -5.43704212e-12, 6.79109678e-12, 2.63515372e-09, -8.00226976e-09 ] else: m.meta.wcs = wcs1 poly = [ -2.50000000e+00, -6.33668043e-15, -4.51516905e-13, 4.70180795e-13, 1.70322157e-12, -1.90132506e-12, 9.10032350e-10, -9.96695272e-10 ] poly = np.asarray(poly) m.meta.background.polynomial_info.append({ 'degree': degree, 'refpoint': center, 'coefficients': poly.ravel().tolist(), 'channel': channel }) # input 10 output 12.5 for all images apply_background_2d(m, channel, subtract=True) new_container.append(m) # all the data should now be the same for the science pixels # the edge pixels are located in non-science region and no # background subtraction is done on these pixels data1 = new_container[0].data[:, 16:] data2 = new_container[1].data[:, 16:] data3 = new_container[2].data[:, 16:] data4 = new_container[3].data[:, 16:] assert (np.allclose(data1, data2, rtol=1e-6)) assert (np.allclose(data2, data3, rtol=1e-6)) assert (np.allclose(data3, data4, rtol=1e-6)) assert (np.allclose(data1, data4, rtol=1e-6))
def pipeline_mirisim(input_dir): """ """ # set up logging log_file = os.path.join(os.path.abspath(input_dir),'pipeline_MIRISim.log') # check if the pipeline log file exists if os.path.isfile(log_file): os.remove(log_file) else: pass testing_logger = logging.getLogger(__name__) testing_logger.setLevel(logging.DEBUG) handler = logging.FileHandler(log_file) handler.setLevel(logging.DEBUG) formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') handler.setFormatter(formatter) testing_logger.addHandler(handler) # check if the simulation folder exists if os.path.isdir(input_dir): pass else: print("Simulation folder not found") sys.exit(0) # go through individual simulations and run through pipeline simulations = glob.glob(os.path.join(input_dir,'IMA*','20*')) simulations.extend(glob.glob(os.path.join(input_dir,'MRS*','20*'))) simulations.extend(glob.glob(os.path.join(input_dir,'LRS*','20*'))) #for simulation in simulations: print(simulation) # get the full path of the cwd cwd = os.path.abspath(os.getcwd()) # set the output figure directory out_fig_dir = os.path.join(cwd,input_dir,'pipeline_plots') try: shutil.rmtree(out_fig_dir) except: pass os.mkdir(out_fig_dir) # cycle through the simulations for simulation in simulations: os.chdir(os.path.join(simulation,'det_images')) level1b_files = glob.glob('*.fits') # isolate the simulation name for logging sim_name = simulation.split(os.sep)[1] # check which MIRI mode with datamodels.open(level1b_files[0]) as level1b_dm: miri_mode = level1b_dm.meta.exposure.type # IMAGER -------------------------------------------- if miri_mode == 'MIR_IMAGE': # run level 1 and 2 imager pipelines for f in level1b_files: with datamodels.open(f) as level1b_dm: try: level2a_dm = Detector1Pipeline.call(level1b_dm, output_use_model=True, save_results=True, steps={'ipc': {'skip': True}}) level2a_dm.meta.wcsinfo.wcsaxes = 2 Image2Pipeline.call(level2a_dm, save_results=True, output_use_model=True) # log pass testing_logger.info('%s levels 1 and 2 passed' % sim_name) levels12_check = True except Exception as e: testing_logger.warning('%s failed' % sim_name) testing_logger.warning(' %s: %s' % (e.__class__.__name__, str(e))) levels12_check = False # run level 3 pipeline if len(level1b_files) > 1: try: level2B_files = glob.glob(os.path.join('*_cal.fits')) call(["asn_from_list", "-o", "IMA_asn.json"] + level2B_files + ["--product-name", "dither"]) dm_3_container = datamodels.ModelContainer("IMA_asn.json") Image3Pipeline.call(dm_3_container, save_results=True, steps={'tweakreg': {'skip': True}}) # log pass testing_logger.info('%s level 3 passed' % sim_name) level3_check = True except Exception as e: testing_logger.warning('%s failed' % sim_name) testing_logger.warning(' %s: %s' % (e.__class__.__name__, str(e))) level3_check = False if len(level1b_files) == 1 and levels12_check == True: level2A_file = glob.glob(os.path.join('*_rate.fits'))[0] level2B_file = glob.glob(os.path.join('*_cal.fits'))[0] # set up output plots fig, axs = plt.subplots(1, 3) fig.set_figwidth(15.0) fig.set_figheight(5.0) axs = axs.ravel() # plot level 1b, 2a, 2b with datamodels.open(level1b_files[0]) as level1b_dm: with datamodels.open(level2A_file) as level2a_dm: with datamodels.open(level2B_file) as level2b_dm: axs[0].imshow(level1b_dm.data[0][-1], cmap='jet', interpolation='nearest', norm=LogNorm(), origin='lower') axs[0].annotate('level 1B', xy=(0.7, 0.95), xycoords='axes fraction', fontsize=10, fontweight='bold', color='w') axs[1].imshow(level2a_dm.data, cmap='jet', interpolation='nearest', norm=LogNorm(), origin='lower') axs[1].annotate('level 2A', xy=(0.7, 0.95), xycoords='axes fraction', fontsize=10, fontweight='bold', color='w') axs[2].imshow(level2b_dm.data, cmap='jet', interpolation='nearest', norm=LogNorm(), origin='lower') axs[2].annotate('level 2B', xy=(0.7, 0.95), xycoords='axes fraction', fontsize=10, fontweight='bold', color='w') # save the pipeline plot out_fig_name = sim_name + '.pdf' fig.savefig(os.path.join(out_fig_dir, out_fig_name), dpi=200) del fig elif len(level1b_files) > 1 and level3_check == True: driz_dm = datamodels.open('dither_i2d.fits') # set up output plots fig, axs = plt.subplots(1, 1) fig.set_figwidth(8.0) fig.set_figheight(8.0) # plot drizzled image axs.imshow(driz_dm.data, cmap='jet', interpolation='nearest', origin='lower', norm=LogNorm(vmin=1, vmax=1000)) axs.annotate('Drizzled image', xy=(0.0, 1.02), xycoords='axes fraction', fontsize=12, fontweight='bold', color='k') axs.set_facecolor('black') # save the pipeline plot out_fig_name = sim_name + '.pdf' fig.savefig(os.path.join(out_fig_dir, out_fig_name), dpi=200) del fig # MRS -------------------------------------------- elif miri_mode == 'MIR_MRS': # run level 1 and 2 pipelines for f in level1b_files: with datamodels.open(f) as level1b_dm: try: level2a_dm = Detector1Pipeline.call(level1b_dm, output_use_model=True, save_results=True, steps={'ipc': {'skip': True}}) Spec2Pipeline.call(level2a_dm, save_results=True, steps={'straylight':{'skip':True}, 'extract_1d':{'save_results':True}}) # log pass testing_logger.info('%s levels 1 and 2 passed' % sim_name) levels12_check = True except Exception as e: testing_logger.warning('%s failed' % sim_name) testing_logger.warning(' %s: %s' % (e.__class__.__name__, str(e))) levels12_check = False # run level 3 pipeline if len(level1b_files) > 1: try: level2B_files = glob.glob(os.path.join('*_cal.fits')) call(["asn_from_list", "-o", "MRS_asn.json"] + level2B_files + ["--product-name", "dither"]) dm_3_container = datamodels.ModelContainer("MRS_asn.json") Spec3Pipeline.call(dm_3_container, save_results=True, steps={'outlier_detection': {'skip': True}, 'cube_build': {'save_results': True}, 'extract_1d': {'save_results': True}}) # log pass testing_logger.info('%s level 3 passed' % sim_name) level3_check = True except Exception as e: testing_logger.warning('%s failed' % sim_name) testing_logger.warning(' %s: %s' % (e.__class__.__name__, str(e))) level3_check = False if len(level1b_files) == 1 and levels12_check == True: level2A_file = glob.glob('*_rate.fits')[0] level2B_file = glob.glob('*_cal.fits')[0] spec_file = glob.glob('*x1d.fits')[0] # set up output plots fig, axs = plt.subplots(2, 2) fig.set_figwidth(15.0) fig.set_figheight(15.0) axs = axs.ravel() with datamodels.open(level1b_files[0]) as level1b_dm: with datamodels.open(level2A_file) as level2a_dm: with datamodels.open(level2B_file) as level2b_dm: with datamodels.open(spec_file) as spec_dm: axs[0].imshow(level1b_dm.data[0][-1], cmap='jet', interpolation='nearest', norm=LogNorm(), origin='lower') axs[0].annotate('level 1B', xy=(0.7, 0.95), xycoords='axes fraction', fontsize=10, fontweight='bold', color='w') axs[1].imshow(level2a_dm.data, cmap='jet', interpolation='nearest', norm=LogNorm(), origin='lower') axs[1].annotate('level 2A', xy=(0.7, 0.95), xycoords='axes fraction', fontsize=10, fontweight='bold', color='w') axs[2].imshow(level2b_dm.data, cmap='jet', interpolation='nearest', norm=LogNorm(), origin='lower') axs[2].annotate('level 2B', xy=(0.7, 0.95), xycoords='axes fraction', fontsize=10, fontweight='bold', color='w') # plot the spectrum axs[3].plot(spec_dm.spec[0].spec_table['WAVELENGTH'], spec_dm.spec[0].spec_table['FLUX'], c='b', marker='.', markersize=3, linestyle='-', linewidth=2) axs[3].set_ylabel(r'Flux ($\mu$Jy)') axs[3].set_xlabel(r'Wavelength ($\mu$m)') axs[3].set_xlim(4.0, 28.0) # axs[0].set_ylim(0,6000) # save the pipeline plot out_fig_name = sim_name + '.pdf' fig.savefig(os.path.join(out_fig_dir, out_fig_name), dpi=200) del fig elif len(level1b_files) > 1 and level3_check == True: # cube cube_file = glob.glob("dither*s3d.fits")[0] cube_dm = datamodels.open(cube_file) # spec spec_file = glob.glob( "dither*1d.fits")[0] dm = datamodels.open(spec_file) fig, axs = plt.subplots(1,2, figsize=(12, 8)) axs[0].imshow(np.sum(cube_dm.data, axis=0), cmap='jet', interpolation='nearest', origin='lower', norm=LogNorm(vmin=100, vmax=5e5)) axs[0].annotate('Collapsed cube', xy=(0.0, 1.02), xycoords='axes fraction', fontsize=12, fontweight='bold', color='k') axs[0].set_facecolor('black') # plot the spectrum axs[1].plot(dm.spec[0].spec_table['WAVELENGTH'], dm.spec[0].spec_table['FLUX'], c='b', marker='.', markersize=3, linestyle='-', linewidth=2) axs[1].set_ylabel(r'Flux ($\mu$Jy)') axs[1].set_xlabel(r'Wavelength ($\mu$m)') axs[1].set_xlim(4.0, 28.0) # axs[0].set_ylim(0,6000) axs[1].annotate('Spectrum)', xy=(0.0, 1.02), xycoords='axes fraction', fontsize=12, fontweight='bold', color='k') # save the pipeline plot out_fig_name = sim_name + '.pdf' fig.savefig(os.path.join(out_fig_dir, out_fig_name), dpi=200) del fig # LRS-FIXEDSLIT -------------------------------------------- elif miri_mode == 'MIR_LRS-FIXEDSLIT': # run level 1 and 2 pipelines for f in level1b_files: with datamodels.open(f) as level1b_dm: try: level2a_dm = Detector1Pipeline.call(level1b_dm, output_use_model=True, save_results=True, steps={'ipc': {'skip': True}}) Spec2Pipeline.call(level2a_dm, save_results=True, steps={'extract_1d': {'save_results': True}}) # log pass testing_logger.info('%s levels 1 and 2 passed' % sim_name) levels12_check = True except Exception as e: testing_logger.warning('%s failed' % sim_name) testing_logger.warning(' %s: %s' % (e.__class__.__name__, str(e))) levels12_check = False if len(level1b_files) == 1 and levels12_check == True: level2A_file = glob.glob('*_rate.fits')[0] level2B_file = glob.glob('*_cal.fits')[0] spec_file = glob.glob('*x1d.fits')[0] # set up output plots fig, axs = plt.subplots(2, 2) fig.set_figwidth(15.0) fig.set_figheight(15.0) axs = axs.ravel() with datamodels.open(level1b_files[0]) as level1b_dm: with datamodels.open(level2A_file) as level2a_dm: with datamodels.open(level2B_file) as level2b_dm: with datamodels.open(spec_file) as spec_dm: axs[0].imshow(level1b_dm.data[0][-1], cmap='jet', interpolation='nearest', norm=LogNorm(), origin='lower') axs[0].annotate('level 1B', xy=(0.7, 0.95), xycoords='axes fraction', fontsize=10, fontweight='bold', color='w') axs[1].imshow(level2a_dm.data, cmap='jet', interpolation='nearest', norm=LogNorm(), origin='lower') axs[1].annotate('level 2A', xy=(0.7, 0.95), xycoords='axes fraction', fontsize=10, fontweight='bold', color='w') axs[2].imshow(level2b_dm.data, cmap='jet', interpolation='nearest', norm=LogNorm(), origin='lower') axs[2].annotate('level 2B', xy=(0.7, 0.95), xycoords='axes fraction', fontsize=10, fontweight='bold',color='w') # plot the spectrum axs[3].plot(spec_dm.spec[0].spec_table['WAVELENGTH'][1:-1], spec_dm.spec[0].spec_table['FLUX'][1:-1], c='b', marker='.', markersize=3, linestyle='-', linewidth=2) axs[3].set_ylabel(r'Flux ($\mu$Jy)') axs[3].set_xlabel(r'Wavelength ($\mu$m)') axs[3].set_xlim(3.0, 15.0) # axs[0].set_ylim(0,6000) axs[3].annotate('Spectrum)', xy=(0.0, 1.02), xycoords='axes fraction', fontsize=12, fontweight='bold', color='k') axs[3].set_facecolor('white') # save the pipeline plot out_fig_name = sim_name + '.pdf' fig.savefig(os.path.join(out_fig_dir, out_fig_name), dpi=200) del fig # LRS-SLITLESS -------------------------------------------- elif miri_mode == 'MIR_LRS-SLITLESS': level1b_dm = datamodels.open(level1b_files[0]) # correct for the 'SLITLESSPRISM', 'SUBPRISM' conflict #if level1b_dm.meta.exposure.type == 'MIR_LRS-SLITLESS': # level1b_dm.meta.subarray.name = 'SUBPRISM' try: # fix subarray name problem level1b_dm.meta.subarray.name = 'SUBPRISM' Detector1Pipeline.call(level1b_dm, output_use_model=True, save_results=True, steps={'ipc': {'skip': True}, 'lastframe': {'skip': True}}) level2a_ints = glob.glob('*rateints.fits')[0] Spec2Pipeline.call(level2a_ints, save_results=True, steps={'extract_1d': {'save_results': True}}) level2b_ints = glob.glob('*calints.fits')[0] #Tso3Pipeline.call(level2b_ints) # log pass testing_logger.info('%s passed' % sim_name) levels12_check = True except Exception as e: testing_logger.warning('%s failed' % sim_name) testing_logger.warning(' %s: %s' % (e.__class__.__name__, str(e))) levels12_check = False try: level2B_files = glob.glob('*_calints.fits') call(["asn_from_list", "-o", "LRS-SLITLESS_asn.json"] + level2B_files + ["--product-name", "exposures"]) Tso3Pipeline.call('LRS-SLITLESS_asn.json') # , steps={'white_light':{'skip':True}}) # log pass testing_logger.info('%s level 3 passed' % sim_name) level3_check = True except Exception as e: testing_logger.warning('%s failed' % sim_name) testing_logger.warning(' %s: %s' % (e.__class__.__name__, str(e))) level3_check = False if len(level1b_files) == 1 and levels12_check == True: spec_file = glob.glob('*x1dints.fits')[0] # set up output plots fig, axs = plt.subplots(3, 3, sharey=True, sharex=True) fig.set_figwidth(15.0) fig.set_figheight(15.0) axs = axs.ravel() with datamodels.open(spec_file) as spec_dm: for n in range(9): # plot the spectrum axs[n].plot(spec_dm.spec[n].spec_table['WAVELENGTH'][1:-1], spec_dm.spec[n].spec_table['FLUX'][1:-1], c='b', marker='.', markersize=0, linestyle='-', linewidth=2) if n in [0, 3, 6]: axs[n].set_ylabel(r'Flux ($\mu$Jy)') if n in [6, 7, 8]: axs[n].set_xlabel(r'Wavelength ($\mu$m)') # save the pipeline plot out_fig_name = sim_name + '.pdf' plt.tight_layout(pad=0.0) fig.savefig(os.path.join(out_fig_dir, out_fig_name), dpi=200) del fig if level3_check == True: my_lightcurve = glob.glob('*.ecsv')[0] lightcurve_data = Table.read(my_lightcurve, format='ascii.ecsv') fig, axs = plt.subplots(1, 1, figsize=(10, 8)) # plot input and output ramps of the first integration axs.plot(lightcurve_data[0][:], lightcurve_data[1][:], c='b', marker='o', markersize=3, linestyle='-', linewidth=2, label='white light curve') axs.set_title('White light curve', fontsize=15) axs.set_ylabel('white light flux', fontsize=15) axs.set_xlabel('MJD', fontsize=15) plt.tight_layout(h_pad=0) # save the pipeline plot out_fig_name = sim_name + '_whitelight.pdf' plt.tight_layout(pad=0.0) fig.savefig(os.path.join(out_fig_dir, out_fig_name), dpi=200) del fig os.chdir(cwd)