Ejemplo n.º 1
0
def test_master_background_userbg(_jail, user_background):
    """Verify data can run through the step with a user-supplied background"""

    image = datamodels.ImageModel((10, 10))
    image.meta.instrument.name = 'MIRI'
    image.meta.instrument.detector = 'MIRIMAGE'
    image.meta.exposure.type = 'MIR_LRS-FIXEDSLIT'
    image.meta.observation.date = '2018-01-01'
    image.meta.observation.time = '00:00:00'
    image.meta.subarray.xstart = 1
    image.meta.subarray.ystart = 1
    image.meta.wcsinfo.v2_ref = 0
    image.meta.wcsinfo.v3_ref = 0
    image.meta.wcsinfo.roll_ref = 0
    image.meta.wcsinfo.ra_ref = 0
    image.meta.wcsinfo.dec_ref = 0
    image = AssignWcsStep.call(image)

    # Run with a user-supplied background and verify this is recorded in header
    result = MasterBackgroundStep.call(image, user_background=user_background)

    collect_pipeline_cfgs('./config')
    result = MasterBackgroundStep.call(
        image,
        config_file='config/master_background.cfg',
        user_background=user_background,
    )

    # For inputs that are not files, the following should be true
    assert type(image) is type(result)
    assert result is not image
    assert result.meta.cal_step.master_background == 'COMPLETE'
    assert result.meta.background.master_background_file == 'user_background.fits'
Ejemplo n.º 2
0
def test_master_background_logic(_jail, user_background, science_image):
    """Verify if calspec 2 background step was run the master background step will be skipped"""

    # the background step in calspec2 was done
    science_image.meta.cal_step.back_sub = 'COMPLETE'

    # Run with a user-supplied background
    collect_pipeline_cfgs('./config')
    result = MasterBackgroundStep.call(
        science_image,
        config_file='config/master_background.cfg',
        user_background=user_background,
    )

    assert result.meta.cal_step.master_background == 'SKIPPED'
    assert type(science_image) is type(result)

    # Now force it
    result = MasterBackgroundStep.call(
        science_image,
        config_file='config/master_background.cfg',
        user_background=user_background,
        force_subtract=True)

    assert result.meta.cal_step.master_background == 'COMPLETE'
    assert type(science_image) is type(result)
Ejemplo n.º 3
0
def test_master_background_init(input_data, status, _jail, user_background):
    """Verify data can run through the step"""

    result = MasterBackgroundStep.call(input_data)

    collect_pipeline_cfgs('./config')
    result = MasterBackgroundStep.call(
        input_data, config_file='config/master_background.cfg')

    # For inputs that are not files, the following should be true
    assert type(input_data) is type(result)
    assert result is not input_data

    if isinstance(result, datamodels.ModelContainer):
        for model in result:
            assert model.meta.cal_step.master_background == status
    else:
        assert result.meta.cal_step.master_background == status

    # Run with a user-supplied background and verify this is recorded in header
    result = MasterBackgroundStep.call(input_data,
                                       user_background=user_background)

    if isinstance(result, datamodels.ModelContainer):
        for model in result:
            if model.meta.cal_step.master_background == 'COMPLETE':
                assert model.meta.background.master_background_file == 'user_background.fits'
    else:
        if result.meta.cal_step.master_background == 'COMPLETE':
            assert result.meta.background.master_background_file == 'user_background.fits'

    # Make sure saving the computed background works
    result = MasterBackgroundStep.call(input_data, save_background=True)
Ejemplo n.º 4
0
def test_master_background_userbg(_jail, user_background):
    """Verify data can run through the step with a user-supplied background"""

    image = datamodels.ImageModel((10, 10))
    image.meta.instrument.name = 'MIRI'
    image.meta.instrument.detector = 'MIRIMAGE'
    image.meta.exposure.type = 'MIR_LRS-FIXEDSLIT'
    image.meta.observation.date = '2018-01-01'
    image.meta.observation.time = '00:00:00'
    image.meta.subarray.xstart = 1
    image.meta.subarray.ystart = 1
    image.meta.wcsinfo.v2_ref = 0
    image.meta.wcsinfo.v3_ref = 0
    image.meta.wcsinfo.roll_ref = 0
    image.meta.wcsinfo.ra_ref = 0
    image.meta.wcsinfo.dec_ref = 0
    image = AssignWcsStep.call(image)

    # Run with a user-supplied background and verify this is recorded in header
    result = MasterBackgroundStep.call(image, user_background=user_background)

    collect_pipeline_cfgs('./config')
    result = MasterBackgroundStep.call(
        image,
        config_file='config/master_background.cfg',
        user_background=user_background,
        )

    # For inputs that are not files, the following should be true
    assert type(image) is type(result)
    assert result is not image
    assert result.meta.cal_step.master_background == 'COMPLETE'
    assert result.meta.background.master_background_file == 'user_background.fits'
Ejemplo n.º 5
0
def run_pipeline(jail, rtdata_module):

    rtdata = rtdata_module

    rtdata.get_asn("miri/mrs/miri_mrs_mbkg_nodded_spec3_asn.json")

    MasterBackgroundStep.call(rtdata.input, save_results=True, suffix='master_background')

    return rtdata
Ejemplo n.º 6
0
    def test_nirspec_fs_masterbg_user(self):
        """
        Regression test of master background subtraction for NRS FS when a
        user 1-D spectrum is provided.
        """
        # input file has 2-D background image added to it

        input_file = self.get_data(*self.test_dir, 'nrs_sci+bkg_cal.fits')
        # user provided 1-D background was created from the 2-D background image
        input_1dbkg_file = self.get_data(*self.test_dir, 'nrs_bkg_user_clean_x1d.fits')

        result = MasterBackgroundStep.call(input_file,
                                           user_background=input_1dbkg_file,
                                           save_results=True)

        # Compare background-subtracted science data (results)
        # to a truth file. These data are MultiSlitModel data
        result_file = result.meta.filename

        truth_file = self.get_data(*self.ref_loc,
                                  'nrs_sci+bkg_masterbackgroundstep.fits')

        outputs = [(result_file, truth_file)]
        self.compare_outputs(outputs)
        result.close()
Ejemplo n.º 7
0
    def test_miri_masterbg_mrs_nodded(self):
        """Run masterbackground step on MIRI MRS association"""
        asn_file = self.get_data(*self.test_dir,
                                 'miri_mrs_mbkg_spec3_asn.json')
        for file in raw_from_asn(asn_file):
            self.get_data(*self.test_dir, file)

        collect_pipeline_cfgs('./config')
        result = MasterBackgroundStep.call(
            asn_file,
            config_file='config/master_background.cfg',
            save_background=True,
            save_results=True,
        )

        # test 1
        # loop over the background subtracted data and compare to truth files
        # check that the  cal_step master_background ran to complete
        for model in result:
            assert model.meta.cal_step.master_background == 'COMPLETE'
            truth_file = self.get_data(*self.ref_loc, model.meta.filename)
            outputs = [(model.meta.filename, truth_file)]
            self.compare_outputs(outputs)

        # test 2
        # compare the master background combined file to truth file
        master_combined_bkg_file = 'MIRI_MRS_nod_seq1_MIRIFUSHORT_12SHORTexp1_o001_masterbg.fits'
        truth_background = self.get_data(*self.ref_loc,
                                         master_combined_bkg_file)
        outputs = [(master_combined_bkg_file, truth_background)]
        self.compare_outputs(outputs)
Ejemplo n.º 8
0
    def test_nirspec_masterbg_nodded(self):
        """Run masterbackground step on NIRSpec association"""
        asn_file = self.get_data(*self.test_dir, 'nirspec_spec3_asn.json')
        for file in raw_from_asn(asn_file):
            self.get_data(*self.test_dir, file)

        collect_pipeline_cfgs('./config')
        result = MasterBackgroundStep.call(
            asn_file,
            config_file='config/master_background.cfg',
            save_background=True,
            save_results=True)

        # test 1
        # compare  background subtracted data  to truth files
        # check that the  cal_step master_background ran to complete
        outputs = []
        for model in result:
            assert model.meta.cal_step.master_background == 'COMPLETE'

            result_file = model.meta.filename.replace('cal',
                                                      'master_background')
            truth_file = self.get_data(*self.ref_loc, result_file)

            outputs.append((result_file, truth_file))
        self.compare_outputs(outputs)

        # test 2
        # compare the master background combined file to truth file
        master_combined_bkg_file = 'ifu_prism_source_off_fix_NRS1_o001_masterbg.fits'
        truth_background = self.get_data(*self.ref_loc,
                                         master_combined_bkg_file)
        outputs = [(master_combined_bkg_file, truth_background)]
        self.compare_outputs(outputs)
Ejemplo n.º 9
0
    def test_miri_masterbg_lrs_dedicated(self):
        """Run masterbackground step on MIRI LRS association"""
        asn_file = self.get_data(*self.test_dir,
                                  'miri_lrs_mbkg_dedicated_spec3_asn.json')
        for file in raw_from_asn(asn_file):
            self.get_data(*self.test_dir, file)

        collect_pipeline_cfgs('./config')
        result = MasterBackgroundStep.call(
            asn_file,
            config_file='config/master_background.cfg',
            save_background = True,
            save_results = True)

        # test 1
        # loop over the background subtracted data and compare to truth files
        for model in result:
            assert model.meta.cal_step.master_background == 'COMPLETE'

            result_file = model.meta.filename.replace('cal', 'master_background')
            truth_file = self.get_data(*self.ref_loc,
                                        result_file)

            outputs = [(result_file, truth_file)]
            self.compare_outputs(outputs)

        # test 2
        # compare the master background combined file to truth file
        master_combined_bkg_file = 'MIRI_LRS_seq1_MIRIMAGE_P750Lexp1_o001_masterbg.fits'
        truth_background = self.get_data(*self.ref_loc,
                                          master_combined_bkg_file)
        outputs = [(master_combined_bkg_file, truth_background)]
        self.compare_outputs(outputs)
Ejemplo n.º 10
0
    def test_nirspec_fs_masterbg_user(self):
        """
        Regression test of master background subtraction for NRS FS when a
        user 1-D spectrum is provided.
        """
        # input file has 2-D background image added to it

        input_file = self.get_data(*self.test_dir, 'nrs_sci+bkg_cal.fits')
        # user provided 1-D background was created from the 2-D background image
        input_1dbkg_file = self.get_data(*self.test_dir, 'nrs_bkg_user_clean_x1d.fits')

        result = MasterBackgroundStep.call(input_file,
                                           user_background=input_1dbkg_file,
                                           save_results=True)

        # Compare background-subtracted science data (results)
        # to a truth file. These data are MultiSlitModel data
        result_file = result.meta.filename

        truth_file = self.get_data(*self.ref_loc,
                                  'nrs_sci+bkg_masterbackgroundstep.fits')

        outputs = [(result_file, truth_file)]
        self.compare_outputs(outputs)
        result.close()
Ejemplo n.º 11
0
    def test_miri_masterbg_lrs_dedicated(self):
        """Run masterbackground step on MIRI LRS association"""
        asn_file = self.get_data(*self.test_dir,
                                 'miri_lrs_mbkg_dedicated_spec3_asn.json')
        for file in raw_from_asn(asn_file):
            self.get_data(*self.test_dir, file)

        collect_pipeline_cfgs('./config')
        result = MasterBackgroundStep.call(
            asn_file,
            config_file='config/master_background.cfg',
            save_background=True,
            save_results=True)

        # test 1
        # loop over the background subtracted data and compare to truth files
        for model in result:
            assert model.meta.cal_step.master_background == 'COMPLETE'

            result_file = model.meta.filename.replace('cal',
                                                      'master_background')
            truth_file = self.get_data(*self.ref_loc, result_file)

            outputs = [(result_file, truth_file)]
            self.compare_outputs(outputs)

        # test 2
        # compare the master background combined file to truth file
        master_combined_bkg_file = 'MIRI_LRS_seq1_MIRIMAGE_P750Lexp1_o001_masterbg.fits'
        truth_background = self.get_data(*self.ref_loc,
                                         master_combined_bkg_file)
        outputs = [(master_combined_bkg_file, truth_background)]
        self.compare_outputs(outputs)
def test_nirspec_ifu_mbkg_nod(rtdata, fitsdiff_default_kwargs, output_file):
    """Test NIRSpec IFU prism nodded data."""
    # Get input data
    rtdata.get_asn("nirspec/ifu/nirspec_spec3_asn.json")

    MasterBackgroundStep.call(rtdata.input, save_background=True, save_results=True,
                              suffix='master_background')

    rtdata.output = output_file

    # Get the truth file
    rtdata.get_truth(f"truth/test_nirspec_ifu_mbkg_nod/{output_file}")

    # Compare the results
    diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs)
    assert diff.identical, diff.report()
Ejemplo n.º 13
0
    def test_nirspec_ifu_masterbg_user(self):
        """
        Regression test of master background subtraction for NRS IFU when a
        user 1-D spectrum is provided.
        """
        # input file has 2-D background image added to it
        input_file = self.get_data(*self.test_dir, 'prism_sci_bkg_cal.fits')

        # user-provided 1-D background was created from the 2-D background image
        user_background = self.get_data(*self.test_dir, 'prism_bkg_x1d.fits')

        result = MasterBackgroundStep.call(input_file,
                                           user_background=user_background,
                                           save_results=True)

        # Test 2  compare the science  data with no background
        # to the output from the masterBackground Subtraction step
        # background subtracted science image.
        input_sci_cal_file = self.get_data(*self.test_dir,
                                           'prism_sci_cal.fits')
        input_sci_model = datamodels.open(input_sci_cal_file)

        # We don't want the slices gaps to impact the statisitic
        # loop over the 30 Slices
        for i in range(30):
            slice_wcs = nirspec.nrs_wcs_set_input(input_sci_model, i)
            x, y = grid_from_bounding_box(slice_wcs.bounding_box)
            ra, dec, lam = slice_wcs(x, y)
            valid = np.isfinite(lam)
            result_slice_region = result.data[y.astype(int), x.astype(int)]
            sci_slice_region = input_sci_model.data[y.astype(int),
                                                    x.astype(int)]
            sci_slice = sci_slice_region[valid]
            result_slice = result_slice_region[valid]
            sub = result_slice - sci_slice

            # check for outliers in the science image
            sci_mean = np.nanmean(sci_slice)
            sci_std = np.nanstd(sci_slice)
            upper = sci_mean + sci_std * 5.0
            lower = sci_mean - sci_std * 5.0
            mask_clean = np.logical_and(sci_slice < upper, sci_slice > lower)

            sub_mean = np.absolute(np.nanmean(sub[mask_clean]))
            atol = 2.0
            assert_allclose(sub_mean, 0, atol=atol)

        # Test 3 Compare background sutracted science data (results)
        #  to a truth file. This data is MultiSlit data

        input_sci_model.close()
        result_file = result.meta.filename
        truth_file = self.get_data(*self.ref_loc,
                                   'prism_sci_bkg_masterbackgroundstep.fits')

        outputs = [(result_file, truth_file)]
        self.compare_outputs(outputs)
        input_sci_model.close()
        result.close()
Ejemplo n.º 14
0
def run_pipeline(jail, rtdata_module):

    rtdata = rtdata_module

    # This is the user-supplied background file.
    rtdata.get_data("miri/lrs/miri_lrs_bkg_x1d.fits")
    user_bkg = rtdata.input

    # This is the input file for the master_background step.
    rtdata.get_data("miri/lrs/miri_lrs_sci+bkg_cal.fits")

    MasterBackgroundStep.call(rtdata.input,
                              user_background=user_bkg,
                              save_results=True,
                              suffix='master_background')

    return rtdata
Ejemplo n.º 15
0
    def test_miri_lrs_masterbg_user(self):
        """
        Regression test of masterbackgound subtraction with lrs, with user provided 1-D background
        """

        # input file has the background added
        input_file = self.get_data(*self.test_dir, 'miri_lrs_sci+bkg_cal.fits')
        # user provided 1-D background
        user_background = self.get_data(*self.test_dir,
                                        'miri_lrs_bkg_x1d.fits')

        result = MasterBackgroundStep.call(input_file,
                                           user_background=user_background,
                                           save_results=True)

        # Compare result (background subtracted image) to science image with no
        # background. Subtract these images, smooth the subtracted image and
        # the mean should be close to zero.
        input_sci_cal_file = self.get_data(*self.test_dir,
                                           'miri_lrs_sci_cal.fits')
        input_sci = datamodels.open(input_sci_cal_file)

        # find the LRS region
        bb = result.meta.wcs.bounding_box
        x, y = grid_from_bounding_box(bb)
        result_lrs_region = result.data[y.astype(int), x.astype(int)]
        sci_lrs_region = input_sci.data[y.astype(int), x.astype(int)]

        # do a 5 sigma clip on the science image
        sci_mean = np.nanmean(sci_lrs_region)
        sci_std = np.nanstd(sci_lrs_region)
        upper = sci_mean + sci_std * 5.0
        lower = sci_mean - sci_std * 5.0
        mask_clean = np.logical_and(sci_lrs_region < upper,
                                    sci_lrs_region > lower)

        sub = result_lrs_region - sci_lrs_region
        mean_sub = np.absolute(np.mean(sub[mask_clean]))

        atol = 0.1
        rtol = 0.001
        assert_allclose(mean_sub, 0, atol=atol, rtol=rtol)

        # Test 3 Compare background subtracted science data (results)
        #  to a truth file.
        truth_file = self.get_data(
            *self.ref_loc, 'miri_lrs_sci+bkg_masterbackgroundstep.fits')

        result_file = result.meta.filename
        outputs = [(result_file, truth_file)]
        self.compare_outputs(outputs)
        result.close()
        input_sci.close()
def test_nirspec_ifu_mbkg_user(rtdata, fitsdiff_default_kwargs):
    """Test NIRSpec IFU data with a user-supplied background file."""
    # Get user-supplied background
    user_background = "prism_bkg_x1d.fits"
    rtdata.get_data(f"nirspec/ifu/{user_background}")

    # Get input data
    rtdata.get_data("nirspec/ifu/prism_sci_bkg_cal.fits")

    MasterBackgroundStep.call(rtdata.input, user_background=user_background,
                              save_results=True, suffix='master_background')

    output = "prism_sci_bkg_master_background.fits"
    rtdata.output = output

    # Get the truth file
    rtdata.get_truth(f"truth/test_nirspec_ifu_mbkg_user/{output}")

    # Compare the results
    diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs)
    assert diff.identical, diff.report()
def test_nirspec_fs_mbkg_user(rtdata, fitsdiff_default_kwargs):
    """Run a test for NIRSpec FS data with a user-supplied background file."""

    # Get user-supplied background
    user_background = "v2_nrs_bkg_user_clean_x1d.fits"
    rtdata.get_data(f"nirspec/fs/{user_background}")

    # Get input data
    rtdata.get_data("nirspec/fs/nrs_sci+bkg_cal.fits")

    MasterBackgroundStep.call(rtdata.input, save_results=True, suffix='master_background',
                              user_background=user_background)

    output = "nrs_sci+bkg_master_background.fits"
    rtdata.output = output

    # Get the truth file
    rtdata.get_truth(f"truth/test_nirspec_fs_mbkg_user/{output}")

    # Compare the results
    diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs)
    assert diff.identical, diff.report()
Ejemplo n.º 18
0
    def test_miri_lrs_masterbg_user(self):
        """
        Regression test of masterbackgound subtraction with lrs, with user provided 1-D background
        """

        # input file has the background added
        input_file = self.get_data(*self.test_dir, 'miri_lrs_sci+bkg_cal.fits')
        # user provided 1-D background
        user_background = self.get_data(*self.test_dir, 'miri_lrs_bkg_x1d.fits')

        result = MasterBackgroundStep.call(input_file,
                                           user_background=user_background,
                                           save_results=True)

        # Compare result (background subtracted image) to science image with no
        # background. Subtract these images, smooth the subtracted image and
        # the mean should be close to zero.
        input_sci_cal_file = self.get_data(*self.test_dir,
                                            'miri_lrs_sci_cal.fits')
        input_sci = datamodels.open(input_sci_cal_file)

        # find the LRS region
        bb = result.meta.wcs.bounding_box
        x, y = grid_from_bounding_box(bb)
        result_lrs_region = result.data[y.astype(int), x.astype(int)]
        sci_lrs_region = input_sci.data[y.astype(int), x.astype(int)]

        # do a 5 sigma clip on the science image
        sci_mean = np.nanmean(sci_lrs_region)
        sci_std = np.nanstd(sci_lrs_region)
        upper = sci_mean + sci_std*5.0
        lower = sci_mean - sci_std*5.0
        mask_clean = np.logical_and(sci_lrs_region < upper, sci_lrs_region > lower)

        sub = result_lrs_region - sci_lrs_region
        mean_sub = np.absolute(np.mean(sub[mask_clean]))

        atol = 0.1
        rtol = 0.001
        assert_allclose(mean_sub, 0, atol=atol, rtol=rtol)

        # Test 3 Compare background subtracted science data (results)
        #  to a truth file.
        truth_file = self.get_data(*self.ref_loc,
                                  'miri_lrs_sci+bkg_masterbackgroundstep.fits')

        result_file = result.meta.filename
        outputs = [(result_file, truth_file)]
        self.compare_outputs(outputs)
        result.close()
        input_sci.close()
Ejemplo n.º 19
0
def test_master_background_userbg(_jail, user_background, science_image):
    """Verify data can run through the step with a user-supplied background"""

    # Run with a user-supplied background and verify this is recorded in header
    collect_pipeline_cfgs('./config')
    result = MasterBackgroundStep.call(
        science_image,
        config_file='config/master_background.cfg',
        user_background=user_background,
    )

    assert type(science_image) is type(result)
    assert result is not science_image
    assert result.meta.cal_step.master_background == 'COMPLETE'
    assert result.meta.background.master_background_file == 'user_background.fits'
Ejemplo n.º 20
0
    def test_miri_masterbackground_mrs(self):
        """Run masterbackground step on MIRI LRS association"""
        asn_file = self.get_data(*self.test_dir,
                                   'miri_mrs_mbkg_0304_spec3_asn.json')
        for file in raw_from_asn(asn_file):
            self.get_data(*self.test_dir, file)

        collect_pipeline_cfgs('./config')
        result = MasterBackgroundStep.call(
            asn_file,
            config_file='config/master_background.cfg',
            save_background=True
            )

        for model in result:
            assert model.meta.cal_step.master_background == 'COMPLETE'
Ejemplo n.º 21
0
    def test_nirspec_masterbackground_mos_user1d(self):
        """
        Regression test of master background subtraction for NRS MOS when a user 1-D spectrum is provided.

        """
        # input file has 2-D background image added to it
        input_file = self.get_data(*self.test_dir,
                                    'nrs_mos_sci+bkg_cal.fits')
        # user provide 1-D background was created from the 2-D background image
        input_1dbkg_file = self.get_data(*self.test_dir,
                                          'nrs_mos_bkg_x1d.fits')

        result = MasterBackgroundStep.call(input_file,
                                           user_background=input_1dbkg_file,
                                           save_results=True)
        # _________________________________________________________________________
        # One of out tests is to compare the 1-D extracted spectra from
        # the science image (no background added) and the masterbackground subtracted
        # data.

        # run resample_spec  on results from MasterBackground step
        result_2d = ResampleSpecStep.call(result)
        # run 1-D extract on results from MasterBackground step
        result_1d = Extract1dStep.call(result_2d)

        # get input science data with background added
        input_sci_cal_file = self.get_data(*self.test_dir,
                                            'nrs_mos_sci_cal.fits')
        # get 1-D extract on original science data without background
        # this reference data was also run through ResampleSpec
        input_sci_1d_file = self.get_data(*self.ref_loc,
                                            'nrs_mos_sci_extract1dstep.fits')

        input_sci = datamodels.open(input_sci_cal_file)
        sci_cal_1d = datamodels.open(input_sci_1d_file)
        num_spec = len(sci_cal_1d.spec)

        # the user 1D spectrum may not cover the entire wavelength range of the
        # science data.  Find the wavelength range of user 1-D spectra

        input_1dbkg_1d = datamodels.open(input_1dbkg_file)
        user_wave = input_1dbkg_1d.spec[0].spec_table['wavelength']
        user_flux = input_1dbkg_1d.spec[0].spec_table['flux']
        user_wave_valid = np.where(user_flux > 0)
        min_user_wave = np.amin(user_wave[user_wave_valid])
        max_user_wave = np.amax(user_wave[user_wave_valid])
        input_1dbkg_1d.close()

        # loop over each slit and perform 2 tests on each slit
        for i in range(num_spec):
            # ______________________________________________________________________
            # Test 1 compare extracted spectra data from the science data
            # to extracted spectra from the output
            # from MasterBackground subtraction.
            sci_spec_1d = sci_cal_1d.spec[i].spec_table['flux']
            sci_wave = sci_cal_1d.spec[i].spec_table['wavelength']
            result_spec_1d = result_1d.spec[i].spec_table['flux']

            # find the waverange covered by both user 1-D and science slit
            sci_wave_valid = np.where(sci_spec_1d > 0)
            min_wave = np.amin(sci_wave[sci_wave_valid])
            max_wave = np.amax(sci_wave[sci_wave_valid])
            if min_user_wave > min_wave:
                min_wave = min_user_wave
            if max_user_wave < max_wave:
                max_wave = max_user_wave

            sub_spec = sci_spec_1d - result_spec_1d
            valid = np.where(np.logical_and(sci_wave > min_wave, sci_wave < max_wave))
            sub_spec = sub_spec[valid]
            mean_sub = np.nanmean(sub_spec)
            atol = 1.5
            assert_allclose(mean_sub, 0, atol=atol)
            # ______________________________________________________________________
            # Test 2  compare the science  data with no background
            # to the output from the masterBackground Subtraction step
            # background subtracted science image.
            bb = input_sci.slits[i].meta.wcs.bounding_box
            x, y = grid_from_bounding_box(bb)
            ra, dec, lam = input_sci.slits[i].meta.wcs(x, y)
            valid = np.isfinite(lam)

            sci = input_sci.slits[i].data
            result_slit = result.slits[i].data

            # check for outliers in the science image that could cause test
            # to fail. These could be cosmic ray hits or other yeck that
            # messes up the science data - ignores these cases
            sci_mean = np.nanmean(sci[valid])
            sci_std = np.nanstd(sci[valid])
            upper = sci_mean + sci_std*5.0
            lower = sci_mean - sci_std*5.0
            mask_clean = np.logical_and(sci[valid] < upper, sci[valid] > lower)

            # for this slit subtract the background subtracted data from
            # the science data with no background added
            sub = result_slit - sci
            # do not look at outliers - they confuse things
            sub_valid = sub[valid]
            mean_sub = np.mean(sub_valid[mask_clean])
            atol = 0.1
            assert_allclose(np.absolute(mean_sub), 0, atol=atol)
        # ______________________________________________________________________
        # Test 3 Compare background sutracted science data (results)
        #  to a truth file. This data is MultiSlit data

        result_file = result.meta.filename
        ref_file = self.get_data(*self.ref_loc,
                                  'nrs_mos_sci+bkg_masterbackgroundstep.fits')

        outputs = [(result_file, ref_file)]
        self.compare_outputs(outputs)
        input_sci.close()
        result.close()
Ejemplo n.º 22
0
    def test_nirspec_ifu_masterbg_user(self):
        """
        Regression test of master background subtraction for NRS IFU when a
        user 1-D spectrum is provided.
        """
        # input file has 2-D background image added to it
        input_file = self.get_data(*self.test_dir, 'prism_sci_bkg_cal.fits')

        # user-provided 1-D background was created from the 2-D background image
        user_background = self.get_data(*self.test_dir, 'prism_bkg_x1d.fits')

        result = MasterBackgroundStep.call(input_file,
                                           user_background=user_background,
                                           save_results=True)

        # Test 1 compare extracted spectra data with
        # no background added to extracted spectra from the output
        # from MasterBackground subtraction. First cube_build has to be run
        # on the data.
        result_s3d = CubeBuildStep.call(result)
        # run 1-D extract on results from MasterBackground step
        result_1d = Extract1dStep.call(result_s3d, subtract_background=False)

        # get the 1-D extracted spectrum from the science data in truth directory
        input_sci_1d_file = self.get_data(*self.ref_loc, 'prism_sci_extract1d.fits')
        sci_1d = datamodels.open(input_sci_1d_file)

        # read in the valid wavelengths of the user-1d
        user_background_model = datamodels.open(user_background)
        user_wave = user_background_model.spec[0].spec_table['wavelength']
        user_flux = user_background_model.spec[0].spec_table['flux']
        user_wave_valid = np.where(user_flux > 0)
        min_user_wave = np.amin(user_wave[user_wave_valid])
        max_user_wave = np.amax(user_wave[user_wave_valid])
        user_background_model.close()
        # find the waverange covered by both user and science
        sci_spec_1d = sci_1d.spec[0].spec_table['flux']
        sci_spec_wave = sci_1d.spec[0].spec_table['wavelength']

        result_spec_1d = result_1d.spec[0].spec_table['flux']

        sci_wave_valid = np.where(sci_spec_1d > 0)
        min_wave = np.amin(sci_spec_wave[sci_wave_valid])
        max_wave = np.amax(sci_spec_wave[sci_wave_valid])
        if min_user_wave > min_wave:
            min_wave = min_user_wave
        if max_user_wave < max_wave:
            max_wave = max_user_wave

        sub_spec = sci_spec_1d - result_spec_1d
        valid = np.where(np.logical_and(sci_spec_wave > min_wave, sci_spec_wave < max_wave))
        sub_spec = sub_spec[valid]
        sub_spec = sub_spec[1:-2]  # endpoints are wacky

        mean_sub = np.absolute(np.nanmean(sub_spec))
        atol = 5.0
        assert_allclose(mean_sub, 0, atol=atol)

        # Test 2  compare the science  data with no background
        # to the output from the masterBackground Subtraction step
        # background subtracted science image.
        input_sci_cal_file = self.get_data(*self.test_dir,
                                            'prism_sci_cal.fits')
        input_sci_model = datamodels.open(input_sci_cal_file)

        # We don't want the slices gaps to impact the statisitic
        # loop over the 30 Slices
        for i in range(30):
            slice_wcs = nirspec.nrs_wcs_set_input(input_sci_model, i)
            x, y = grid_from_bounding_box(slice_wcs.bounding_box)
            ra, dec, lam = slice_wcs(x, y)
            valid = np.isfinite(lam)
            result_slice_region = result.data[y.astype(int), x.astype(int)]
            sci_slice_region = input_sci_model.data[y.astype(int),
                                                    x.astype(int)]
            sci_slice = sci_slice_region[valid]
            result_slice = result_slice_region[valid]
            sub = result_slice - sci_slice

            # check for outliers in the science image
            sci_mean = np.nanmean(sci_slice)
            sci_std = np.nanstd(sci_slice)
            upper = sci_mean + sci_std*5.0
            lower = sci_mean - sci_std*5.0
            mask_clean = np.logical_and(sci_slice < upper, sci_slice > lower)

            sub_mean = np.absolute(np.nanmean(sub[mask_clean]))
            atol = 2.0
            assert_allclose(sub_mean, 0, atol=atol)

        # Test 3 Compare background sutracted science data (results)
        #  to a truth file. This data is MultiSlit data

        input_sci_model.close()
        result_file = result.meta.filename
        truth_file = self.get_data(*self.ref_loc,
                                  'prism_sci_bkg_masterbackgroundstep.fits')

        outputs = [(result_file, truth_file)]
        self.compare_outputs(outputs)
        input_sci_model.close()
        result.close()
Ejemplo n.º 23
0
    def test_nirspec_masterbackground_mos_user1d(self):
        """
        Regression test of master background subtraction for NRS MOS when a user 1-D spectrum is provided.

        """
        # input file has 2-D background image added to it
        input_file = self.get_data(*self.test_dir, 'nrs_mos_sci+bkg_cal.fits')
        # user provide 1-D background was created from the 2-D background image
        input_1dbkg_file = self.get_data(*self.test_dir,
                                         'nrs_mos_bkg_x1d.fits')

        result = MasterBackgroundStep.call(input_file,
                                           user_background=input_1dbkg_file,
                                           save_results=True)
        # _________________________________________________________________________
        # One of out tests is to compare the 1-D extracted spectra from
        # the science image (no background added) and the masterbackground subtracted
        # data.

        # run resample_spec  on results from MasterBackground step
        result_2d = ResampleSpecStep.call(result)
        # run 1-D extract on results from MasterBackground step
        result_1d = Extract1dStep.call(result_2d)

        # get input science data with background added
        input_sci_cal_file = self.get_data(*self.test_dir,
                                           'nrs_mos_sci_cal.fits')
        # get 1-D extract on original science data without background
        # this reference data was also run through ResampleSpec
        input_sci_1d_file = self.get_data(*self.ref_loc,
                                          'nrs_mos_sci_extract1dstep.fits')

        input_sci = datamodels.open(input_sci_cal_file)
        sci_cal_1d = datamodels.open(input_sci_1d_file)
        num_spec = len(sci_cal_1d.spec)

        # the user 1D spectrum may not cover the entire wavelength range of the
        # science data.  Find the wavelength range of user 1-D spectra

        input_1dbkg_1d = datamodels.open(input_1dbkg_file)
        user_wave = input_1dbkg_1d.spec[0].spec_table['wavelength']
        user_flux = input_1dbkg_1d.spec[0].spec_table['flux']
        user_wave_valid = np.where(user_flux > 0)
        min_user_wave = np.amin(user_wave[user_wave_valid])
        max_user_wave = np.amax(user_wave[user_wave_valid])
        input_1dbkg_1d.close()

        # loop over each slit and perform 2 tests on each slit
        for i in range(num_spec):
            # ______________________________________________________________________
            # Test 1 compare extracted spectra data from the science data
            # to extracted spectra from the output
            # from MasterBackground subtraction.
            sci_spec_1d = sci_cal_1d.spec[i].spec_table['flux']
            sci_wave = sci_cal_1d.spec[i].spec_table['wavelength']
            result_spec_1d = result_1d.spec[i].spec_table['flux']

            # find the waverange covered by both user 1-D and science slit
            sci_wave_valid = np.where(sci_spec_1d > 0)
            min_wave = np.amin(sci_wave[sci_wave_valid])
            max_wave = np.amax(sci_wave[sci_wave_valid])
            if min_user_wave > min_wave:
                min_wave = min_user_wave
            if max_user_wave < max_wave:
                max_wave = max_user_wave

            sub_spec = sci_spec_1d - result_spec_1d
            valid = np.where(
                np.logical_and(sci_wave > min_wave, sci_wave < max_wave))
            sub_spec = sub_spec[valid]
            mean_sub = np.nanmean(sub_spec)
            atol = 1.5
            assert_allclose(mean_sub, 0, atol=atol)
            # ______________________________________________________________________
            # Test 2  compare the science  data with no background
            # to the output from the masterBackground Subtraction step
            # background subtracted science image.
            bb = input_sci.slits[i].meta.wcs.bounding_box
            x, y = grid_from_bounding_box(bb)
            ra, dec, lam = input_sci.slits[i].meta.wcs(x, y)
            valid = np.isfinite(lam)

            sci = input_sci.slits[i].data
            result_slit = result.slits[i].data

            # check for outliers in the science image that could cause test
            # to fail. These could be cosmic ray hits or other yeck that
            # messes up the science data - ignores these cases
            sci_mean = np.nanmean(sci[valid])
            sci_std = np.nanstd(sci[valid])
            upper = sci_mean + sci_std * 5.0
            lower = sci_mean - sci_std * 5.0
            mask_clean = np.logical_and(sci[valid] < upper, sci[valid] > lower)

            # for this slit subtract the background subtracted data from
            # the science data with no background added
            sub = result_slit - sci
            # do not look at outliers - they confuse things
            sub_valid = sub[valid]
            mean_sub = np.mean(sub_valid[mask_clean])
            atol = 0.1
            assert_allclose(np.absolute(mean_sub), 0, atol=atol)
        # ______________________________________________________________________
        # Test 3 Compare background sutracted science data (results)
        #  to a truth file. This data is MultiSlit data

        result_file = result.meta.filename
        ref_file = self.get_data(*self.ref_loc,
                                 'nrs_mos_sci+bkg_masterbackgroundstep.fits')

        outputs = [(result_file, ref_file)]
        self.compare_outputs(outputs)
        input_sci.close()
        result.close()
Ejemplo n.º 24
0
    def test_miri_masterbackground_lrs_user1d(self):
        """
        Regression test of masterbackgound subtraction with lrs, with user provided 1-D background
        """

        # input file has the background added
        # Copy original version of file to test file, which will get overwritten by test
        input_file = self.get_data(*self.test_dir,
                                   'miri_lrs_sci+bkg_cal.fits')

        # user provided 1-D background
        input_1d_bkg_file = self.get_data(*self.test_dir,
                                         'miri_lrs_bkg_x1d.fits')
        input_1d_bkg_model = datamodels.open(input_1d_bkg_file)

        result = MasterBackgroundStep.call(input_file,
                                           user_background=input_1d_bkg_file,
                                           save_results=True)

        # Test 1
        # Run extract1D on the master background subtracted data (result)  and
        # the science data with no background added

        # run 1-D extract on results from MasterBackground step
        result_1d = Extract1dStep.call(result)

        # get the 1-D extracted Spectrum from the science data with
        # no background added to test against
        sci_cal_1d_file = self.get_data(*self.ref_loc,
                                    'miri_lrs_sci_extract1d.fits')
        sci_cal_1d = datamodels.open(sci_cal_1d_file)

        sci_cal_1d_data = sci_cal_1d.spec[0].spec_table['flux']
        result_1d_data = result_1d.spec[0].spec_table['flux']
        sci_wave = sci_cal_1d.spec[0].spec_table['wavelength']
        # find the valid wavelenth of the user provided spectrun
        user_wave = input_1d_bkg_model.spec[0].spec_table['wavelength']
        user_flux = input_1d_bkg_model.spec[0].spec_table['flux']
        user_wave_valid = np.where(user_flux > 0)
        min_user_wave = np.amin(user_wave[user_wave_valid])
        max_user_wave = np.amax(user_wave[user_wave_valid])
        input_1d_bkg_model.close()

        # find the waverange covered by both user and science                                          
        sci_wave_valid = np.where(sci_cal_1d_data > 0)
        min_wave = np.amin(sci_wave[sci_wave_valid])
        max_wave = np.amax(sci_wave[sci_wave_valid])
        if min_user_wave > min_wave:
            min_wave = min_user_wave
        if max_user_wave < max_wave:
            max_wave = max_user_wave

        # Compare the data
        sub_spec = sci_cal_1d_data - result_1d_data
        valid = np.where(np.logical_and(sci_wave > min_wave, sci_wave < max_wave))
        sub_spec = sub_spec[valid]
        mean_sub = np.absolute(np.nanmean(sub_spec))
        atol = 0.00005
        rtol = 0.000001
        assert_allclose(mean_sub, 0, atol=atol, rtol=rtol)

        # Test 2
        # Compare result (background subtracted image) to science image with no
        # background. Subtract these images, smooth the subtracted image and
        # the mean should be close to zero.
        input_sci_cal_file = self.get_data(*self.test_dir,
                                            'miri_lrs_sci_cal.fits')
        input_sci = datamodels.open(input_sci_cal_file)

        # find the LRS region
        bb = result.meta.wcs.bounding_box
        x, y = grid_from_bounding_box(bb)
        result_lrs_region = result.data[y.astype(int), x.astype(int)]
        sci_lrs_region = input_sci.data[y.astype(int), x.astype(int)]

        # do a 5 sigma clip on the science image
        sci_mean = np.nanmean(sci_lrs_region)
        sci_std = np.nanstd(sci_lrs_region)
        upper = sci_mean + sci_std*5.0
        lower = sci_mean - sci_std*5.0
        mask_clean = np.logical_and(sci_lrs_region < upper, sci_lrs_region > lower)

        sub = result_lrs_region - sci_lrs_region
        mean_sub = np.absolute(np.mean(sub[mask_clean]))

        atol = 0.5
        rtol = 0.001
        assert_allclose(mean_sub, 0, atol=atol, rtol=rtol)

        # Test 3 Compare background subtracted science data (results)
        #  to a truth file.
        truth_file = self.get_data(*self.ref_loc,
                                  'miri_lrs_sci+bkg_masterbackgroundstep.fits')

        result_file = result.meta.filename
        outputs = [(result_file, truth_file)]
        self.compare_outputs(outputs)
        result.close()
        input_sci.close()
Ejemplo n.º 25
0
    def test_nirspec_ifu_masterbg_user(self):
        """
        Regression test of master background subtraction for NRS IFU when a
        user 1-D spectrum is provided.
        """
        # input file has 2-D background image added to it
        input_file = self.get_data(*self.test_dir, 'prism_sci_bkg_cal.fits')

        # user-provided 1-D background was created from the 2-D background image
        user_background = self.get_data(*self.test_dir, 'prism_bkg_x1d.fits')

        result = MasterBackgroundStep.call(input_file,
                                           user_background=user_background,
                                           save_results=True)

        # Test 1 compare extracted spectra data with
        # no background added to extracted spectra from the output
        # from MasterBackground subtraction. First cube_build has to be run
        # on the data.
        result_s3d = CubeBuildStep.call(result)
        # run 1-D extract on results from MasterBackground step
        result_1d = Extract1dStep.call(result_s3d, subtract_background=False)

        # get the 1-D extracted spectrum from the science data in truth directory
        input_sci_1d_file = self.get_data(*self.ref_loc, 'prism_sci_extract1d.fits')
        sci_1d = datamodels.open(input_sci_1d_file)

        # read in the valid wavelengths of the user-1d
        user_background_model = datamodels.open(user_background)
        user_wave = user_background_model.spec[0].spec_table['wavelength']
        user_flux = user_background_model.spec[0].spec_table['net']
        user_wave_valid = np.where(user_flux > 0)
        min_user_wave = np.amin(user_wave[user_wave_valid])
        max_user_wave = np.amax(user_wave[user_wave_valid])
        user_background_model.close()
        # find the waverange covered by both user and science
        sci_spec_1d = sci_1d.spec[0].spec_table['net']
        sci_spec_wave = sci_1d.spec[0].spec_table['wavelength']

        result_spec_1d = result_1d.spec[0].spec_table['net']

        sci_wave_valid = np.where(sci_spec_1d > 0)
        min_wave = np.amin(sci_spec_wave[sci_wave_valid])
        max_wave = np.amax(sci_spec_wave[sci_wave_valid])
        if min_user_wave > min_wave:
            min_wave = min_user_wave
        if max_user_wave < max_wave:
            max_wave = max_user_wave

        sub_spec = sci_spec_1d - result_spec_1d
        valid = np.where(np.logical_and(sci_spec_wave > min_wave, sci_spec_wave < max_wave))
        sub_spec = sub_spec[valid]
        sub_spec = sub_spec[1:-2]  # endpoints are wacky

        mean_sub = np.absolute(np.nanmean(sub_spec))
        atol = 5.0
        assert_allclose(mean_sub, 0, atol=atol)

        # Test 2  compare the science  data with no background
        # to the output from the masterBackground Subtraction step
        # background subtracted science image.
        input_sci_cal_file = self.get_data(*self.test_dir,
                                            'prism_sci_cal.fits')
        input_sci_model = datamodels.open(input_sci_cal_file)

        # We don't want the slices gaps to impact the statisitic
        # loop over the 30 Slices
        for i in range(30):
            slice_wcs = nirspec.nrs_wcs_set_input(input_sci_model, i)
            x, y = grid_from_bounding_box(slice_wcs.bounding_box)
            ra, dec, lam = slice_wcs(x, y)
            valid = np.isfinite(lam)
            result_slice_region = result.data[y.astype(int), x.astype(int)]
            sci_slice_region = input_sci_model.data[y.astype(int),
                                                    x.astype(int)]
            sci_slice = sci_slice_region[valid]
            result_slice = result_slice_region[valid]
            sub = result_slice - sci_slice

            # check for outliers in the science image
            sci_mean = np.nanmean(sci_slice)
            sci_std = np.nanstd(sci_slice)
            upper = sci_mean + sci_std*5.0
            lower = sci_mean - sci_std*5.0
            mask_clean = np.logical_and(sci_slice < upper, sci_slice > lower)

            sub_mean = np.absolute(np.nanmean(sub[mask_clean]))
            atol = 2.0
            assert_allclose(sub_mean, 0, atol=atol)

        # Test 3 Compare background sutracted science data (results)
        #  to a truth file. This data is MultiSlit data

        input_sci_model.close()
        result_file = result.meta.filename
        truth_file = self.get_data(*self.ref_loc,
                                  'prism_sci_bkg_masterbackgroundstep.fits')

        outputs = [(result_file, truth_file)]
        self.compare_outputs(outputs)
        input_sci_model.close()
        result.close()