def test_nis_wfss_spec2(_bigdata):
"""
Regression test of calwebb_spec2 pipeline performed on NIRISS WFSS data.
"""
pipe = Spec2Pipeline()
pipe.save_bsub = True
pipe.save_results = True
pipe.resample_spec.save_results = True
pipe.extract_1d.save_results = True
pipe.run(_bigdata+'/pipelines/jw87600-a3001_20171109T145456_spec2_001_asn.json')
# Compare the _cal file
na = 'jw87600017001_02101_00002_nis_cal.fits'
nb = _bigdata+'/pipelines/jw87600017001_02101_00002_nis_cal_ref.fits'
h = fits.open(na)
href = fits.open(nb)
h = h[:-1]
href = href[:-1]
result = fits.diff.FITSDiff(h, href,
ignore_keywords = ['DATE','CAL_VER','CAL_VCS','CRDS_VER','CRDS_CTX'],
rtol = 0.00001)
assert result.identical, result.report()
# Compare the _x1d file
na = 'jw87600017001_02101_00002_nis_x1d.fits'
nb = _bigdata+'/pipelines/jw87600017001_02101_00002_nis_x1d_ref.fits'
h = fits.open(na)
href = fits.open(nb)
h = h[:-1]
href = href[:-1]
result = fits.diff.FITSDiff(h, href,
ignore_keywords = ['DATE','CAL_VER','CAL_VCS','CRDS_VER','CRDS_CTX'],
rtol = 0.00001)
assert result.identical, result.report()
def test_filenotfounderror_raised(capsys):
# Verify the failure is in the traceback message
with pytest.raises(RuntimeError, match="FileNotFoundError"):
Spec2Pipeline().run('file_does_not_exist.fits')
# Verify the failure is printed to stderr
captured = capsys.readouterr()
assert 'FileNotFoundError' in captured.err
def test_mrs_spec2(_bigdata):
"""
Regression test of calwebb_spec2 pipeline performed on MIRI MRS data.
"""
step = Spec2Pipeline()
step.save_bsub = True,
step.save_results = True
step.resample_spec.save_results = True
step.cube_build.save_results = True
step.extract_1d.save_results = True
step.run(_bigdata +
'/pipelines/jw10001001001_01101_00001_mirifushort_rate.fits')
na = 'jw10001001001_01101_00001_mirifushort_cal.fits'
nb = _bigdata + '/pipelines/jw10001001001_01101_00001_mirifushort_cal_ref.fits'
h = fits.open(na)
href = fits.open(nb)
newh = fits.HDUList(
[h['primary'], h['sci'], h['err'], h['dq'], h['relsens2d']])
newhref = fits.HDUList([
href['primary'], href['sci'], href['err'], href['dq'],
href['relsens2d']
])
result = fits.diff.FITSDiff(
newh,
newhref,
ignore_keywords=['DATE', 'CAL_VER', 'CAL_VCS', 'CRDS_VER', 'CRDS_CTX'],
rtol=0.000001)
assert result.identical, result.report()
na = 'jw10001001001_01101_00001_mirifushort_s3d.fits'
nb = _bigdata + '/pipelines/jw10001001001_01101_00001_mirifushort_s3d_ref.fits'
h = fits.open(na)
href = fits.open(nb)
newh = fits.HDUList([h['primary'], h['sci'], h['err'], h['dq'], h['wmap']])
newhref = fits.HDUList(
[href['primary'], href['sci'], href['err'], href['dq'], href['wmap']])
result = fits.diff.FITSDiff(
newh,
newhref,
ignore_keywords=['DATE', 'CAL_VER', 'CAL_VCS', 'CRDS_VER', 'CRDS_CTX'],
rtol=0.000001)
assert result.identical, result.report()
na = 'jw10001001001_01101_00001_mirifushort_x1d.fits'
nb = _bigdata + '/pipelines/jw10001001001_01101_00001_mirifushort_x1d_ref.fits'
h = fits.open(na)
href = fits.open(nb)
newh = fits.HDUList([h['primary'], h['extract1d', 1]])
newhref = fits.HDUList([href['primary'], href['extract1d', 1]])
result = fits.diff.FITSDiff(
newh,
newhref,
ignore_keywords=['DATE', 'CAL_VER', 'CAL_VCS', 'CRDS_VER', 'CRDS_CTX'],
rtol=0.000001)
assert result.identical, result.report()
def test_nrs_msa_spec2b(_bigdata):
"""
Regression test of calwebb_spec2 pipeline performed on NIRSpec MSA data,
including barshadow correction.
"""
input = os.path.join(_bigdata, 'pipelines',
'jw95065_nrs_msaspec_barshadow.fits')
step = Spec2Pipeline()
step.output_file = 'jw95065_nrs_msaspec_barshadow_cal.fits'
step.save_bsub = False
step.save_results = True
step.resample_spec.save_results = True
step.extract_1d.save_results = True
step.run(input)
ignore_keywords = ['DATE', 'CAL_VER', 'CAL_VCS', 'CRDS_VER', 'CRDS_CTX']
# compare _cal file
na = 'jw95065_nrs_msaspec_barshadow_cal.fits'
nb = os.path.join(_bigdata, 'pipelines',
'jw95065_nrs_msaspec_barshadow_cal_ref.fits')
h = fits.open(na)
href = fits.open(nb)
result = fits.diff.FITSDiff(h,
href,
ignore_hdus=['ASDF'],
ignore_keywords=ignore_keywords,
rtol=0.00001)
assert result.identical, result.report()
# compare _s2d file
na = 'jw95065_nrs_msaspec_barshadow_s2d.fits'
nb = os.path.join(_bigdata, 'pipelines',
'jw95065_nrs_msaspec_barshadow_s2d_ref.fits')
h = fits.open(na)
href = fits.open(nb)
result = fits.diff.FITSDiff(h,
href,
ignore_hdus=['ASDF'],
ignore_keywords=ignore_keywords,
rtol=0.00001)
assert result.identical, result.report()
# compare _x1d file
na = 'jw95065_nrs_msaspec_barshadow_x1d.fits'
nb = os.path.join(_bigdata, 'pipelines',
'jw95065_nrs_msaspec_barshadow_x1d_ref.fits')
h = fits.open(na)
href = fits.open(nb)
result = fits.diff.FITSDiff(h,
href,
ignore_hdus=['ASDF'],
ignore_keywords=ignore_keywords,
rtol=0.00001)
assert result.identical, result.report()
def test_nrs_msa_spec2(_bigdata):
"""
Regression test of calwebb_spec2 pipeline performed on NIRSpec MSA data.
"""
input = 'F170LP-G235M_MOS_observation-6-c0e0_001_DN_NRS1_mod.fits'
# define step for use in test
step = Spec2Pipeline()
step.save_bsub = False
step.output_use_model = True
step.resample_spec.save_results = True
step.extract_1d.save_results = True
step.extract_1d.smoothing_length = 0
step.extract_1d.bkg_order = 0
step.run(os.path.join(_bigdata, 'pipelines', input))
ignore_keywords = ['DATE','CAL_VER','CAL_VCS','CRDS_VER','CRDS_CTX']
# compare _cal files
output = 'F170LP-G235M_MOS_observation-6-c0e0_001_DN_NRS1_mod_cal.fits'
nbname = 'f170lp-g235m_mos_observation-6-c0e0_001_dn_nrs1_mod_cal_ref.fits'
nb = os.path.join(_bigdata,'pipelines', nbname)
h = fits.open(output)
href = fits.open(nb)
result = fits.diff.FITSDiff(h,
href,
ignore_hdus=['ASDF'],
ignore_keywords=ignore_keywords,
rtol = 0.00001)
assert result.identical, result.report()
# compare _s2d files
output = 'F170LP-G235M_MOS_observation-6-c0e0_001_DN_NRS1_mod_s2d.fits'
nbname = 'f170lp-g235m_mos_observation-6-c0e0_001_dn_nrs1_mod_s2d_ref.fits'
nb = os.path.join(_bigdata,'pipelines', nbname)
h = fits.open(output)
href = fits.open(nb)
result = fits.diff.FITSDiff(h,
href,
ignore_hdus=['ASDF'],
ignore_keywords=ignore_keywords,
rtol = 0.00001)
assert result.identical, result.report()
# compare _x1d files
output2 = 'F170LP-G235M_MOS_observation-6-c0e0_001_DN_NRS1_mod_x1d.fits'
nbname = 'f170lp-g235m_mos_observation-6-c0e0_001_dn_nrs1_mod_x1d_ref.fits'
nb = os.path.join(_bigdata, 'pipelines', nbname)
h = fits.open(output2)
href = fits.open(nb)
result = fits.diff.FITSDiff(h,
href,
ignore_hdus=['ASDF'],
ignore_keywords=ignore_keywords,
rtol = 0.00001)
assert result.identical, result.report()
def test_nis_wfss_spec2(self):
"""
Regression test of calwebb_spec2 pipeline performed on NIRISS WFSS data.
"""
# Collect data
asn_file = self.get_data(self.test_dir,
'jw87600-a3001_20171109T145456_spec2_001_asn.json')
for file in raw_from_asn(asn_file):
self.get_data(self.test_dir, file)
# Run the step
collect_pipeline_cfgs('cfgs')
Spec2Pipeline.call(asn_file, config_file='cfgs/calwebb_spec2.cfg', save_bsub=True)
# Test results.
outputs = [('jw87600017001_02101_00002_nis_cal.fits',
'jw87600017001_02101_00002_nis_cal_ref.fits'),
('jw87600017001_02101_00002_nis_x1d.fits',
'jw87600017001_02101_00002_nis_x1d_ref.fits')]
self.compare_outputs(outputs)
def test_miri_lrs_slit_1b(_bigdata):
"""
Regression test of calwebb_spec2 pipeline performed on a single
MIRI LRS fixed-slit exposure with multiple integrations. Compare _calints.
"""
step = Spec2Pipeline()
step.save_bsub = True,
step.save_results = True
step.extract_1d.save_results = True
step.run(_bigdata +
'/pipelines/jw00035001001_01101_00001_MIRIMAGE_rateints.fits')
n_cr = 'jw00035001001_01101_00001_MIRIMAGE_calints.fits'
n_ref = _bigdata + '/pipelines/jw00035001001_01101_00001_MIRIMAGE_calints_ref.fits'
h = fits.open(n_cr)
href = fits.open(n_ref)
newh = fits.HDUList(
[h['primary'], h['sci'], h['err'], h['dq'], h['relsens']])
newhref = fits.HDUList([
href['primary'], href['sci'], href['err'], href['dq'], href['relsens']
])
result = fits.diff.FITSDiff(
newh,
newhref,
ignore_keywords=['DATE', 'CAL_VER', 'CAL_VCS', 'CRDS_VER', 'CRDS_CTX'],
rtol=0.00001)
assert result.identical, result.report()
n_cr = 'jw00035001001_01101_00001_MIRIMAGE_x1dints.fits'
n_ref = _bigdata + '/pipelines/jw00035001001_01101_00001_MIRIMAGE_x1dints_ref.fits'
h = fits.open(n_cr)
href = fits.open(n_ref)
newh = fits.HDUList([
h['primary'], h['extract1d', 1], h['extract1d', 2], h['extract1d', 3],
h['extract1d', 4]
])
newhref = fits.HDUList([
href['primary'], href['extract1d', 1], href['extract1d', 2],
href['extract1d', 3], href['extract1d', 4]
])
result = fits.diff.FITSDiff(
newh,
newhref,
ignore_keywords=['DATE', 'CAL_VER', 'CAL_VCS', 'CRDS_VER', 'CRDS_CTX'],
rtol=0.00001)
assert result.identical, result.report()
def test_nrs_fs_multi_spec2_3(_bigdata):
"""
Regression test of calwebb_spec2 pipeline performed on NIRSpec fixed-slit data
using the ALLSLITS subarray and detector NRS2.
"""
step = Spec2Pipeline()
step.save_bsub = True
step.save_results = True
step.resample_spec.save_results = True
step.cube_build.save_results = True
step.extract_1d.save_results = True
step.run(_bigdata + '/pipelines/jw84600002001_02101_00001_nrs2_rate.fits')
ignore_keywords = ['DATE', 'CAL_VER', 'CAL_VCS', 'CRDS_VER', 'CRDS_CTX']
na = 'jw84600002001_02101_00001_nrs2_cal.fits'
nb = _bigdata + '/pipelines/jw84600002001_02101_00001_nrs2_cal_ref.fits'
h = pf.open(na)
href = pf.open(nb)
result = pf.diff.FITSDiff(h,
href,
ignore_hdus=['ASDF'],
ignore_keywords=ignore_keywords,
rtol=0.00001)
assert result.identical, result.report()
na = 'jw84600002001_02101_00001_nrs2_s2d.fits'
nb = _bigdata + '/pipelines/jw84600002001_02101_00001_nrs2_s2d_ref.fits'
h = pf.open(na)
href = pf.open(nb)
result = pf.diff.FITSDiff(h,
href,
ignore_hdus=['ASDF'],
ignore_keywords=ignore_keywords,
rtol=0.00001)
assert result.identical, result.report()
na = 'jw84600002001_02101_00001_nrs2_x1d.fits'
nb = _bigdata + '/pipelines/jw84600002001_02101_00001_nrs2_x1d_ref.fits'
h = pf.open(na)
href = pf.open(nb)
result = pf.diff.FITSDiff(h,
href,
ignore_hdus=['ASDF'],
ignore_keywords=ignore_keywords,
rtol=0.00001)
assert result.identical, result.report()
def test_nrs_fs_multi_spec2_2(_bigdata):
"""
Regression test of calwebb_spec2 pipeline performed on NIRSpec fixed-slit data.
"""
step = Spec2Pipeline()
step.save_bsub = True
step.save_results = True
step.resample_spec.save_results = True
step.cube_build.save_results = True
step.extract_1d.save_results = True
step.run(_bigdata + '/pipelines/jwtest1013001_01101_00001_NRS1_rate.fits')
ignore_keywords = ['DATE', 'CAL_VER', 'CAL_VCS', 'CRDS_VER', 'CRDS_CTX']
na = 'jwtest1013001_01101_00001_NRS1_cal.fits'
nb = _bigdata + '/pipelines/jwtest1013001_01101_00001_NRS1_cal_ref.fits'
h = fits.open(na)
href = fits.open(nb)
result = fits.diff.FITSDiff(h,
href,
ignore_hdus=['ASDF'],
ignore_keywords=ignore_keywords,
rtol=0.00001)
assert result.identical, result.report()
na = 'jwtest1013001_01101_00001_NRS1_s2d.fits'
nb = _bigdata + '/pipelines/jwtest1013001_01101_00001_NRS1_s2d_ref.fits'
h = fits.open(na)
href = fits.open(nb)
result = fits.diff.FITSDiff(h,
href,
ignore_hdus=['ASDF'],
ignore_keywords=ignore_keywords,
rtol=0.00001)
assert result.identical, result.report()
na = 'jwtest1013001_01101_00001_NRS1_x1d.fits'
nb = _bigdata + '/pipelines/jwtest1013001_01101_00001_NRS1_x1d_ref.fits'
h = fits.open(na)
href = fits.open(nb)
result = fits.diff.FITSDiff(h,
href,
ignore_hdus=['ASDF'],
ignore_keywords=ignore_keywords,
rtol=0.00001)
assert result.identical, result.report()
def test_mrs2pipeline1(_bigdata):
"""
Regression test of calwebb_spec2 pipeline performed on MIRI MRS data.
"""
step = Spec2Pipeline()
step.save_bsub=True,
step.save_results=True
step.resample_spec.save_results = True
step.cube_build.save_results = True
step.extract_1d.save_results = True
step.run(_bigdata+'/miri/test_mrs2pipeline/jw80500018001_02101_00002_MIRIFUSHORT_rate.fits')
n_h = 'jw80500018001_02101_00002_MIRIFUSHORT_cal.fits'
h = pf.open(n_h)
n_href = _bigdata+'/miri/test_mrs2pipeline/jw80500018001_02101_00002_MIRIFUSHORT_cal.fits'
href = pf.open(n_href)
newh = pf.HDUList([h['primary'],h['sci'],h['err'],h['dq']])
newhref = pf.HDUList([href['primary'],href['sci'],href['err'],href['dq']])
result = pf.diff.FITSDiff(newh, newhref,
ignore_keywords = ['DATE','CAL_VER','CAL_VCS','CRDS_VER','CRDS_CTX'],
rtol = 0.00001)
assert result.identical, result.report()
n_h = 'jw80500018001_02101_00002_MIRIFUSHORT_s3d.fits'
h = pf.open(n_h)
n_href = _bigdata+'/miri/test_mrs2pipeline/jw80500018001_02101_00002_MIRIFUSHORT_s3d.fits'
href = pf.open(n_href)
newh = pf.HDUList([h['primary'],h['sci'],h['err'],h['dq'],h['wmap']])
newhref = pf.HDUList([href['primary'],href['sci'],href['err'],href['dq'],href['wmap']])
result = pf.diff.FITSDiff(newh, newhref,
ignore_keywords = ['DATE','CAL_VER','CAL_VCS','CRDS_VER','CRDS_CTX'],
rtol = 0.00001)
assert result.identical, result.report()
n_h = 'jw80500018001_02101_00002_MIRIFUSHORT_x1d.fits'
h = pf.open(n_h)
n_href = _bigdata+'/miri/test_mrs2pipeline/jw80500018001_02101_00002_MIRIFUSHORT_x1d.fits'
href = pf.open(n_href)
newh = pf.HDUList([h['primary'],h['extract1d']])
newhref = pf.HDUList([href['primary'],href['extract1d']])
result = pf.diff.FITSDiff(newh, newhref,
ignore_keywords = ['DATE','CAL_VER','CAL_VCS','CRDS_VER','CRDS_CTX'],
rtol = 0.00001)
assert result.identical, result.report()
def test_nis_wfss_spec2(self):
"""
Regression test of calwebb_spec2 pipeline performed on NIRISS WFSS data.
"""
asn_file = self.get_data(self.test_dir,
'jw87600-a3001_20171109T145456_spec2_001_asn.json')
for file in raw_from_asn(asn_file):
self.get_data(self.test_dir, file)
pipe = Spec2Pipeline()
pipe.save_bsub = True
pipe.save_results = True
pipe.resample_spec.save_results = True
pipe.extract_1d.save_results = True
pipe.run(asn_file)
outputs = [('jw87600017001_02101_00002_nis_cal.fits',
'jw87600017001_02101_00002_nis_cal_ref.fits'),
('jw87600017001_02101_00002_nis_x1d.fits',
'jw87600017001_02101_00002_nis_x1d_ref.fits')]
self.compare_outputs(outputs)
def test_miri_lrs_bkgnod(_bigdata):
"""
Regression test of calwebb_spec2 pipeline performed on an association
of nodded MIRI LRS fixed-slit exposures.
"""
step = Spec2Pipeline()
step.save_bsub = True,
step.save_results = True
step.resample_spec.save_results = True
step.cube_build.save_results = True
step.extract_1d.save_results = True
step.run(_bigdata + '/pipelines/lrs_bkgnod_asn.json')
na = 'test_lrs1_bsub.fits'
nb = _bigdata + '/pipelines/test_lrs1_bsub_ref.fits'
h = fits.open(na)
href = fits.open(nb)
newh = fits.HDUList([h['primary'], h['sci'], h['err'], h['dq']])
newhref = fits.HDUList(
[href['primary'], href['sci'], href['err'], href['dq']])
result = fits.diff.FITSDiff(
newh,
newhref,
ignore_keywords=['DATE', 'CAL_VER', 'CAL_VCS', 'CRDS_VER', 'CRDS_CTX'],
rtol=0.00001)
assert result.identical, result.report()
na = 'test_lrs2_bsub.fits'
nb = _bigdata + '/pipelines/test_lrs2_bsub_ref.fits'
h = fits.open(na)
href = fits.open(nb)
newh = fits.HDUList([h['primary'], h['sci'], h['err'], h['dq']])
newhref = fits.HDUList(
[href['primary'], href['sci'], href['err'], href['dq']])
result = fits.diff.FITSDiff(
newh,
newhref,
ignore_keywords=['DATE', 'CAL_VER', 'CAL_VCS', 'CRDS_VER', 'CRDS_CTX'],
rtol=0.00001)
assert result.identical, result.report()
na = 'test_lrs3_bsub.fits'
nb = _bigdata + '/pipelines/test_lrs3_bsub_ref.fits'
h = fits.open(na)
href = fits.open(nb)
newh = fits.HDUList([h['primary'], h['sci'], h['err'], h['dq']])
newhref = fits.HDUList(
[href['primary'], href['sci'], href['err'], href['dq']])
result = fits.diff.FITSDiff(
newh,
newhref,
ignore_keywords=['DATE', 'CAL_VER', 'CAL_VCS', 'CRDS_VER', 'CRDS_CTX'],
rtol=0.00001)
assert result.identical, result.report()
na = 'test_lrs4_bsub.fits'
nb = _bigdata + '/pipelines/test_lrs4_bsub_ref.fits'
h = fits.open(na)
href = fits.open(nb)
newh = fits.HDUList([h['primary'], h['sci'], h['err'], h['dq']])
newhref = fits.HDUList(
[href['primary'], href['sci'], href['err'], href['dq']])
result = fits.diff.FITSDiff(
newh,
newhref,
ignore_keywords=['DATE', 'CAL_VER', 'CAL_VCS', 'CRDS_VER', 'CRDS_CTX'],
rtol=0.00001)
assert result.identical, result.report()
na = 'test_lrs1_cal.fits'
nb = _bigdata + '/pipelines/test_lrs1_cal_ref.fits'
h = fits.open(na)
href = fits.open(nb)
newh = fits.HDUList(
[h['primary'], h['sci'], h['err'], h['dq'], h['relsens']])
newhref = fits.HDUList([
href['primary'], href['sci'], href['err'], href['dq'], href['relsens']
])
result = fits.diff.FITSDiff(
newh,
newhref,
ignore_keywords=['DATE', 'CAL_VER', 'CAL_VCS', 'CRDS_VER', 'CRDS_CTX'],
rtol=0.00001)
assert result.identical, result.report()
na = 'test_lrs2_cal.fits'
nb = _bigdata + '/pipelines/test_lrs2_cal_ref.fits'
h = fits.open(na)
href = fits.open(nb)
newh = fits.HDUList(
[h['primary'], h['sci'], h['err'], h['dq'], h['relsens']])
newhref = fits.HDUList([
href['primary'], href['sci'], href['err'], href['dq'], href['relsens']
])
result = fits.diff.FITSDiff(
newh,
newhref,
ignore_keywords=['DATE', 'CAL_VER', 'CAL_VCS', 'CRDS_VER', 'CRDS_CTX'],
rtol=0.00001)
assert result.identical, result.report()
na = 'test_lrs3_cal.fits'
nb = _bigdata + '/pipelines/test_lrs3_cal_ref.fits'
h = fits.open(na)
href = fits.open(nb)
newh = fits.HDUList(
[h['primary'], h['sci'], h['err'], h['dq'], h['relsens']])
newhref = fits.HDUList([
href['primary'], href['sci'], href['err'], href['dq'], href['relsens']
])
result = fits.diff.FITSDiff(
newh,
newhref,
ignore_keywords=['DATE', 'CAL_VER', 'CAL_VCS', 'CRDS_VER', 'CRDS_CTX'],
rtol=0.00001)
assert result.identical, result.report()
na = 'test_lrs4_cal.fits'
nb = _bigdata + '/pipelines/test_lrs4_cal_ref.fits'
h = fits.open(na)
href = fits.open(nb)
newh = fits.HDUList(
[h['primary'], h['sci'], h['err'], h['dq'], h['relsens']])
newhref = fits.HDUList([
href['primary'], href['sci'], href['err'], href['dq'], href['relsens']
])
result = fits.diff.FITSDiff(
newh,
newhref,
ignore_keywords=['DATE', 'CAL_VER', 'CAL_VCS', 'CRDS_VER', 'CRDS_CTX'],
rtol=0.00001)
assert result.identical, result.report()
def test_nrs_ifu_spec2(_bigdata):
"""
Regression test of calwebb_spec2 pipeline performed on NIRSpec IFU data.
"""
pipe = Spec2Pipeline()
pipe.save_bsub = True
pipe.save_results = True
pipe.resample_spec.save_results = True
pipe.cube_build.save_results = True
pipe.extract_1d.save_results = True
pipe.run(_bigdata + '/pipelines/jw95175001001_02104_00001_nrs1_rate.fits')
na = 'jw95175001001_02104_00001_nrs1_cal.fits'
nb = _bigdata + '/pipelines/jw95175001001_02104_00001_nrs1_cal_ref.fits'
h = fits.open(na)
href = fits.open(nb)
newh = fits.HDUList([
h['primary'], h['sci'], h['err'], h['dq'], h['relsens2d'],
h['pathloss_pointsource'], h['wavelength_pointsource'],
h['pathloss_uniformsource'], h['wavelength_uniformsource']
])
newhref = fits.HDUList([
href['primary'], href['sci'], href['err'], href['dq'],
href['relsens2d'], href['pathloss_pointsource'],
href['wavelength_pointsource'], href['pathloss_uniformsource'],
href['wavelength_uniformsource']
])
result = fits.diff.FITSDiff(
newh,
newhref,
ignore_keywords=['DATE', 'CAL_VER', 'CAL_VCS', 'CRDS_VER', 'CRDS_CTX'],
rtol=0.00001)
assert result.identical, result.report()
na = 'jw95175001001_02104_00001_nrs1_s3d.fits'
nb = _bigdata + '/pipelines/jw95175001001_02104_00001_nrs1_s3d_ref.fits'
h = fits.open(na)
href = fits.open(nb)
newh = fits.HDUList([h['primary'], h['sci'], h['err'], h['dq'], h['wmap']])
newhref = fits.HDUList(
[href['primary'], href['sci'], href['err'], href['dq'], href['wmap']])
result = fits.diff.FITSDiff(
newh,
newhref,
ignore_keywords=['DATE', 'CAL_VER', 'CAL_VCS', 'CRDS_VER', 'CRDS_CTX'],
rtol=0.00001)
assert result.identical, result.report()
na = 'jw95175001001_02104_00001_nrs1_x1d.fits'
nb = _bigdata + '/pipelines/jw95175001001_02104_00001_nrs1_x1d_ref.fits'
h = fits.open(na)
href = fits.open(nb)
newh = fits.HDUList([h['primary'], h['extract1d']])
newhref = fits.HDUList([href['primary'], href['extract1d']])
result = fits.diff.FITSDiff(
newh,
newhref,
ignore_keywords=['DATE', 'CAL_VER', 'CAL_VCS', 'CRDS_VER', 'CRDS_CTX'],
rtol=0.00001)
assert result.identical, result.report()
def output_hdul(set_inandout_filenames, config):
set_inandout_filenames_info = core_utils.read_info4outputhdul(
config, set_inandout_filenames)
step, txt_name, step_input_file, step_output_file, run_calwebb_spec2, outstep_file_suffix = set_inandout_filenames_info
# start the timer to compute the step running time of PTT
PTT_start_time = time.time()
# check if the filter is to be changed
change_filter_opaque = config.getboolean("calwebb_spec2_input_file",
"change_filter_opaque")
if change_filter_opaque:
_, step_input_file = change_filter_opaque2science.change_filter_opaque(
step_input_file)
change_filter_opaque_msg = " * With FILTER=OPAQUE, the calwebb_spec2 will run up to the extract_2d step. Further steps will be skipped. \n"
print(change_filter_opaque_msg)
# determine if the pipeline is to be run in full
run_calwebb_spec2 = config.getboolean("run_calwebb_spec2_in_full",
"run_calwebb_spec2")
# determine which steps are to be run, if not run in full
run_pipe_step = config.getboolean("run_pipe_steps", step)
# determine which tests are to be run
assign_wcs_completion_tests = config.getboolean(
"run_pytest", "_".join((step, "completion", "tests")))
assign_wcs_reffile_tests = config.getboolean(
"run_pytest", "_".join((step, "reffile", "tests")))
assign_wcs_validation_tests = config.getboolean(
"run_pytest", "_".join((step, "validation", "tests")))
run_pytests = [
assign_wcs_completion_tests, assign_wcs_reffile_tests,
assign_wcs_validation_tests
]
# get other relevant info from PTT config file
esa_files_path = config.get("esa_intermediary_products", "esa_files_path")
wcs_threshold_diff = config.get("additional_arguments",
"wcs_threshold_diff")
save_wcs_plots = config.getboolean("additional_arguments",
"save_wcs_plots")
output_directory = config.get("calwebb_spec2_input_file",
"output_directory")
# Get the detector used
detector = fits.getval(step_input_file, "DETECTOR", 0)
# get main header from input file
inhdu = core_utils.read_hdrfits(step_input_file,
info=False,
show_hdr=False)
# if run_calwebb_spec2 is True calwebb_spec2 will be called, else individual steps will be ran
step_completed = False
end_time = '0.0'
# Check if data is IFU that the Image Model keyword is correct
mode_used = config.get("calwebb_spec2_input_file", "mode_used").lower()
if mode_used == "ifu":
DATAMODL = fits.getval(step_input_file, "DATAMODL", 0)
if DATAMODL != "IFUImageModel":
fits.setval(step_input_file, "DATAMODL", 0, value="IFUImageModel")
print("DATAMODL keyword changed to IFUImageModel.")
# get the shutter configuration file for MOS data only
msa_shutter_conf = "No shutter configuration file will be used."
if core_utils.check_MOS_true(inhdu):
msa_shutter_conf = config.get("esa_intermediary_products",
"msa_conf_name")
# check if the configuration shutter file name is in the header of the fits file and if not add it
msametfl = fits.getval(step_input_file, "MSAMETFL", 0)
if os.path.basename(msa_shutter_conf) != msametfl:
msametfl = os.path.basename(msa_shutter_conf)
fits.setval(step_input_file, "MSAMETFL", 0, value=msametfl)
# check if processing an image, then set proper variables
imaging_mode = False
if mode_used in ('image', 'confirm', 'taconfirm', 'wata', 'msata', 'bota',
'focus', 'mimf'):
run_calwebb_spec2 = True
imaging_mode = True
print(
'\n * Image processing will only be run in full with PTT. All intermediary products will be saved.'
)
print(
' -> For now, all pytests will be skipped since there are now image-specific routines yet. \n'
)
# TODO: add imaging tests
# get the name of the configuration file and run the pipeline
calwebb_spec2_cfg = config.get("run_calwebb_spec2_in_full",
"calwebb_spec2_cfg")
# copy the configuration file to create the pipeline log
stpipelogcfg = calwebb_spec2_cfg.replace("calwebb_spec2.cfg",
"stpipe-log.cfg")
subprocess.run(["cp", stpipelogcfg, os.getcwd()])
# run the pipeline
if run_calwebb_spec2:
# Create the logfile for PTT, but remove the previous log file
PTTcalspec2_log = os.path.join(output_directory,
'PTT_calspec2_' + detector + '.log')
if imaging_mode:
PTTcalspec2_log = PTTcalspec2_log.replace('calspec2', 'calimage2')
if os.path.isfile(PTTcalspec2_log):
os.remove(PTTcalspec2_log)
print(
"Information outputed to screen from PTT will be logged in file: ",
PTTcalspec2_log)
for handler in logging.root.handlers[:]:
logging.root.removeHandler(handler)
logging.basicConfig(filename=PTTcalspec2_log, level=logging.INFO)
logging.info(pipeline_version)
if change_filter_opaque:
logging.info(change_filter_opaque_msg)
run_calwebb_spec2_msg = " *** Will run pipeline in full ... "
print(run_calwebb_spec2_msg)
logging.info(run_calwebb_spec2_msg)
# create the map
txt_name = "full_run_map_" + detector + ".txt"
if os.path.isfile(txt_name):
os.remove(txt_name)
assign_wcs_utils.create_completed_steps_txtfile(
txt_name, step_input_file)
# start the timer to compute the step running time of PTT
core_utils.start_end_PTT_time(txt_name,
start_time=PTT_start_time,
end_time=None)
if mode_used == "bots":
calwebb_spec2_cfg = calwebb_spec2_cfg.replace(
"calwebb_spec2.cfg", "calwebb_tso-spec2.cfg")
print(
'\nUsing the following configuration file to run TSO pipeline:'
)
print(calwebb_spec2_cfg, '\n')
if imaging_mode:
calwebb_image2_cfg = calwebb_spec2_cfg.replace(
"calwebb_spec2.cfg", "calwebb_image2.cfg")
print(
'\nUsing the following configuration file to run imaging pipeline:'
)
print(calwebb_image2_cfg, '\n')
else:
print(
'\nUsing the following configuration file to run spectroscopic pipeline:'
)
print(calwebb_spec2_cfg, '\n')
input_file = config.get("calwebb_spec2_input_file", "input_file")
if "_uncal_rate" in input_file:
input_file = input_file.replace("_uncal_rate", "")
if "_uncal" in input_file:
input_file = input_file.replace("_uncal", "")
final_output_name = input_file.replace(".fits", "_cal.fits")
if core_utils.check_MOS_true(inhdu):
# copy the MSA shutter configuration file into the pytest directory
subprocess.run(["cp", msa_shutter_conf, "."])
# start the timer to compute the step running time
start_time = time.time()
# run the pipeline
print('Running pipeline... \n')
if not imaging_mode:
Spec2Pipeline.call(
step_input_file,
config_file=calwebb_spec2_cfg) #, logcfg=stpipelogcfg)
else:
Image2Pipeline.call(step_input_file,
config_file=calwebb_image2_cfg)
subprocess.run(["rm", "stpipe-log.cfg"])
# For the moment, the pipeline is using the wrong reference file for slit 400A1, so the
# file needs to be re-processed with the right reference file
if core_utils.check_FS_true(inhdu):
print(
"\n * WARNING: For the moment, the wrong reference file is being used for "
"processing slit 400A1. The file will be re-processed ")
# print(" $ jwst.pathloss.PathLossStep final_output_caldet1_NRS1_srctype.fits "
# "--override_pathloss=jwst-nirspec-a400.plrf.fits \n")
pathloss_400a1 = step_input_file.replace("srctype.fits",
"pathloss_400A1.fits")
reffile_400a1 = "jwst-nirspec-a400.plrf.fits"
print("Re-processing slit with new reference file: ",
reffile_400a1)
pl = PathLossStep()
pl.override_pathloss = reffile_400a1
pl.run(step_input_file)
subprocess.run([
"mv",
step_input_file.replace("srctype", "pathlossstep"),
pathloss_400a1
])
print("Saved pipeline re-processed file as: ", pathloss_400a1)
# end the timer to compute calwebb_spec2 running time
end_time = repr(time.time() - start_time) # this is in seconds
calspec2_time = " * Pipeline took " + end_time + " seconds to finish.\n"
print(calspec2_time)
logging.info(calspec2_time)
# remove the copy of the MSA shutter configuration file
if core_utils.check_MOS_true(inhdu):
subprocess.run(["rm", msametfl])
# add the detector string to the name of the files and move them to the working directory
core_utils.add_detector2filename(output_directory, step_input_file)
final_output_name_msg = "\nThe final pipeline product was saved in: " + final_output_name
print(final_output_name_msg)
logging.info(final_output_name_msg)
# read the assign wcs fits file
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
# scihdul = core_utils.read_hdrfits(step_output_file, info=False, show_hdr=False, ext=1)
# rename and move the pipeline log file
try:
calspec2_pipelog = "calspec2_pipeline_" + detector + ".log"
if imaging_mode:
calspec2_pipelog = calspec2_pipelog.replace(
'calspec2', 'calimage2')
path_where_pipeline_was_run = os.getcwd()
logfile = glob(path_where_pipeline_was_run + "/pipeline.log")[0]
print(logfile)
os.rename(logfile, os.path.join(output_directory,
calspec2_pipelog))
except IndexError:
print(
"\nWARNING: Something went wrong. Could not find a pipeline.log file \n"
)
# make sure we are able to find calspec2_pipelog either in the calwebb_spec2 directory or in the working dir
if not os.path.isfile(calspec2_pipelog):
calspec2_pipelog = os.path.join(output_directory, calspec2_pipelog)
# add the running time for all steps
step_running_times = core_utils.calculate_step_run_time(
calspec2_pipelog)
end_time_list = []
for stp in core_utils.step_string_dict:
if stp in step_running_times:
step_completed = True
step_time = step_running_times[stp]["run_time"]
out_suffix = core_utils.step_string_dict[stp]["suffix"]
core_utils.add_completed_steps(txt_name, stp, out_suffix,
step_completed, step_time)
end_time_list.append(step_time)
# print total running time in the text file and move it to the indicated directory
string2print = "pipeline_total_time"
if float(end_time) <= sum(end_time_list):
tot_time = repr(sum(end_time_list))
else:
tot_time = end_time
assign_wcs_utils.print_time2file(txt_name, tot_time, string2print)
PTT_runtimes_msg = "Pipeline and PTT run times written in file: " + os.path.basename(
txt_name) + " in working directory. \n"
print(PTT_runtimes_msg)
logging.info(PTT_runtimes_msg)
# move the final reporting text files to the working directory
core_utils.move_txt_files_2workdir(config, detector)
# end script for imaging case
if imaging_mode:
print('\nPTT finished processing imaging mode. \n')
pytest.exit(
"Skipping pytests for now because they need to be written for imaging mode."
)
return hdul, step_output_file, msa_shutter_conf, esa_files_path, wcs_threshold_diff, save_wcs_plots, run_pytests, mode_used
else:
# create the map but remove a previous one if it exists
if os.path.isfile(txt_name):
os.remove(txt_name)
assign_wcs_utils.create_completed_steps_txtfile(
txt_name, step_input_file)
# start the timer to compute the step running time of PTT
core_utils.start_end_PTT_time(txt_name,
start_time=PTT_start_time,
end_time=None)
msg = "\n Pipeline and PTT run times will be written in file: " + os.path.basename(
txt_name) + " in working directory. \n"
print(msg)
logging.info(msg)
if run_pipe_step:
# Create the logfile for PTT, but erase the previous one if it exists
PTTcalspec2_log = os.path.join(
output_directory,
'PTT_calspec2_' + detector + '_' + step + '.log')
if imaging_mode:
PTTcalspec2_log = PTTcalspec2_log.replace(
'calspec2', 'calimage2')
if os.path.isfile(PTTcalspec2_log):
os.remove(PTTcalspec2_log)
print(
"Information outputed to screen from PTT will be logged in file: ",
PTTcalspec2_log)
for handler in logging.root.handlers[:]:
logging.root.removeHandler(handler)
logging.basicConfig(filename=PTTcalspec2_log, level=logging.INFO)
logging.info(pipeline_version)
if change_filter_opaque:
logging.info(change_filter_opaque_msg)
# check that previous pipeline steps were run up to this point
core_utils.check_completed_steps(step, step_input_file)
if os.path.isfile(step_input_file):
msg = " *** Step " + step + " set to True"
print(msg)
logging.info(msg)
stp = AssignWcsStep()
if core_utils.check_MOS_true(inhdu):
# copy the MSA shutter configuration file into the pytest directory
subprocess.run(["cp", msa_shutter_conf, "."])
# get the right configuration files to run the step
local_pipe_cfg_path = config.get("calwebb_spec2_input_file",
"local_pipe_cfg_path")
# start the timer to compute the step running time
print("running pipeline...")
start_time = time.time()
if local_pipe_cfg_path == "pipe_source_tree_code":
result = stp.call(step_input_file)
else:
result = stp.call(step_input_file,
config_file=local_pipe_cfg_path +
'/assign_wcs.cfg')
result.save(step_output_file)
# end the timer to compute the step running time
end_time = repr(time.time() - start_time) # this is in seconds
msg = "Step " + step + " took " + end_time + " seconds to finish"
print(msg)
logging.info(msg)
if core_utils.check_MOS_true(inhdu):
# remove the copy of the MSA shutter configuration file
subprocess.run(["rm", msametfl])
# rename and move the pipeline log file
try:
calspec2_pipelog = "calspec2_pipeline_" + step + "_" + detector + ".log"
if imaging_mode:
calspec2_pipelog = calspec2_pipelog.replace(
'calspec2', 'calimage2')
pytest_workdir = TESTSDIR
logfile = glob(pytest_workdir + "/pipeline.log")[0]
os.rename(logfile,
os.path.join(output_directory, calspec2_pipelog))
except IndexError:
print(
"\n* WARNING: Something went wrong. Could not find a pipeline.log file \n"
)
else:
msg = "Skipping step. Input file " + step_input_file + " does not exit."
print(msg)
logging.info(msg)
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed, end_time)
pytest.skip("Skipping " + step +
" because the input file does not exist.")
else:
print("Skipping running pipeline step ", step)
# add the running time for this step
end_time = core_utils.get_stp_run_time_from_screenfile(
step, detector, output_directory)
if os.path.isfile(step_output_file):
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
step_completed = True
# add the running time for this step
core_utils.add_completed_steps(txt_name, step, outstep_file_suffix,
step_completed, end_time)
return hdul, step_output_file, msa_shutter_conf, esa_files_path, wcs_threshold_diff, save_wcs_plots, run_pytests, mode_used
else:
step_completed = False
# add the running time for this step
core_utils.add_completed_steps(txt_name, step, outstep_file_suffix,
step_completed, end_time)
pytest.skip("Test skipped because input file " + step_output_file +
" does not exist.")
def output_hdul(set_inandout_filenames, config):
set_inandout_filenames_info = core_utils.read_info4outputhdul(config, set_inandout_filenames)
step, txt_name, step_input_file, step_output_file, run_calwebb_spec2, outstep_file_suffix = set_inandout_filenames_info
stp = AssignWcsStep()
run_calwebb_spec2 = config.getboolean("run_calwebb_spec2_in_full", "run_calwebb_spec2")
skip_runing_pipe_step = config.getboolean("tests_only", "_".join((step, "tests")))
esa_files_path = config.get("esa_intermediary_products", "esa_files_path")
wcs_threshold_diff = config.get("additional_arguments", "wcs_threshold_diff")
save_wcs_plots = config.getboolean("additional_arguments", "save_wcs_plots")
# if run_calwebb_spec2 is True calwebb_spec2 will be called, else individual steps will be ran
step_completed = False
end_time = '0.0'
# get the MSA shutter configuration file full path only for MOS data
inhdu = core_utils.read_hdrfits(step_input_file, info=False, show_hdr=False)
if core_utils.check_MOS_true(inhdu):
msa_shutter_conf = config.get("esa_intermediary_products", "msa_conf_name")
# check if the configuration shutter file name is in the header of the fits file and if not add it
msametfl = fits.getval(step_input_file, "MSAMETFL", 0)
if os.path.basename(msa_shutter_conf) != msametfl:
msametfl = os.path.basename(msa_shutter_conf)
fits.setval(step_input_file, "MSAMETFL", 0, value=msametfl)
# run the pipeline
if run_calwebb_spec2:
print ("*** Will run calwebb_spec2... ")
# create the map
full_run_map = "full_run_map.txt"
assign_wcs_utils.create_map_from_full_run(full_run_map, step_input_file)
# get the name of the configuration file and run the pipeline
calwebb_spec2_cfg = config.get("run_calwebb_spec2_in_full", "calwebb_spec2_cfg")
input_file_basename_list = step_input_file.split("_")[:4]
input_file_basename = "_".join((input_file_basename_list))
final_output_name = "_".join((input_file_basename, "cal.fits"))
final_output_name_basename = os.path.basename(final_output_name)
if core_utils.check_MOS_true(inhdu):
# copy the MSA shutter configuration file into the pytest directory
subprocess.run(["cp", msa_shutter_conf, "."])
# start the timer to compute the step running time
start_time = time.time()
Spec2Pipeline.call(step_input_file, config_file=calwebb_spec2_cfg)
# end the timer to compute calwebb_spec2 running time
end_time = repr(time.time() - start_time) # this is in seconds
print(" * calwebb_spec2 took "+end_time+" seconds to finish.")
if core_utils.check_MOS_true(inhdu):
# remove the copy of the MSA shutter configuration file
subprocess.run(["rm", msametfl])
# move the output file into the working directory
print ("The final calwebb_spec2 product was saved in: ", final_output_name)
subprocess.run(["mv", final_output_name_basename, final_output_name])
# read the assign wcs fits file
step_output_file = core_utils.read_completion_to_full_run_map(full_run_map, step)
hdul = core_utils.read_hdrfits(step_output_file, info=True, show_hdr=True)
scihdul = core_utils.read_hdrfits(step_output_file, info=False, show_hdr=False, ext=1)
return hdul, scihdul, step_output_file, esa_files_path, wcs_threshold_diff, save_wcs_plots
else:
# create the Map of file names
assign_wcs_utils.create_completed_steps_txtfile(txt_name, step_input_file)
if config.getboolean("steps", step):
print ("*** Step "+step+" set to True")
if os.path.isfile(step_input_file):
if not skip_runing_pipe_step:
if core_utils.check_MOS_true(inhdu):
# copy the MSA shutter configuration file into the pytest directory
subprocess.run(["cp", msa_shutter_conf, "."])
# get the right configuration files to run the step
local_pipe_cfg_path = config.get("calwebb_spec2_input_file", "local_pipe_cfg_path")
# start the timer to compute the step running time
start_time = time.time()
if local_pipe_cfg_path == "pipe_source_tree_code":
result = stp.call(step_input_file)
else:
result = stp.call(step_input_file, config_file=local_pipe_cfg_path+'/assign_wcs.cfg')
result.save(step_output_file)
# end the timer to compute the step running time
end_time = repr(time.time() - start_time) # this is in seconds
print("Step "+step+" took "+end_time+" seconds to finish")
if core_utils.check_MOS_true(inhdu):
# remove the copy of the MSA shutter configuration file
subprocess.run(["rm", msametfl])
step_completed = True
core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time)
hdul = core_utils.read_hdrfits(step_output_file, info=False, show_hdr=False, ext=0)
scihdul = core_utils.read_hdrfits(step_output_file, info=False, show_hdr=False, ext=1)
return hdul, scihdul, step_output_file, esa_files_path, wcs_threshold_diff, save_wcs_plots
else:
print("Skipping step. Intput file "+step_input_file+" does not exit.")
core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time)
pytest.skip("Skipping "+step+" because the input file does not exist.")
else:
core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time)
pytest.skip("Skipping "+step+". Step set to False in configuration file.")