def test_master_background_userbg(_jail, user_background):
"""Verify data can run through the step with a user-supplied background"""
image = datamodels.ImageModel((10, 10))
image.meta.instrument.name = 'MIRI'
image.meta.instrument.detector = 'MIRIMAGE'
image.meta.exposure.type = 'MIR_LRS-FIXEDSLIT'
image.meta.observation.date = '2018-01-01'
image.meta.observation.time = '00:00:00'
image.meta.subarray.xstart = 1
image.meta.subarray.ystart = 1
image.meta.wcsinfo.v2_ref = 0
image.meta.wcsinfo.v3_ref = 0
image.meta.wcsinfo.roll_ref = 0
image.meta.wcsinfo.ra_ref = 0
image.meta.wcsinfo.dec_ref = 0
image = AssignWcsStep.call(image)
# Run with a user-supplied background and verify this is recorded in header
result = MasterBackgroundStep.call(image, user_background=user_background)
collect_pipeline_cfgs('./config')
result = MasterBackgroundStep.call(
image,
config_file='config/master_background.cfg',
user_background=user_background,
)
# For inputs that are not files, the following should be true
assert type(image) is type(result)
assert result is not image
assert result.meta.cal_step.master_background == 'COMPLETE'
assert result.meta.background.master_background_file == 'user_background.fits'
def test_master_background_userbg(_jail, user_background):
"""Verify data can run through the step with a user-supplied background"""
image = datamodels.ImageModel((10, 10))
image.meta.instrument.name = 'MIRI'
image.meta.instrument.detector = 'MIRIMAGE'
image.meta.exposure.type = 'MIR_LRS-FIXEDSLIT'
image.meta.observation.date = '2018-01-01'
image.meta.observation.time = '00:00:00'
image.meta.subarray.xstart = 1
image.meta.subarray.ystart = 1
image.meta.wcsinfo.v2_ref = 0
image.meta.wcsinfo.v3_ref = 0
image.meta.wcsinfo.roll_ref = 0
image.meta.wcsinfo.ra_ref = 0
image.meta.wcsinfo.dec_ref = 0
image = AssignWcsStep.call(image)
# Run with a user-supplied background and verify this is recorded in header
result = MasterBackgroundStep.call(image, user_background=user_background)
collect_pipeline_cfgs('./config')
result = MasterBackgroundStep.call(
image,
config_file='config/master_background.cfg',
user_background=user_background,
)
# For inputs that are not files, the following should be true
assert type(image) is type(result)
assert result is not image
assert result.meta.cal_step.master_background == 'COMPLETE'
assert result.meta.background.master_background_file == 'user_background.fits'
def run_pipeline(jail, rtdata_module):
"""Run the calwebb_spec2 pipeline on a single NIRSpec MOS exposure."""
rtdata = rtdata_module
# Get the cfg files
collect_pipeline_cfgs("config")
# Get the MSA metadata file referenced in the input exposure
rtdata.get_data("nirspec/mos/jw95065006001_0_short_msa.fits")
# Get the input exposure
rtdata.get_data(
"nirspec/mos/f170lp-g235m_mos_observation-6-c0e0_001_dn_nrs1_mod.fits")
# Run the calwebb_spec2 pipeline; save results from intermediate steps
args = [
"config/calwebb_spec2.cfg", rtdata.input,
"--steps.assign_wcs.save_results=true",
"--steps.msa_flagging.save_results=true",
"--steps.extract_2d.save_results=true",
"--steps.flat_field.save_results=true",
"--steps.srctype.save_results=true",
"--steps.pathloss.save_results=true",
"--steps.barshadow.save_results=true"
]
Step.from_cmdline(args)
return rtdata
def test_image3_pipeline2(self):
"""Regression test definitions for CALIMAGE3 pipeline.
Regression test of calwebb_image3 pipeline on NIRCam
simulated long-wave data with a 6-point dither.
"""
asn_file = self.get_data(self.test_dir,
"jw10002-o001_20171116t191235_image3_002_asn.json")
for file in raw_from_asn(asn_file):
self.get_data(self.test_dir, file)
collect_pipeline_cfgs('config')
args = [
'config/calwebb_image3.cfg',
asn_file,
'--steps.tweakreg.kernel_fwhm=2',
'--steps.tweakreg.snr_threshold=5',
'--steps.tweakreg.enforce_user_order=True',
'--steps.tweakreg.searchrad=10',
'--steps.tweakreg.fitgeometry=rscale'
]
Step.from_cmdline(args)
outputs = [('jw10002001001_01101_00004_nrcblong_o001_crf.fits',
'jw10002001001_01101_00004_nrcblong_o001_crf_ref.fits'),
{'files':('jw10002-o001_t002_nircam_f444w_i2d.fits',
'jw10002-o001_t002_nircam_f444w_i2d_ref.fits'),
'pars':{'ignore_keywords':self.ignore_keywords+['NAXIS1','TFORM*'],
'ignore_fields':self.ignore_keywords,
'rtol':0.0001}
}
]
self.compare_outputs(outputs)
def test_run_msaflagging(self, caplog):
"""Test msa flagging operation"""
# Retrieve the data.
collect_pipeline_cfgs('cfgs')
self.get_data(*self.test_dir, 'jw95065006001_0_msa_twoslit.fits')
asn_path = self.get_data(*self.test_dir,
'mos_udf_g235m_twoslit_spec2_asn.json')
with open(asn_path) as fp:
asn = load_asn(fp)
for product in asn['products']:
for member in product['members']:
self.get_data(*self.test_dir, 'level2a_twoslit',
member['expname'])
# Run step.
args = [
op.join('cfgs', 'calwebb_spec2.cfg'), asn_path,
'--steps.msa_flagging.skip=false'
]
Step.from_cmdline(args)
# Test.
assert 'Step msa_flagging running with args' in caplog.text
assert 'Step msa_flagging done' in caplog.text
for product in asn['products']:
prod_name = product['name'] + '_cal.fits'
assert op.isfile(prod_name)
def test_nirspec_masterbg_nodded(self):
"""Run masterbackground step on NIRSpec association"""
asn_file = self.get_data(*self.test_dir, 'nirspec_spec3_asn.json')
for file in raw_from_asn(asn_file):
self.get_data(*self.test_dir, file)
collect_pipeline_cfgs('./config')
result = MasterBackgroundStep.call(
asn_file,
config_file='config/master_background.cfg',
save_background=True,
save_results=True)
# test 1
# compare background subtracted data to truth files
# check that the cal_step master_background ran to complete
outputs = []
for model in result:
assert model.meta.cal_step.master_background == 'COMPLETE'
result_file = model.meta.filename.replace('cal',
'master_background')
truth_file = self.get_data(*self.ref_loc, result_file)
outputs.append((result_file, truth_file))
self.compare_outputs(outputs)
# test 2
# compare the master background combined file to truth file
master_combined_bkg_file = 'ifu_prism_source_off_fix_NRS1_o001_masterbg.fits'
truth_background = self.get_data(*self.ref_loc,
master_combined_bkg_file)
outputs = [(master_combined_bkg_file, truth_background)]
self.compare_outputs(outputs)
def test_wfs_image2(self):
"""
Regression test of the WFS&C `calwebb_wfs-image2.cfg` pipeline
"""
data_base = 'jw82600026001_02101_00001_nrca1_rate'
ext = '.fits'
input_name = '{}{}'.format(data_base, ext)
input_file = self.get_data(self.test_dir, 'sdp_jw82600_wfs', 'level2a',
input_name)
collect_pipeline_cfgs('cfgs')
Step.from_cmdline([os.path.join('cfgs', 'calwebb_wfs-image2.cfg'), input_file])
cal_name = input_name.replace('rate', 'cal')
output_name = input_name.replace('rate','cal_ref')
outputs = [(cal_name, output_name)]
self.compare_outputs(outputs)
output_files = glob.glob('*')
output_files.remove('cfgs')
# these would happen when docopy=True
if input_name in output_files:
output_files.remove(input_name)
if output_name in output_files:
output_files.remove(output_name)
assert cal_name in output_files
output_files.remove(cal_name)
assert not output_files, 'Unexpected output files {}'.format(output_files)
def test_mirilrs2pipeline1(self):
"""
Regression test of calwebb_spec2 pipeline performed on
MIRI LRS slitless data.
"""
input_file = self.get_data(
'test_spec2pipeline',
'jw80600012001_02101_00003_mirimage_rateints.fits')
collect_pipeline_cfgs()
args = [
'calwebb_tso-spec2.cfg',
input_file,
]
Step.from_cmdline(args)
outputs = [('jw80600012001_02101_00003_mirimage_calints.fits',
'jw80600012001_02101_00003_mirimage_calints_ref.fits',
['primary', 'sci', 'err', 'dq', 'relsens']),
('jw80600012001_02101_00003_mirimage_x1dints.fits',
'jw80600012001_02101_00003_mirimage_x1dints_ref.fits', [
'primary', ('extract1d', 1), ('extract1d', 2),
('extract1d', 3), ('extract1d', 4)
])]
self.compare_outputs(outputs)
def test_master_background_logic(_jail, user_background, science_image):
"""Verify if calspec 2 background step was run the master background step will be skipped"""
# the background step in calspec2 was done
science_image.meta.cal_step.back_sub = 'COMPLETE'
# Run with a user-supplied background
collect_pipeline_cfgs('./config')
result = MasterBackgroundStep.call(
science_image,
config_file='config/master_background.cfg',
user_background=user_background,
)
assert result.meta.cal_step.master_background == 'SKIPPED'
assert type(science_image) is type(result)
# Now force it
result = MasterBackgroundStep.call(
science_image,
config_file='config/master_background.cfg',
user_background=user_background,
force_subtract=True)
assert result.meta.cal_step.master_background == 'COMPLETE'
assert type(science_image) is type(result)
def run_pipeline(jail, rtdata_module, request):
"""Run the calwebb_spec2 pipeline on NIRSpec Fixed-Slit exposures.
We currently test the following types of inputs:
1) Full-frame exposure (all slits will be extracted)
2) ALLSLITS subarray exposure (all slits will be extracted)
3) S400A1 subarray exposure (1 slit extracted)"""
rtdata = rtdata_module
# Get the cfg files
collect_pipeline_cfgs("config")
# Get the input exposure
rtdata.get_data('nirspec/fs/' + request.param + 'rate.fits')
# Run the calwebb_spec2 pipeline; save results from intermediate steps
args = [
"config/calwebb_spec2.cfg", rtdata.input,
"--steps.assign_wcs.save_results=true",
"--steps.extract_2d.save_results=true",
"--steps.wavecorr.save_results=true",
"--steps.srctype.save_results=true",
"--steps.flat_field.save_results=true",
"--steps.pathloss.save_results=true"
]
Step.from_cmdline(args)
return rtdata
def test_image3_pipeline1(self):
asn_name = "mosaic_long_asn.json"
asn_file = self.get_data('test_calimage3', asn_name)
for file in raw_from_asn(asn_file):
self.get_data('test_calimage3', file)
collect_pipeline_cfgs('config')
args = [
'config/calwebb_image3.cfg',
asn_file,
'--steps.tweakreg.skip=True',
]
Step.from_cmdline(args)
outputs = [
( # Compare level-2c crf product
'nrca5_47Tuc_subpix_dither1_newpos_a3001_crf.fits',
'nrca5_47Tuc_subpix_dither1_newpos_cal-a3001_ref.fits'),
{
'files': ( # Compare i2d product
'mosaic_long_i2d.fits', 'mosaic_long_i2d_ref.fits'),
'pars': {
'ignore_keywords':
self.ignore_keywords + ['NAXIS1', 'TFORM*'],
'ignore_fields': self.ignore_keywords,
'rtol': 0.0001
}
}
]
self.compare_outputs(outputs)
def run_pipelines(jail, rtdata_module):
"""Run stage 2-3 pipelines on NIRISS SOSS data."""
rtdata = rtdata_module
collect_pipeline_cfgs("config")
# Run tso-spec2 pipeline on the first _rateints file, saving intermediate products
rtdata.get_data(
"niriss/soss/jw00625023001_03101_00001-seg001_nis_rateints.fits")
args = [
"config/calwebb_tso-spec2.cfg",
rtdata.input,
"--steps.flat_field.save_results=True",
"--steps.srctype.save_results=True",
]
Step.from_cmdline(args)
# Run tso-spec2 pipeline on the second _rateints file, without saving or
# checking any results (simply create a fresh input for level-3 test)
rtdata.get_data(
"niriss/soss/jw00625023001_03101_00001-seg002_nis_rateints.fits")
args = ["config/calwebb_tso-spec2.cfg", rtdata.input]
Step.from_cmdline(args)
# Get the level3 assocation json file (though not its members) and run
# the tso3 pipeline on all _calints files listed in association
rtdata.get_data(
"niriss/soss/jw00625-o023_20191210t204036_tso3_001_asn.json")
args = ["config/calwebb_tso3.cfg", rtdata.input]
Step.from_cmdline(args)
return rtdata
def test_image3_pipeline1(self):
asn_name = "mosaic_long_asn.json"
asn_file = self.get_data('test_calimage3', asn_name)
for file in raw_from_asn(asn_file):
self.get_data('test_calimage3', file)
collect_pipeline_cfgs('config')
args = [
'config/calwebb_image3.cfg',
asn_file,
'--steps.tweakreg.skip=True',
]
Step.from_cmdline(args)
self.ignore_keywords += ['NAXIS1', 'TFORM*']
self.ignore_fields = self.ignore_keywords
self.rtol = 0.0001
outputs = [('nrca5_47Tuc_subpix_dither1_newpos_a3001_crf.fits',
'nrca5_47Tuc_subpix_dither1_newpos_cal-a3001_ref.fits'),
('mosaic_long_i2d.fits',
'mosaic_long_i2d_ref.fits'),
('mosaic_long_cat.ecsv',
'mosaic_long_cat_ref.ecsv'),
]
self.compare_outputs(outputs)
def test_miri_masterbg_lrs_dedicated(self):
"""Run masterbackground step on MIRI LRS association"""
asn_file = self.get_data(*self.test_dir,
'miri_lrs_mbkg_dedicated_spec3_asn.json')
for file in raw_from_asn(asn_file):
self.get_data(*self.test_dir, file)
collect_pipeline_cfgs('./config')
result = MasterBackgroundStep.call(
asn_file,
config_file='config/master_background.cfg',
save_background=True,
save_results=True)
# test 1
# loop over the background subtracted data and compare to truth files
for model in result:
assert model.meta.cal_step.master_background == 'COMPLETE'
result_file = model.meta.filename.replace('cal',
'master_background')
truth_file = self.get_data(*self.ref_loc, result_file)
outputs = [(result_file, truth_file)]
self.compare_outputs(outputs)
# test 2
# compare the master background combined file to truth file
master_combined_bkg_file = 'MIRI_LRS_seq1_MIRIMAGE_P750Lexp1_o001_masterbg.fits'
truth_background = self.get_data(*self.ref_loc,
master_combined_bkg_file)
outputs = [(master_combined_bkg_file, truth_background)]
self.compare_outputs(outputs)
def run_pipelines(jail, rtdata_module):
"""Run stage 2-3 tso pipelines on NIRCAM TSO grism data."""
rtdata = rtdata_module
collect_pipeline_cfgs("config")
# Run tso-spec2 pipeline on the _rateints file, saving intermediate products
rtdata.get_data(
"nircam/tsgrism/jw00721012001_03103_00001-seg001_nrcalong_rateints.fits"
)
args = [
"config/calwebb_tso-spec2.cfg",
rtdata.input,
"--steps.flat_field.save_results=True",
"--steps.extract_2d.save_results=True",
"--steps.srctype.save_results=True",
]
Step.from_cmdline(args)
# Get the level3 assocation json file (though not its members) and run
# the tso3 pipeline on all _calints files listed in association
rtdata.get_data(
"nircam/tsgrism/jw00721-o012_20191119t043909_tso3_001_asn.json")
args = ["config/calwebb_tso3.cfg", rtdata.input]
Step.from_cmdline(args)
return rtdata
def run_spec3(jail, rtdata_module):
"""Run the Spec3Pipeline on the single cal result from the Spec2Pipeline run"""
rtdata = rtdata_module
collect_pipeline_cfgs("config")
# Note that we use the truth file from spec2 processing as the input to spec3
rtdata.get_data(TRUTH_PATH + '/' + 'ifushort_ch12_cal.fits')
asn = asn_from_list([rtdata.input], product_name='ifushort_ch12_spec3')
asn.data["program"] = "00024"
asn.data["asn_type"] = "spec3"
asn.sequence = 1
asn_name, serialized = asn.dump(format="json")
with open(asn_name, "w") as f:
f.write(serialized)
rtdata.input = asn_name
args = [
"config/calwebb_spec3.cfg",
rtdata.input,
'--steps.master_background.save_results=true',
'--steps.mrs_imatch.save_results=true',
'--steps.outlier_detection.save_results=true',
'--steps.resample_spec.save_results=true',
'--steps.cube_build.save_results=true',
'--steps.extract_1d.save_results=true',
'--steps.combine_1d.save_results=true',
]
Step.from_cmdline(args)
return rtdata
def test_nirspec_ifu_mbkg_user(rtdata, fitsdiff_default_kwargs):
"""Test NIRSpec IFU data with a user-supplied background file."""
# Get user-supplied background
user_background = "prism_bkg_x1d.fits"
rtdata.get_data(f"nirspec/ifu/{user_background}")
# Get input data
rtdata.get_data("nirspec/ifu/prism_sci_bkg_cal.fits")
collect_pipeline_cfgs("config")
args = [
"config/master_background.cfg", rtdata.input, "--user_background",
user_background
]
Step.from_cmdline(args)
output = "prism_sci_bkg_master_background.fits"
rtdata.output = output
# Get the truth file
rtdata.get_truth(f"truth/test_nirspec_ifu_mbkg_user/{output}")
# Compare the results
diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs)
assert diff.identical, diff.report()
def test_master_background_init(input_data, status, _jail, user_background):
"""Verify data can run through the step"""
result = MasterBackgroundStep.call(input_data)
collect_pipeline_cfgs('./config')
result = MasterBackgroundStep.call(
input_data, config_file='config/master_background.cfg')
# For inputs that are not files, the following should be true
assert type(input_data) is type(result)
assert result is not input_data
if isinstance(result, datamodels.ModelContainer):
for model in result:
assert model.meta.cal_step.master_background == status
else:
assert result.meta.cal_step.master_background == status
# Run with a user-supplied background and verify this is recorded in header
result = MasterBackgroundStep.call(input_data,
user_background=user_background)
if isinstance(result, datamodels.ModelContainer):
for model in result:
if model.meta.cal_step.master_background == 'COMPLETE':
assert model.meta.background.master_background_file == 'user_background.fits'
else:
if result.meta.cal_step.master_background == 'COMPLETE':
assert result.meta.background.master_background_file == 'user_background.fits'
# Make sure saving the computed background works
result = MasterBackgroundStep.call(input_data, save_background=True)
def test_nrs_fs_spec3(self):
"""
Regression test of calwebb_spec3 pipeline performed on
NIRSpec fixed-slit data.
"""
cfg_dir = './cfgs'
collect_pipeline_cfgs(cfg_dir)
datapath = ['test_datasets', 'fss', '93045', 'level2b']
asn_file = self.get_data(
*datapath, 'jw93045-o010_20180725t035735_spec3_001_asn.json')
for file in raw_from_asn(asn_file):
self.get_data(*datapath, file)
args = [path.join(cfg_dir, 'calwebb_spec3.cfg'), asn_file]
Step.from_cmdline(args)
# Compare results
outputs = [('jw00023001001_01101_00001_NRS1_cal.fits',
'jw00023001001_01101_00001_NRS1_cal_ref.fits'),
('jw00023001001_01101_00001_NRS1_s2d.fits',
'jw00023001001_01101_00001_NRS1_s2d_ref.fits'),
('jw00023001001_01101_00001_NRS1_x1d.fits',
'jw00023001001_01101_00001_NRS1_x1d_ref.fits')]
self.compare_outputs(outputs)
def test_nirspec_gwa(_jail, background, science_image):
"""Verify NIRSPEC GWA logic for in the science and background"""
# open the background to read in the GWA values
back_image = datamodels.open(background)
science_image.meta.instrument.gwa_xtilt = back_image.meta.instrument.gwa_xtilt
science_image.meta.instrument.gwa_ytilt = back_image.meta.instrument.gwa_ytilt
science_image.meta.instrument.gwa_tilt = back_image.meta.instrument.gwa_tilt
bkg = [background]
# Test Run with GWA values the same - confirm it runs
# And gives the predicted result
collect_pipeline_cfgs('./config')
result = BackgroundStep.call(
science_image,
bkg,
config_file='config/background.cfg',
)
test = science_image.data - back_image.data
assert_allclose(result.data, test)
assert type(result) is type(science_image)
assert result.meta.cal_step.back_sub == 'COMPLETE'
back_image.close()
def test_miri_masterbg_mrs_nodded(self):
"""Run masterbackground step on MIRI MRS association"""
asn_file = self.get_data(*self.test_dir,
'miri_mrs_mbkg_spec3_asn.json')
for file in raw_from_asn(asn_file):
self.get_data(*self.test_dir, file)
collect_pipeline_cfgs('./config')
result = MasterBackgroundStep.call(
asn_file,
config_file='config/master_background.cfg',
save_background=True,
save_results=True,
)
# test 1
# loop over the background subtracted data and compare to truth files
# check that the cal_step master_background ran to complete
for model in result:
assert model.meta.cal_step.master_background == 'COMPLETE'
truth_file = self.get_data(*self.ref_loc, model.meta.filename)
outputs = [(model.meta.filename, truth_file)]
self.compare_outputs(outputs)
# test 2
# compare the master background combined file to truth file
master_combined_bkg_file = 'MIRI_MRS_nod_seq1_MIRIFUSHORT_12SHORTexp1_o001_masterbg.fits'
truth_background = self.get_data(*self.ref_loc,
master_combined_bkg_file)
outputs = [(master_combined_bkg_file, truth_background)]
self.compare_outputs(outputs)
def test_image3_pipeline1(self):
asn_name = "mosaic_long_asn.json"
asn_file = self.get_data('test_calimage3', asn_name)
for file in raw_from_asn(asn_file):
self.get_data('test_calimage3', file)
collect_pipeline_cfgs('config')
args = [
'config/calwebb_image3.cfg',
asn_file,
'--steps.tweakreg.skip=True',
]
Step.from_cmdline(args)
outputs = [(# Compare level-2c crf product
'nrca5_47Tuc_subpix_dither1_newpos_a3001_crf.fits',
'nrca5_47Tuc_subpix_dither1_newpos_cal-a3001_ref.fits'),
{'files':(# Compare i2d product
'mosaic_long_i2d.fits',
'mosaic_long_i2d_ref.fits'),
'pars': {'ignore_keywords':
self.ignore_keywords+['NAXIS1', 'TFORM*'],
'ignore_fields':self.ignore_keywords,
'rtol': 0.0001}
},
('mosaic_long_cat.ecsv', 'mosaic_long_cat_ref.ecsv'),
]
self.compare_outputs(outputs)
def test_run_msaflagging(self, caplog):
"""Test msa flagging operation"""
# Retrieve the data.
collect_pipeline_cfgs('cfgs')
self.get_data(
*self.test_dir, 'jw95065006001_0_msa_twoslit.fits'
)
asn_path = self.get_data(
*self.test_dir, 'mos_udf_g235m_twoslit_spec2_asn.json'
)
with open(asn_path) as fp:
asn = load_asn(fp)
for product in asn['products']:
for member in product['members']:
self.get_data(
*self.test_dir, 'level2a_twoslit', member['expname']
)
# Run step.
args = [
op.join('cfgs', 'calwebb_spec2.cfg'),
asn_path,
'--steps.msa_flagging.skip=false'
]
Step.from_cmdline(args)
# Test.
assert 'Step msa_flagging running with args' in caplog.text
assert 'Step msa_flagging done' in caplog.text
for product in asn['products']:
prod_name = product['name'] + '_cal.fits'
assert op.isfile(prod_name)
def test_call_straylight1(_jail, miri_mrs):
""" test possible user options are set up correctly """
collect_pipeline_cfgs('./config')
# Test the ModShepard power is in the correct range
# set the power to outside the upper range
step = StraylightStep.from_config_file('config/straylight.cfg')
step.override_straylight = 'dummy.asdf'
step.method = 'ModShepard'
step.power = 6
result = step.run(miri_mrs)
assert result.meta.cal_step.straylight == 'SKIPPED'
# set the power to outside the lower range
step.power = 0.01
result = step.run(miri_mrs)
assert result.meta.cal_step.straylight == 'SKIPPED'
# Test that roi is in the correct range
step.power = 2 # reasonable value
step.roi = 1
result = step.run(miri_mrs)
assert result.meta.cal_step.straylight == 'SKIPPED'
step.power = 2 # reasonable value
step.roi = 1500
result = step.run(miri_mrs)
assert result.meta.cal_step.straylight == 'SKIPPED'
def run_pipelines(jail, rtdata_module):
"""Run stage 2 and 3 pipelines on NIRCam TSIMG data."""
rtdata = rtdata_module
collect_pipeline_cfgs("config")
# Run the calwebb_tso-image2 pipeline on each of the 2 inputs
rate_files = [
"nircam/tsimg/jw00312006001_02102_00001-seg001_nrcb1_rateints.fits",
"nircam/tsimg/jw00312006001_02102_00001-seg002_nrcb1_rateints.fits"
]
for rate_file in rate_files:
rtdata.get_data(rate_file)
args = ["config/calwebb_tso-image2.cfg", rtdata.input]
Step.from_cmdline(args)
# Get the level3 assocation json file (though not its members) and run
# the tso3 pipeline on all _calints files listed in association
rtdata.get_data(
"nircam/tsimg/jw00312-o006_20191225t115310_tso3_001_asn.json")
args = ["config/calwebb_tso3.cfg", rtdata.input]
Step.from_cmdline(args)
return rtdata
def test_miri_masterbg_lrs_dedicated(self):
"""Run masterbackground step on MIRI LRS association"""
asn_file = self.get_data(*self.test_dir,
'miri_lrs_mbkg_dedicated_spec3_asn.json')
for file in raw_from_asn(asn_file):
self.get_data(*self.test_dir, file)
collect_pipeline_cfgs('./config')
result = MasterBackgroundStep.call(
asn_file,
config_file='config/master_background.cfg',
save_background = True,
save_results = True)
# test 1
# loop over the background subtracted data and compare to truth files
for model in result:
assert model.meta.cal_step.master_background == 'COMPLETE'
result_file = model.meta.filename.replace('cal', 'master_background')
truth_file = self.get_data(*self.ref_loc,
result_file)
outputs = [(result_file, truth_file)]
self.compare_outputs(outputs)
# test 2
# compare the master background combined file to truth file
master_combined_bkg_file = 'MIRI_LRS_seq1_MIRIMAGE_P750Lexp1_o001_masterbg.fits'
truth_background = self.get_data(*self.ref_loc,
master_combined_bkg_file)
outputs = [(master_combined_bkg_file, truth_background)]
self.compare_outputs(outputs)
def test_nrs_fs_spec3(self):
"""
Regression test of calwebb_spec3 pipeline performed on
NIRSpec fixed-slit data.
"""
cfg_dir = './cfgs'
collect_pipeline_cfgs(cfg_dir)
datapath = ['test_datasets', 'fss', '93045', 'level2b']
asn_file = self.get_data(*datapath,
'jw93045-o010_20180725t035735_spec3_001_asn.json')
for file in raw_from_asn(asn_file):
self.get_data(*datapath, file)
args = [
path.join(cfg_dir, 'calwebb_spec3.cfg'),
asn_file
]
Step.from_cmdline(args)
# Compare results
outputs = [('jw00023001001_01101_00001_NRS1_cal.fits',
'jw00023001001_01101_00001_NRS1_cal_ref.fits'),
('jw00023001001_01101_00001_NRS1_s2d.fits',
'jw00023001001_01101_00001_NRS1_s2d_ref.fits'),
('jw00023001001_01101_00001_NRS1_x1d.fits',
'jw00023001001_01101_00001_NRS1_x1d_ref.fits')
]
self.compare_outputs(outputs)
def run_pipeline(jail, rtdata_module):
"""Run the calwebb_spec2 pipeline on an ASN of nodded MIRI LRS
fixedslit exposures."""
rtdata = rtdata_module
# Get the cfg files
collect_pipeline_cfgs("config")
# Get the spec2 ASN and its members
rtdata.get_asn("miri/lrs/jw00623-o032_20191210t195246_spec2_001_asn.json")
# Run the calwebb_spec2 pipeline; save results from intermediate steps
# NOTE: THE RESAMPLE_SPEC STEP IS SKIPPED FOR NOW, BECAUSE IT HAS A BUG
# (the s2d image is all zeros)
args = [
"config/calwebb_spec2.cfg",
rtdata.input,
"--steps.resample_spec.skip=true", # remove when bug fixed
"--save_bsub=true",
"--steps.assign_wcs.save_results=true",
"--steps.flat_field.save_results=true",
"--steps.srctype.save_results=true"
]
Step.from_cmdline(args)
return rtdata
def run_image3_closedfile(rtdata, jail):
"""Run calwebb_image3 on NIRCam imaging with data that had a closed file issue."""
rtdata.get_asn("nircam/image/fail_short_image3_asn.json")
collect_pipeline_cfgs("config")
args = ["config/calwebb_image3.cfg", rtdata.input]
Step.from_cmdline(args)
def run_pipeline(jail, rtdata_module):
"""Run calwebb_image2 pipeline on MIRI imaging data."""
rtdata = rtdata_module
rtdata.get_data("miri/image/jw00001001001_01101_00001_mirimage_rate.fits")
collect_pipeline_cfgs("config")
args = ["config/calwebb_image2.cfg", rtdata.input]
Step.from_cmdline(args)
return rtdata
def test_call_straylight2(_jail, miri_mrs_long):
""" test step is skipped for MRS IFULONG data """
collect_pipeline_cfgs('./config')
# Test the ModShepard power is in the correct range
step = StraylightStep.from_config_file('config/straylight.cfg')
step.override_straylight = 'dummy.asdf'
step.method = 'ModShepard'
result = step.run(miri_mrs_long)
assert result.meta.cal_step.straylight == 'SKIPPED'
def run_pipeline(jail, rtdata_module):
"""Run calwebb_spec3 on NIRSpec MOS data."""
rtdata = rtdata_module
rtdata.get_asn("nirspec/mos/jw00626-o030_20191210t193826_spec3_001_asn.json")
# Run the calwebb_spec3 pipeline on the association
collect_pipeline_cfgs("config")
args = ["config/calwebb_spec3.cfg", rtdata.input]
Step.from_cmdline(args)
return rtdata
def test_nrs_fs_spec3(_bigdata):
"""
Regression test of calwebb_spec3 pipeline performed on
NIRSpec fixed-slit data.
"""
cfg_dir = './cfgs'
collect_pipeline_cfgs(cfg_dir)
datapath = path.join(_bigdata, 'nirspec', 'test_datasets', 'fss', '93045',
'level2b')
args = [
path.join(cfg_dir, 'calwebb_spec3.cfg'),
path.join(datapath, 'jw93045-o010_20180725t035735_spec3_001_asn.json')
]
Step.from_cmdline(args)
# Compare results
assert False
ignore_keywords = ['DATE', 'CAL_VER', 'CAL_VCS', 'CRDS_VER', 'CRDS_CTX']
na = 'jw00023001001_01101_00001_NRS1_cal.fits'
nb = _bigdata + '/pipelines/jw00023001001_01101_00001_NRS1_cal_ref.fits'
h = pf.open(na)
href = pf.open(nb)
result = pf.diff.FITSDiff(h,
href,
ignore_hdus=['ASDF'],
ignore_keywords=ignore_keywords,
rtol=0.00001)
assert result.identical, result.report()
na = 'jw00023001001_01101_00001_NRS1_s2d.fits'
nb = _bigdata + '/pipelines/jw00023001001_01101_00001_NRS1_s2d_ref.fits'
h = pf.open(na)
href = pf.open(nb)
result = pf.diff.FITSDiff(h,
href,
ignore_hdus=['ASDF'],
ignore_keywords=ignore_keywords,
rtol=0.00001)
assert result.identical, result.report()
na = 'jw00023001001_01101_00001_NRS1_x1d.fits'
nb = _bigdata + '/pipelines/jw00023001001_01101_00001_NRS1_x1d_ref.fits'
h = pf.open(na)
href = pf.open(nb)
result = pf.diff.FITSDiff(h,
href,
ignore_hdus=['ASDF'],
ignore_keywords=ignore_keywords,
rtol=0.00001)
assert result.identical, result.report()
def test_nirspec_image2(_jail, rtdata, fitsdiff_default_kwargs):
rtdata.get_data("nirspec/imaging/jw84600010001_02102_00001_nrs2_rate.fits")
collect_pipeline_cfgs("config")
args = ["config/calwebb_image2.cfg", rtdata.input]
Step.from_cmdline(args)
rtdata.output = "jw84600010001_02102_00001_nrs2_cal.fits"
rtdata.get_truth("truth/test_nirspec_image2/jw84600010001_02102_00001_nrs2_cal.fits")
diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs)
assert diff.identical, diff.report()
def test_nircam_image_moving_target(rtdata, fitsdiff_default_kwargs):
"""Test resampled i2d of moving target exposures for NIRCam imaging"""
collect_pipeline_cfgs("config")
rtdata.get_asn("nircam/image/mt_asn.json")
rtdata.output = "mt_assoc_i2d.fits"
args = ["config/calwebb_image3.cfg", rtdata.input]
Step.from_cmdline(args)
rtdata.get_truth("truth/test_nircam_mtimage/mt_assoc_i2d.fits")
fitsdiff_default_kwargs["atol"] = 1e-5
diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs)
assert diff.identical, diff.report()
def test_fgs_toobig(rtdata, fitsdiff_default_kwargs, caplog, monkeypatch):
"""Test for the situation where the combined mosaic is too large"""
# Set the environment to not allow the resultant too-large image.
monkeypatch.setenv('DMODEL_ALLOWED_MEMORY', "0.9")
rtdata.get_asn('fgs/image3/image3_asn.json')
collect_pipeline_cfgs('config')
args = ['config/calwebb_image3.cfg', rtdata.input]
with pytest.raises(OutputTooLargeError):
Step.from_cmdline(args)
def test_msa_missing_nofail(self, caplog):
"""Test MSA missing failure"""
input_file = self.get_data(
*self.test_dir, 'level2a_twoslit', 'F170LP-G235M_MOS_observation-6-c0e0_001_DN_NRS1_mod.fits'
)
collect_pipeline_cfgs('cfgs')
args = [
op.join('cfgs', 'calwebb_spec2.cfg'),
input_file,
'--fail_on_exception=false'
]
Step.from_cmdline(args)
assert 'Unable to open MSA FITS file (MSAMETFL)' in caplog.text
def test_msa_missing_skip(self, caplog):
"""Test MSA missing failure"""
input_file = self.get_data(
*self.test_dir, 'level2a_twoslit', 'F170LP-G235M_MOS_observation-6-c0e0_001_DN_NRS1_mod.fits'
)
collect_pipeline_cfgs('cfgs')
args = [
op.join('cfgs', 'calwebb_spec2.cfg'),
input_file,
'--steps.assign_wcs.skip=true'
]
Step.from_cmdline(args)
assert 'Aborting remaining processing for this exposure.' in caplog.text
def test_image2pipeline1(self):
"""
Regression test of calwebb_image2 pipeline performed on MIRI data.
"""
input_file = self.get_data('test_image2pipeline',
'jw00001001001_01101_00001_mirimage_rate.fits')
collect_pipeline_cfgs('cfgs')
Image2Pipeline.call(input_file,
config_file='cfgs/calwebb_image2.cfg'
)
outputs = [('jw00001001001_01101_00001_mirimage_cal.fits',
'jw00001001001_01101_00001_mirimage_cal_ref.fits',
['primary','sci','err','dq','area']),
('jw00001001001_01101_00001_mirimage_i2d.fits',
'jw00001001001_01101_00001_mirimage_i2d_ref.fits',
['primary','sci','wht','con'])
]
self.compare_outputs(outputs)
def test_nrs_fs_brightobj_spec2(self):
"""
Regression test of calwebb_spec2 pipeline performed on NIRSpec
fixed-slit data that uses the NRS_BRIGHTOBJ mode (S1600A1 slit).
"""
input_file = self.get_data(self.test_dir,
'jw84600042001_02101_00001_nrs2_rateints.fits')
collect_pipeline_cfgs()
args = [
'calwebb_tso-spec2.cfg',
input_file
]
Step.from_cmdline(args)
outputs = [('jw84600042001_02101_00001_nrs2_calints.fits',
'jw84600042001_02101_00001_nrs2_calints_ref.fits'),
('jw84600042001_02101_00001_nrs2_x1dints.fits',
'jw84600042001_02101_00001_nrs2_x1dints_ref.fits')
]
self.compare_outputs(outputs)
def test_image2pipeline2_cal(self):
"""
Regression test of calwebb_image2 pipeline performed on NIRCam data.
"""
input_file = self.get_data('test_image2pipeline',
'jw82500001003_02101_00001_NRCALONG_rate.fits')
output_file = 'jw82500001003_02101_00001_NRCALONG_cal.fits'
collect_pipeline_cfgs('cfgs')
Image2Pipeline.call(input_file,
config_file='cfgs/calwebb_image2.cfg',
output_file=output_file)
outputs = [(output_file,
'jw82500001003_02101_00001_NRCALONG_cal_ref.fits',
['primary','sci','err','dq','area']),
('jw82500001003_02101_00001_NRCALONG_i2d.fits',
'jw82500001003_02101_00001_NRCALONG_i2d_ref.fits',
['primary','sci','con','wht'])
]
self.compare_outputs(outputs)
def test_nirisssoss2pipeline1(self):
"""
Regression test of calwebb_tso_spec2 pipeline performed on NIRISS SOSS data.
"""
input_file = self.get_data(self.test_dir,
'jw10003001002_03101_00001-seg003_nis_rateints.fits')
collect_pipeline_cfgs()
args = [
'calwebb_tso-spec2.cfg',
input_file
]
Step.from_cmdline(args)
outputs = [{'files':('jw10003001002_03101_00001-seg003_nis_calints.fits',
'jw10003001002_03101_00001-seg003_nis_calints_ref.fits'),
'pars':dict(ignore_hdus=['INT_TIMES', 'VAR_POISSON',
'VAR_RNOISE', 'ASDF'])},
{'files':('jw10003001002_03101_00001-seg003_nis_x1dints.fits',
'jw10003001002_03101_00001-seg003_nis_x1dints_ref.fits'),
'pars':dict(ignore_hdus=['INT_TIMES', 'ASDF'])}
]
self.compare_outputs(outputs)
def test_asn_naming(self):
"""Test a full run"""
# Get the data
collect_pipeline_cfgs('cfgs')
asn_path = self.get_data(
self.test_dir, 'wfs_3sets_asn.json'
)
with open(asn_path) as fh:
asn = load_asn(fh)
for product in asn['products']:
for member in product['members']:
self.get_data(
self.test_dir, member['expname']
)
input_files = glob('*')
# Run the step.
args = [
op.join('cfgs', 'calwebb_wfs-image3.cfg'),
asn_path
]
Step.from_cmdline(args)
# Test.
output_files = glob('*')
for input_file in input_files:
output_files.remove(input_file)
print('output_files = {}'.format(output_files))
for product in asn['products']:
prod_name = product['name']
prod_name = format_product(prod_name, suffix='wfscmb')
prod_name += '.fits'
assert prod_name in output_files
output_files.remove(prod_name)
# There should be no more files
assert len(output_files) == 0
