def test_image3_pipeline1(self):
asn_name = "mosaic_long_asn.json"
asn_file = self.get_data('test_calimage3', asn_name)
for file in raw_from_asn(asn_file):
self.get_data('test_calimage3', file)
collect_pipeline_cfgs('config')
args = [
'config/calwebb_image3.cfg',
asn_file,
'--steps.tweakreg.skip=True',
]
Step.from_cmdline(args)
self.ignore_keywords += ['NAXIS1', 'TFORM*']
self.ignore_fields = self.ignore_keywords
self.rtol = 0.0001
outputs = [('nrca5_47Tuc_subpix_dither1_newpos_a3001_crf.fits',
'nrca5_47Tuc_subpix_dither1_newpos_cal-a3001_ref.fits'),
('mosaic_long_i2d.fits',
'mosaic_long_i2d_ref.fits'),
('mosaic_long_cat.ecsv',
'mosaic_long_cat_ref.ecsv'),
]
self.compare_outputs(outputs)
def test_miri_masterbg_lrs_dedicated(self):
"""Run masterbackground step on MIRI LRS association"""
asn_file = self.get_data(*self.test_dir,
'miri_lrs_mbkg_dedicated_spec3_asn.json')
for file in raw_from_asn(asn_file):
self.get_data(*self.test_dir, file)
collect_pipeline_cfgs('./config')
result = MasterBackgroundStep.call(
asn_file,
config_file='config/master_background.cfg',
save_background = True,
save_results = True)
# test 1
# loop over the background subtracted data and compare to truth files
for model in result:
assert model.meta.cal_step.master_background == 'COMPLETE'
result_file = model.meta.filename.replace('cal', 'master_background')
truth_file = self.get_data(*self.ref_loc,
result_file)
outputs = [(result_file, truth_file)]
self.compare_outputs(outputs)
# test 2
# compare the master background combined file to truth file
master_combined_bkg_file = 'MIRI_LRS_seq1_MIRIMAGE_P750Lexp1_o001_masterbg.fits'
truth_background = self.get_data(*self.ref_loc,
master_combined_bkg_file)
outputs = [(master_combined_bkg_file, truth_background)]
self.compare_outputs(outputs)
def test_image3_pipeline2(self):
"""Regression test definitions for CALIMAGE3 pipeline.
Regression test of calwebb_image3 pipeline on NIRCam
simulated long-wave data with a 6-point dither.
"""
asn_file = self.get_data(self.test_dir,
"jw10002-o001_20171116t191235_image3_002_asn.json")
for file in raw_from_asn(asn_file):
self.get_data(self.test_dir, file)
collect_pipeline_cfgs('config')
args = [
'config/calwebb_image3.cfg',
asn_file,
'--steps.tweakreg.kernel_fwhm=2',
'--steps.tweakreg.snr_threshold=5',
'--steps.tweakreg.enforce_user_order=True',
'--steps.tweakreg.searchrad=10',
'--steps.tweakreg.fitgeometry=rscale'
]
Step.from_cmdline(args)
outputs = [('jw10002001001_01101_00004_nrcblong_o001_crf.fits',
'jw10002001001_01101_00004_nrcblong_o001_crf_ref.fits'),
{'files':('jw10002-o001_t002_nircam_f444w_i2d.fits',
'jw10002-o001_t002_nircam_f444w_i2d_ref.fits'),
'pars':{'ignore_keywords':self.ignore_keywords+['NAXIS1','TFORM*'],
'ignore_fields':self.ignore_keywords,
'rtol':0.0001}
}
]
self.compare_outputs(outputs)
def test_nrs_fs_spec3(self):
"""
Regression test of calwebb_spec3 pipeline performed on
NIRSpec fixed-slit data.
"""
cfg_dir = './cfgs'
collect_pipeline_cfgs(cfg_dir)
datapath = ['test_datasets', 'fss', '93045', 'level2b']
asn_file = self.get_data(
*datapath, 'jw93045-o010_20180725t035735_spec3_001_asn.json')
for file in raw_from_asn(asn_file):
self.get_data(*datapath, file)
args = [path.join(cfg_dir, 'calwebb_spec3.cfg'), asn_file]
Step.from_cmdline(args)
# Compare results
outputs = [('jw00023001001_01101_00001_NRS1_cal.fits',
'jw00023001001_01101_00001_NRS1_cal_ref.fits'),
('jw00023001001_01101_00001_NRS1_s2d.fits',
'jw00023001001_01101_00001_NRS1_s2d_ref.fits'),
('jw00023001001_01101_00001_NRS1_x1d.fits',
'jw00023001001_01101_00001_NRS1_x1d_ref.fits')]
self.compare_outputs(outputs)
def test_tso3_pipeline_nrc2(self):
"""Regression test of calwebb_tso3 pipeline on NIRCam simulated data.
Scaled imaging mode outlier_detection will be tested here.
"""
asn_file = self.get_data(
self.test_dir, "jw93065-a3002_20170511t111213_tso3_001_asn.json")
for file in raw_from_asn(asn_file):
self.get_data(self.test_dir, file)
step = Tso3Pipeline()
step.scale_detection = True
step.outlier_detection.weight_type = 'exptime'
step.outlier_detection.pixfrac = 1.0
step.outlier_detection.kernel = 'square'
step.outlier_detection.fillval = 'INDEF'
step.outlier_detection.nlow = 0
step.outlier_detection.nhigh = 0
step.outlier_detection.maskpt = 0.7
step.outlier_detection.grow = 1
step.outlier_detection.snr = '4.0 3.0'
step.outlier_detection.scale = '0.5 0.4'
step.outlier_detection.backg = 0.0
step.outlier_detection.save_intermediate_results = False
step.outlier_detection.resample_data = False
step.outlier_detection.good_bits = 4
step.extract_1d.smoothing_length = 0
step.extract_1d.bkg_order = 0
step.run(asn_file)
outputs = [('jw93065002002_02101_00001_nrca1_a3002_crfints.fits',
'jw93065002002_02101_00001_nrca1_a3002_crfints_ref.fits',
['primary', 'sci', 'dq', 'err'])]
self.compare_outputs(outputs)
def test_miri_masterbg_mrs_nodded(self):
"""Run masterbackground step on MIRI MRS association"""
asn_file = self.get_data(*self.test_dir,
'miri_mrs_mbkg_spec3_asn.json')
for file in raw_from_asn(asn_file):
self.get_data(*self.test_dir, file)
collect_pipeline_cfgs('./config')
result = MasterBackgroundStep.call(
asn_file,
config_file='config/master_background.cfg',
save_background=True,
save_results=True,
)
# test 1
# loop over the background subtracted data and compare to truth files
# check that the cal_step master_background ran to complete
for model in result:
assert model.meta.cal_step.master_background == 'COMPLETE'
truth_file = self.get_data(*self.ref_loc, model.meta.filename)
outputs = [(model.meta.filename, truth_file)]
self.compare_outputs(outputs)
# test 2
# compare the master background combined file to truth file
master_combined_bkg_file = 'MIRI_MRS_nod_seq1_MIRIFUSHORT_12SHORTexp1_o001_masterbg.fits'
truth_background = self.get_data(*self.ref_loc,
master_combined_bkg_file)
outputs = [(master_combined_bkg_file, truth_background)]
self.compare_outputs(outputs)
def test_save_source_only(self):
"""Test saving the source-based files only"""
datapath = ['test_datasets', 'fss', '93045', 'level2b']
asn_file = self.get_data(
*datapath, 'jw93045-o010_20180725t035735_spec3_001_asn.json')
for file in raw_from_asn(asn_file):
self.get_data(*datapath, file)
pipe = Spec3Pipeline()
pipe.mrs_imatch.skip = True
pipe.outlier_detection.skip = True
pipe.resample_spec.skip = True
pipe.cube_build.skip = True
pipe.extract_1d.skip = True
pipe.run(asn_file)
# Check resulting product
with open(asn_file) as fh:
asn = load_asn(fh)
base_name = asn['products'][0]['name']
product_name = base_name.format(source_id='s00000') + '_cal.fits'
output_files = glob('*')
if product_name in output_files:
output_files.remove(product_name)
else:
assert False
def test_image3_pipeline1(self):
asn_name = "mosaic_long_asn.json"
asn_file = self.get_data('test_calimage3', asn_name)
for file in raw_from_asn(asn_file):
self.get_data('test_calimage3', file)
collect_pipeline_cfgs('config')
args = [
'config/calwebb_image3.cfg',
asn_file,
'--steps.tweakreg.skip=True',
]
Step.from_cmdline(args)
outputs = [(# Compare level-2c crf product
'nrca5_47Tuc_subpix_dither1_newpos_a3001_crf.fits',
'nrca5_47Tuc_subpix_dither1_newpos_cal-a3001_ref.fits'),
{'files':(# Compare i2d product
'mosaic_long_i2d.fits',
'mosaic_long_i2d_ref.fits'),
'pars': {'ignore_keywords':
self.ignore_keywords+['NAXIS1', 'TFORM*'],
'ignore_fields':self.ignore_keywords,
'rtol': 0.0001}
},
('mosaic_long_cat.ecsv', 'mosaic_long_cat_ref.ecsv'),
]
self.compare_outputs(outputs)
def test_save_source_only(self):
"""Test saving the source-based files only"""
datapath = ['test_datasets', 'fss', '93045', 'level2b']
asn_file = self.get_data(*datapath,
'jw93045-o010_20180725t035735_spec3_001_asn.json')
for file in raw_from_asn(asn_file):
self.get_data(*datapath, file)
pipe = Spec3Pipeline()
pipe.mrs_imatch.skip = True
pipe.outlier_detection.skip = True
pipe.resample_spec.skip = True
pipe.cube_build.skip = True
pipe.extract_1d.skip = True
pipe.run(asn_file)
# Check resulting product
with open(asn_file) as fh:
asn = load_asn(fh)
base_name = asn['products'][0]['name']
product_name = base_name.format(source_id='ss400a1') + '_cal.fits'
output_files = glob('*')
if product_name in output_files:
output_files.remove(product_name)
else:
assert False
def test_nirspec_masterbg_nodded(self):
"""Run masterbackground step on NIRSpec association"""
asn_file = self.get_data(*self.test_dir, 'nirspec_spec3_asn.json')
for file in raw_from_asn(asn_file):
self.get_data(*self.test_dir, file)
collect_pipeline_cfgs('./config')
result = MasterBackgroundStep.call(
asn_file,
config_file='config/master_background.cfg',
save_background=True,
save_results=True)
# test 1
# compare background subtracted data to truth files
# check that the cal_step master_background ran to complete
outputs = []
for model in result:
assert model.meta.cal_step.master_background == 'COMPLETE'
result_file = model.meta.filename.replace('cal',
'master_background')
truth_file = self.get_data(*self.ref_loc, result_file)
outputs.append((result_file, truth_file))
self.compare_outputs(outputs)
# test 2
# compare the master background combined file to truth file
master_combined_bkg_file = 'ifu_prism_source_off_fix_NRS1_o001_masterbg.fits'
truth_background = self.get_data(*self.ref_loc,
master_combined_bkg_file)
outputs = [(master_combined_bkg_file, truth_background)]
self.compare_outputs(outputs)
def test_tso3_pipeline_nrc2(self):
"""Regression test of calwebb_tso3 pipeline on NIRCam simulated data.
Scaled imaging mode outlier_detection will be tested here.
"""
asn_file = self.get_data(self.test_dir,
"jw93065-a3002_20170511t111213_tso3_001_asn.json")
for file in raw_from_asn(asn_file):
self.get_data(self.test_dir, file)
step = Tso3Pipeline()
step.scale_detection = True
step.outlier_detection.weight_type = 'exptime'
step.outlier_detection.pixfrac = 1.0
step.outlier_detection.kernel = 'square'
step.outlier_detection.fillval = 'INDEF'
step.outlier_detection.nlow = 0
step.outlier_detection.nhigh = 0
step.outlier_detection.maskpt = 0.7
step.outlier_detection.grow = 1
step.outlier_detection.snr = '4.0 3.0'
step.outlier_detection.scale = '0.5 0.4'
step.outlier_detection.backg = 0.0
step.outlier_detection.save_intermediate_results = False
step.outlier_detection.resample_data = False
step.outlier_detection.good_bits = 4
step.extract_1d.smoothing_length = 0
step.extract_1d.bkg_order = 0
step.run(asn_file)
outputs = [('jw93065002002_02101_00001_nrca1_a3002_crfints.fits',
'jw93065002002_02101_00001_nrca1_a3002_crfints_ref.fits',
['primary', 'sci', 'dq', 'err'])
]
self.compare_outputs(outputs)
def test_nrs_fs_spec3(self):
"""
Regression test of calwebb_spec3 pipeline performed on
NIRSpec fixed-slit data.
"""
cfg_dir = './cfgs'
collect_pipeline_cfgs(cfg_dir)
datapath = ['test_datasets', 'fss', '93045', 'level2b']
asn_file = self.get_data(*datapath,
'jw93045-o010_20180725t035735_spec3_001_asn.json')
for file in raw_from_asn(asn_file):
self.get_data(*datapath, file)
args = [
path.join(cfg_dir, 'calwebb_spec3.cfg'),
asn_file
]
Step.from_cmdline(args)
# Compare results
outputs = [('jw00023001001_01101_00001_NRS1_cal.fits',
'jw00023001001_01101_00001_NRS1_cal_ref.fits'),
('jw00023001001_01101_00001_NRS1_s2d.fits',
'jw00023001001_01101_00001_NRS1_s2d_ref.fits'),
('jw00023001001_01101_00001_NRS1_x1d.fits',
'jw00023001001_01101_00001_NRS1_x1d_ref.fits')
]
self.compare_outputs(outputs)
def test_image3_pipeline1(self):
asn_name = "mosaic_long_asn.json"
asn_file = self.get_data('test_calimage3', asn_name)
for file in raw_from_asn(asn_file):
self.get_data('test_calimage3', file)
collect_pipeline_cfgs('config')
args = [
'config/calwebb_image3.cfg',
asn_file,
'--steps.tweakreg.skip=True',
]
Step.from_cmdline(args)
outputs = [
( # Compare level-2c crf product
'nrca5_47Tuc_subpix_dither1_newpos_a3001_crf.fits',
'nrca5_47Tuc_subpix_dither1_newpos_cal-a3001_ref.fits'),
{
'files': ( # Compare i2d product
'mosaic_long_i2d.fits', 'mosaic_long_i2d_ref.fits'),
'pars': {
'ignore_keywords':
self.ignore_keywords + ['NAXIS1', 'TFORM*'],
'ignore_fields': self.ignore_keywords,
'rtol': 0.0001
}
}
]
self.compare_outputs(outputs)
def test_miri_masterbg_lrs_dedicated(self):
"""Run masterbackground step on MIRI LRS association"""
asn_file = self.get_data(*self.test_dir,
'miri_lrs_mbkg_dedicated_spec3_asn.json')
for file in raw_from_asn(asn_file):
self.get_data(*self.test_dir, file)
collect_pipeline_cfgs('./config')
result = MasterBackgroundStep.call(
asn_file,
config_file='config/master_background.cfg',
save_background=True,
save_results=True)
# test 1
# loop over the background subtracted data and compare to truth files
for model in result:
assert model.meta.cal_step.master_background == 'COMPLETE'
result_file = model.meta.filename.replace('cal',
'master_background')
truth_file = self.get_data(*self.ref_loc, result_file)
outputs = [(result_file, truth_file)]
self.compare_outputs(outputs)
# test 2
# compare the master background combined file to truth file
master_combined_bkg_file = 'MIRI_LRS_seq1_MIRIMAGE_P750Lexp1_o001_masterbg.fits'
truth_background = self.get_data(*self.ref_loc,
master_combined_bkg_file)
outputs = [(master_combined_bkg_file, truth_background)]
self.compare_outputs(outputs)
def test_coron3_1(self):
"""Regression test of calwebb_coron3 pipeline.
Test will be performed on NIRCam simulated data.
"""
asn_name = 'jw99999-a3001_20170327t121212_coron3_001_asn.json'
override_psfmask = 'jwst_nircam_psfmask_somb.fits'
# get a local copy of the inputs
asn_file = self.get_data('test_coron3', asn_name)
psfmask_file = self.get_data('test_coron3', override_psfmask)
for file in raw_from_asn(asn_file):
self.get_data('test_coron3', file)
pipe = Coron3Pipeline()
pipe.align_refs.override_psfmask = psfmask_file
pipe.outlier_detection.resample_data = False
pipe.run(asn_file)
outputs = [
( # Compare psfstack product
'jw99999-a3001_t1_nircam_f140m-maskbar_psfstack.fits',
'jw99999-a3001_t1_nircam_f140m-maskbar_psfstack_ref.fits'),
( # Compare psfalign product
'jw9999947001_02102_00001_nrcb3_a3001_psfalign.fits',
'jw99999-a3001_t1_nircam_f140m-maskbar_psfalign_ref.fits'),
( # Compare psfsub product
'jw9999947001_02102_00001_nrcb3_a3001_psfsub.fits',
'jw9999947001_02102_00001_nrcb3_psfsub_ref.fits'),
( # Compare level-2c products
'jw9999947001_02102_00001_nrcb3_a3001_crfints.fits',
'jw9999947001_02102_00001_nrcb3_a3001_crfints_ref.fits'),
('jw9999947001_02102_00002_nrcb3_a3001_crfints.fits',
'jw9999947001_02102_00002_nrcb3_a3001_crfints_ref.fits'),
{
'files': ( # Compare i2d product
'jw99999-a3001_t1_nircam_f140m-maskbar_i2d.fits',
'jw99999-a3001_t1_nircam_f140m-maskbar_i2d_ref.fits'),
'pars': {
'ignore_hdus': self.ignore_hdus + ['HDRTAB']
}
},
{
'files': ( # Compare the HDRTAB in the i2d product
'jw99999-a3001_t1_nircam_f140m-maskbar_i2d.fits[hdrtab]',
'jw99999-a3001_t1_nircam_f140m-maskbar_i2d_ref.fits[hdrtab]'
),
'pars': {
'ignore_keywords':
self.ignore_keywords + ['NAXIS1', 'TFORM*'],
'ignore_fields': self.ignore_keywords
}
}
]
self.compare_outputs(outputs)
def test_coron3_1(self):
"""Regression test of calwebb_coron3 pipeline.
Test will be performed on NIRCam simulated data.
"""
asn_name = 'jw99999-a3001_20170327t121212_coron3_001_asn.json'
override_psfmask = 'jwst_nircam_psfmask_somb.fits'
# get a local copy of the inputs
asn_file = self.get_data('test_coron3', asn_name)
psfmask_file = self.get_data('test_coron3', override_psfmask)
for file in raw_from_asn(asn_file):
self.get_data('test_coron3', file)
pipe = Coron3Pipeline()
pipe.align_refs.override_psfmask = psfmask_file
pipe.outlier_detection.resample_data = False
pipe.run(asn_file)
outputs = [( # Compare psfstack product
'jw99999-a3001_t1_nircam_f140m-maskbar_psfstack.fits',
'jw99999-a3001_t1_nircam_f140m-maskbar_psfstack_ref.fits'
),
( # Compare psfalign product
'jw9999947001_02102_00001_nrcb3_a3001_psfalign.fits',
'jw99999-a3001_t1_nircam_f140m-maskbar_psfalign_ref.fits'
),
( # Compare psfsub product
'jw9999947001_02102_00001_nrcb3_a3001_psfsub.fits',
'jw9999947001_02102_00001_nrcb3_psfsub_ref.fits'
),
( # Compare level-2c products
'jw9999947001_02102_00001_nrcb3_a3001_crfints.fits',
'jw9999947001_02102_00001_nrcb3_a3001_crfints_ref.fits'
),
(
'jw9999947001_02102_00002_nrcb3_a3001_crfints.fits',
'jw9999947001_02102_00002_nrcb3_a3001_crfints_ref.fits'
),
{'files':( # Compare i2d product
'jw99999-a3001_t1_nircam_f140m-maskbar_i2d.fits',
'jw99999-a3001_t1_nircam_f140m-maskbar_i2d_ref.fits'
),
'pars': {'ignore_hdus':self.ignore_hdus+['HDRTAB']}
},
{'files':( # Compare the HDRTAB in the i2d product
'jw99999-a3001_t1_nircam_f140m-maskbar_i2d.fits[hdrtab]',
'jw99999-a3001_t1_nircam_f140m-maskbar_i2d_ref.fits[hdrtab]'
),
'pars': {'ignore_keywords':self.ignore_keywords+['NAXIS1', 'TFORM*'],
'ignore_fields':self.ignore_keywords}
}
]
self.compare_outputs(outputs)
def test_miri_masterbackground_mrs(self):
"""Run masterbackground step on MIRI LRS association"""
asn_file = self.get_data(*self.test_dir,
'miri_mrs_mbkg_0304_spec3_asn.json')
for file in raw_from_asn(asn_file):
self.get_data(*self.test_dir, file)
collect_pipeline_cfgs('./config')
result = MasterBackgroundStep.call(
asn_file,
config_file='config/master_background.cfg',
save_background=True
)
for model in result:
assert model.meta.cal_step.master_background == 'COMPLETE'
def test_spec3_pipeline1(self):
"""
Regression test of calwebb_spec3 pipeline on simulated
MIRI MRS dithered data.
"""
asn_file = self.get_data(self.test_dir, 'test_asn4.json')
for file in raw_from_asn(asn_file):
self.get_data(self.test_dir, file)
step = Spec3Pipeline()
step.save_bsub = False
step.mrs_imatch.suffix = 'mrs_imatch'
step.mrs_imatch.bkg_degree = 1
step.mrs_imatch.subtract = False
step.outlier_detection.skip = True
step.output_use_model = True
step.resample_spec.save_results = True
step.resample_spec.suffix = 's2d'
step.cube_build.save_results = True
step.cube_build.suffix = 's3d'
step.extract_1d.save_results = True
step.extract_1d.suffix = 'x1d'
step.run(asn_file)
outputs = [(# Compare cube product 1
'det_image_ch1-short_s3d.fits',
'det_image_ch1-short_s3d_ref.fits',
['primary', 'sci', 'err', 'dq', 'wmap']),
(# Compare cube product 2
'det_image_ch2-short_s3d.fits',
'det_image_ch2-short_s3d_ref.fits',
['primary', 'sci', 'err', 'dq', 'wmap']),
(# Compare x1d product 1
'det_image_ch1-short_x1d.fits',
'det_image_ch1-short_x1d_ref.fits',
['primary', 'extract1d']),
(# Compare x1d product 2
'det_image_ch2-short_x1d.fits',
'det_image_ch2-short_x1d_ref.fits',
['primary', 'extract1d'])
]
self.compare_outputs(outputs)
def test_spec3_pipeline1(self):
"""
Regression test of calwebb_spec3 pipeline on simulated
MIRI MRS dithered data.
"""
asn_file = self.get_data(self.test_dir, 'test_asn4.json')
for file in raw_from_asn(asn_file):
self.get_data(self.test_dir, file)
step = Spec3Pipeline()
step.save_bsub = False
step.mrs_imatch.suffix = 'mrs_imatch'
step.mrs_imatch.bkg_degree = 1
step.mrs_imatch.subtract = False
step.outlier_detection.skip = True
step.output_use_model = True
step.resample_spec.save_results = True
step.resample_spec.suffix = 's2d'
step.cube_build.save_results = True
step.cube_build.suffix = 's3d'
step.extract_1d.save_results = True
step.extract_1d.suffix = 'x1d'
step.run(asn_file)
outputs = [
( # Compare cube product 1
'det_image_ch1-short_s3d.fits',
'det_image_ch1-short_s3d_ref.fits',
['primary', 'sci', 'err', 'dq', 'wmap']),
( # Compare cube product 2
'det_image_ch2-short_s3d.fits',
'det_image_ch2-short_s3d_ref.fits',
['primary', 'sci', 'err', 'dq', 'wmap']),
( # Compare x1d product 1
'det_image_ch1-short_x1d.fits',
'det_image_ch1-short_x1d_ref.fits', ['primary', 'extract1d']),
( # Compare x1d product 2
'det_image_ch2-short_x1d.fits',
'det_image_ch2-short_x1d_ref.fits', ['primary', 'extract1d'])
]
self.compare_outputs(outputs)
def test_ami_pipeline(self):
"""
Regression test of the AMI pipeline performed on NIRISS AMI data.
"""
asn_file = self.get_data(self.test_dir,
'test_lg1_asn.json')
for file in raw_from_asn(asn_file):
self.get_data(self.test_dir, file)
pipe = Ami3Pipeline()
pipe.save_averages = True
pipe.ami_analyze.oversample = 3
pipe.ami_analyze.rotation = 1.49
pipe.run(asn_file)
outputs = [('test_targ_aminorm.fits',
'ami_pipeline_targ_lgnorm.fits'),
]
self.compare_outputs(outputs, rtol=0.00001,
ignore_hdus=['ASDF', 'HDRTAB'])
def test_tso3_pipeline_nis(self):
"""Regression test of calwebb_tso3 on NIRISS SOSS simulated data.
"""
asn_file = self.get_data(
self.test_dir, "jw87600-a3001_20170527t111213_tso3_001_asn.json")
for file in raw_from_asn(asn_file):
self.get_data(self.test_dir, file)
Tso3Pipeline.call(asn_file)
outputs = [ # Compare level-2c product
('jw87600024001_02101_00001_nis_a3001_crfints.fits',
'jw87600-a3001_t1_niriss_clear-gr700xd_crfints_ref.fits',
['primary', 'sci', 'dq', 'err']),
# Compare level-3 product
('jw87600-a3001_t1_niriss_clear-gr700xd_x1dints.fits',
'jw87600-a3001_t1_niriss_clear-gr700xd_x1dints_ref.fits',
['primary', 'extract1d'])
]
self.compare_outputs(outputs)
def test_nrs_msa_spec3(self):
"""
Regression test of calwebb_spec3 pipeline performed on
NIRSpec MSA data
"""
cfg_dir = './cfgs'
collect_pipeline_cfgs(cfg_dir)
asn_file = self.get_data(*self.test_dir, 'single_asn.json')
for file in raw_from_asn(asn_file):
self.get_data(*self.test_dir, file)
args = [str(Path(cfg_dir) / 'calwebb_spec3.cfg'), asn_file]
Step.from_cmdline(args)
# Compare results
truths = self.data_glob(*self.ref_loc, glob='*.fits')
outputs = [(Path(output_file).name, ) * 2 for output_file in truths]
self.compare_outputs(outputs)
def test_wfs_combine2(self):
"""
Regression test of wfs_combine using do_refine=True
"""
asn_file = self.get_data(self.test_dir,
'wfs_3sets_asn3.json')
for file in raw_from_asn(asn_file):
self.get_data(self.test_dir, file)
WfsCombineStep.call(asn_file,
do_refine=True)
outputs = [('test_wfscom3_wfscmb.fits',
'test_wfscom_do_ref.fits'),
('test_wfscom3a_wfscmb.fits',
'test_wfscoma_do_ref.fits'),
('test_wfscom3b_wfscmb.fits',
'test_wfscomb_do_ref.fits')
]
self.compare_outputs(outputs)
def test_nis_wfss_spec2(self):
"""
Regression test of calwebb_spec2 pipeline performed on NIRISS WFSS data.
"""
# Collect data
asn_file = self.get_data(self.test_dir,
'jw87600-a3001_20171109T145456_spec2_001_asn.json')
for file in raw_from_asn(asn_file):
self.get_data(self.test_dir, file)
# Run the step
collect_pipeline_cfgs('cfgs')
Spec2Pipeline.call(asn_file, config_file='cfgs/calwebb_spec2.cfg', save_bsub=True)
# Test results.
outputs = [('jw87600017001_02101_00002_nis_cal.fits',
'jw87600017001_02101_00002_nis_cal_ref.fits'),
('jw87600017001_02101_00002_nis_x1d.fits',
'jw87600017001_02101_00002_nis_x1d_ref.fits')]
self.compare_outputs(outputs)
def test_tso3_pipeline_nis(self):
"""Regression test of calwebb_tso3 on NIRISS SOSS simulated data.
"""
asn_file = self.get_data(self.test_dir,
"jw87600-a3001_20170527t111213_tso3_001_asn.json")
for file in raw_from_asn(asn_file):
self.get_data(self.test_dir, file)
Tso3Pipeline.call(asn_file)
outputs = [# Compare level-2c product
('jw87600024001_02101_00001_nis_a3001_crfints.fits',
'jw87600-a3001_t1_niriss_clear-gr700xd_crfints_ref.fits',
['primary', 'sci', 'dq', 'err']),
# Compare level-3 product
('jw87600-a3001_t1_niriss_clear-gr700xd_x1dints.fits',
'jw87600-a3001_t1_niriss_clear-gr700xd_x1dints_ref.fits',
['primary', 'extract1d'])
]
self.compare_outputs(outputs)
def test_ami_pipeline(self):
"""
Regression test of the AMI pipeline performed on NIRISS AMI data.
"""
asn_file = self.get_data(self.test_dir,
'test_lg1_asn.json')
for file in raw_from_asn(asn_file):
self.get_data(self.test_dir, file)
pipe = Ami3Pipeline()
pipe.save_averages = True
pipe.ami_analyze.oversample = 3
pipe.ami_analyze.rotation = 1.49
pipe.run(asn_file)
outputs = [('test_targ_aminorm.fits',
'ami_pipeline_targ_lgnorm.fits'),
]
self.compare_outputs(outputs, rtol=0.00001,
ignore_hdus=['ASDF', 'HDRTAB'])
def test_wfs_combine(self):
"""
Regression test of wfs_combine using do_refine=False (default)
Association table has 3 (identical) pairs of input files to combine
"""
asn_file = self.get_data(self.test_dir,
'wfs_3sets_asn.json')
for file in raw_from_asn(asn_file):
self.get_data(self.test_dir, file)
WfsCombineStep.call(asn_file)
outputs = [('test_wfscom_wfscmb.fits',
'test_wfscom.fits'),
('test_wfscoma_wfscmb.fits',
'test_wfscoma.fits'),
('test_wfscomb_wfscmb.fits',
'test_wfscomb.fits')
]
self.compare_outputs(outputs)
def test_nis_wfss_spec2(self):
"""
Regression test of calwebb_spec2 pipeline performed on NIRISS WFSS data.
"""
asn_file = self.get_data(self.test_dir,
'jw87600-a3001_20171109T145456_spec2_001_asn.json')
for file in raw_from_asn(asn_file):
self.get_data(self.test_dir, file)
pipe = Spec2Pipeline()
pipe.save_bsub = True
pipe.save_results = True
pipe.resample_spec.save_results = True
pipe.extract_1d.save_results = True
pipe.run(asn_file)
outputs = [('jw87600017001_02101_00002_nis_cal.fits',
'jw87600017001_02101_00002_nis_cal_ref.fits'),
('jw87600017001_02101_00002_nis_x1d.fits',
'jw87600017001_02101_00002_nis_x1d_ref.fits')]
self.compare_outputs(outputs)
def test_image3_pipeline2(self):
"""Regression test definitions for CALIMAGE3 pipeline.
Regression test of calwebb_image3 pipeline on NIRCam
simulated long-wave data with a 6-point dither.
"""
asn_file = self.get_data(
self.test_dir, "jw10002-o001_20171116t191235_image3_002_asn.json")
for file in raw_from_asn(asn_file):
self.get_data(self.test_dir, file)
collect_pipeline_cfgs('config')
args = [
'config/calwebb_image3.cfg', asn_file,
'--steps.tweakreg.kernel_fwhm=2',
'--steps.tweakreg.snr_threshold=5',
'--steps.tweakreg.enforce_user_order=True',
'--steps.tweakreg.searchrad=10',
'--steps.tweakreg.fitgeometry=rscale'
]
Step.from_cmdline(args)
outputs = [('jw10002001001_01101_00004_nrcblong_o001_crf.fits',
'jw10002001001_01101_00004_nrcblong_o001_crf_ref.fits'), {
'files':
('jw10002-o001_t002_nircam_f444w_i2d.fits',
'jw10002-o001_t002_nircam_f444w_i2d_ref.fits'),
'pars': {
'ignore_keywords':
self.ignore_keywords + ['NAXIS1', 'TFORM*'],
'ignore_fields':
self.ignore_keywords,
'rtol':
0.0001
}
}]
self.compare_outputs(outputs)
def test_miri_lrs_bkgnod(self):
"""
Regression test of calwebb_spec2 pipeline performed on an association
of nodded MIRI LRS fixed-slit exposures.
"""
asn_file = self.get_data(self.test_dir,
'lrs_bkgnod_asn.json')
for file in raw_from_asn(asn_file):
self.get_data(self.test_dir, file)
step = Spec2Pipeline()
step.save_bsub=True,
step.save_results=True
step.resample_spec.save_results = True
step.cube_build.save_results = True
step.extract_1d.save_results = True
step.run(asn_file)
outputs = [('test_lrs1_bsub.fits', 'test_lrs1_bsub_ref.fits',
['primary','sci','err','dq']),
('test_lrs2_bsub.fits','test_lrs2_bsub_ref.fits',
['primary','sci','err','dq']),
('test_lrs3_bsub.fits','test_lrs3_bsub_ref.fits',
['primary','sci','err','dq']),
('test_lrs4_bsub.fits','test_lrs4_bsub_ref.fits',
['primary','sci','err','dq']),
('test_lrs1_cal.fits', 'test_lrs1_cal_ref.fits',
['primary','sci','err','dq','relsens']),
('test_lrs2_cal.fits', 'test_lrs2_cal_ref.fits',
['primary','sci','err','dq','relsens']),
('test_lrs3_cal.fits', 'test_lrs3_cal_ref.fits',
['primary','sci','err','dq','relsens']),
('test_lrs4_cal.fits', 'test_lrs4_cal_ref.fits',
['primary','sci','err','dq','relsens'])
]
self.compare_outputs(outputs)
