def test_wfs_image2(self):
"""
Regression test of the WFS&C `calwebb_wfs-image2.cfg` pipeline
"""
data_base = 'jw82600026001_02101_00001_nrca1_rate'
ext = '.fits'
input_name = '{}{}'.format(data_base, ext)
input_file = self.get_data(self.test_dir, 'sdp_jw82600_wfs', 'level2a',
input_name)
collect_pipeline_cfgs('cfgs')
Step.from_cmdline([os.path.join('cfgs', 'calwebb_wfs-image2.cfg'), input_file])
cal_name = input_name.replace('rate', 'cal')
output_name = input_name.replace('rate','cal_ref')
outputs = [(cal_name, output_name)]
self.compare_outputs(outputs)
output_files = glob.glob('*')
output_files.remove('cfgs')
# these would happen when docopy=True
if input_name in output_files:
output_files.remove(input_name)
if output_name in output_files:
output_files.remove(output_name)
assert cal_name in output_files
output_files.remove(cal_name)
assert not output_files, 'Unexpected output files {}'.format(output_files)
def test_image3_pipeline2(self):
"""Regression test definitions for CALIMAGE3 pipeline.
Regression test of calwebb_image3 pipeline on NIRCam
simulated long-wave data with a 6-point dither.
"""
asn_file = self.get_data(self.test_dir,
"jw10002-o001_20171116t191235_image3_002_asn.json")
for file in raw_from_asn(asn_file):
self.get_data(self.test_dir, file)
collect_pipeline_cfgs('config')
args = [
'config/calwebb_image3.cfg',
asn_file,
'--steps.tweakreg.kernel_fwhm=2',
'--steps.tweakreg.snr_threshold=5',
'--steps.tweakreg.enforce_user_order=True',
'--steps.tweakreg.searchrad=10',
'--steps.tweakreg.fitgeometry=rscale'
]
Step.from_cmdline(args)
outputs = [('jw10002001001_01101_00004_nrcblong_o001_crf.fits',
'jw10002001001_01101_00004_nrcblong_o001_crf_ref.fits'),
{'files':('jw10002-o001_t002_nircam_f444w_i2d.fits',
'jw10002-o001_t002_nircam_f444w_i2d_ref.fits'),
'pars':{'ignore_keywords':self.ignore_keywords+['NAXIS1','TFORM*'],
'ignore_fields':self.ignore_keywords,
'rtol':0.0001}
}
]
self.compare_outputs(outputs)
def test_nrs_fs_spec3(self):
"""
Regression test of calwebb_spec3 pipeline performed on
NIRSpec fixed-slit data.
"""
cfg_dir = './cfgs'
collect_pipeline_cfgs(cfg_dir)
datapath = ['test_datasets', 'fss', '93045', 'level2b']
asn_file = self.get_data(*datapath,
'jw93045-o010_20180725t035735_spec3_001_asn.json')
for file in raw_from_asn(asn_file):
self.get_data(*datapath, file)
args = [
path.join(cfg_dir, 'calwebb_spec3.cfg'),
asn_file
]
Step.from_cmdline(args)
# Compare results
outputs = [('jw00023001001_01101_00001_NRS1_cal.fits',
'jw00023001001_01101_00001_NRS1_cal_ref.fits'),
('jw00023001001_01101_00001_NRS1_s2d.fits',
'jw00023001001_01101_00001_NRS1_s2d_ref.fits'),
('jw00023001001_01101_00001_NRS1_x1d.fits',
'jw00023001001_01101_00001_NRS1_x1d_ref.fits')
]
self.compare_outputs(outputs)
def test_image3_pipeline1(self):
asn_name = "mosaic_long_asn.json"
asn_file = self.get_data('test_calimage3', asn_name)
for file in raw_from_asn(asn_file):
self.get_data('test_calimage3', file)
collect_pipeline_cfgs('config')
args = [
'config/calwebb_image3.cfg',
asn_file,
'--steps.tweakreg.skip=True',
]
Step.from_cmdline(args)
outputs = [(# Compare level-2c crf product
'nrca5_47Tuc_subpix_dither1_newpos_a3001_crf.fits',
'nrca5_47Tuc_subpix_dither1_newpos_cal-a3001_ref.fits'),
{'files':(# Compare i2d product
'mosaic_long_i2d.fits',
'mosaic_long_i2d_ref.fits'),
'pars': {'ignore_keywords':
self.ignore_keywords+['NAXIS1', 'TFORM*'],
'ignore_fields':self.ignore_keywords,
'rtol': 0.0001}
},
('mosaic_long_cat.ecsv', 'mosaic_long_cat_ref.ecsv'),
]
self.compare_outputs(outputs)
def run_pipeline(rtdata_module):
"""Run the calwebb_spec2 pipeline on a NIRSpec lamp exposure."""
rtdata = rtdata_module
# Get the input exposure
rtdata.get_data('nirspec/lamp/jw00626030001_02103_00005_nrs1_rate.fits')
# Run the calwebb_spec2 pipeline; save results from intermediate steps
args = [
"jwst.pipeline.Spec2Pipeline",
rtdata.input,
"--steps.assign_wcs.save_results=true",
"--steps.msa_flagging.save_results=true",
"--steps.flat_field.save_results=true",
# Hardwire extract1d ref file selection until CRDS-525 is implemented
"--steps.extract_1d.override_extract1d=/grp/crds/jwst/references/jwst/jwst_nirspec_extract1d_0001.asdf"
]
Step.from_cmdline(args)
return rtdata
def run_pipeline(jail, rtdata_module):
"""Run the calwebb_spec2 pipeline on an ASN of nodded MIRI LRS
fixedslit exposures."""
rtdata = rtdata_module
# Get the spec2 ASN and its members
rtdata.get_data("miri/lrs/MIRI_FM_MIRIMAGE_P750L_PATHLOSS_8C.00.00.fits")
rtdata.get_asn("miri/lrs/jw00623-o032_20191210t195246_spec2_001_asn.json")
# Run the calwebb_spec2 pipeline; save results from intermediate steps
args = ["calwebb_spec2", rtdata.input,
"--save_bsub=true",
"--steps.assign_wcs.save_results=true",
"--steps.flat_field.save_results=true",
"--steps.srctype.save_results=true",
"--steps.pathloss.save_results=true",
"--steps.pathloss.override_pathloss=MIRI_FM_MIRIMAGE_P750L_PATHLOSS_8C.00.00.fits",
"--steps.bkg_subtract.save_combined_background=true"
]
Step.from_cmdline(args)
def test_set_input_dir(mk_tmp_dirs):
"""Simply set the path"""
input_file = t_path('data/flat.fits')
step = Step.from_cmdline([
'jwst.stpipe.tests.steps.StepWithModel', input_file, '--input_dir',
'junkdir'
])
# Check that `input_dir` is set.
assert step.input_dir == 'junkdir'
def test_cube_build_miri_internal_cal(rtdata, fitsdiff_default_kwargs):
"""Run cube_build on single file using coord system = internal_cal"""
input_file = 'det_image_seq2_MIRIFUSHORT_12SHORTexp1_cal.fits'
rtdata.get_data(f"miri/mrs/{input_file}")
args = [
'jwst.cube_build.CubeBuildStep',
input_file,
'--save_results=true',
'--coord_system=internal_cal'
]
Step.from_cmdline(args)
output = input_file.replace('cal', 'ch1-short_internal_s3d')
rtdata.output = output
rtdata.get_truth(f"truth/test_miri_cubebuild/{output}")
diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs)
assert diff.identical, diff.report()
def run_detector1(rtdata_module):
"""Run detector1 pipeline on MIRI imaging data."""
rtdata = rtdata_module
rtdata.get_data("miri/image/det_image_1_MIRIMAGE_F770Wexp1_5stars_uncal.fits")
# Run detector1 pipeline only on one of the _uncal files
args = ["jwst.pipeline.Detector1Pipeline", rtdata.input,
"--save_calibrated_ramp=True",
"--steps.dq_init.save_results=True",
"--steps.saturation.save_results=True",
"--steps.refpix.save_results=True",
"--steps.rscd.save_results=True",
"--steps.lastframe.save_results=True",
"--steps.firstframe.save_results=True",
"--steps.reset.save_results=True",
"--steps.linearity.save_results=True",
"--steps.dark_current.save_results=True",
"--steps.jump.rejection_threshold=200",
]
Step.from_cmdline(args)
def test_residual_fringe_cal(rtdata, fitsdiff_default_kwargs):
"""Run residual fringe correction on MIRI IFUShort """
input_file = 'MIRM108-SHORT-6021192005_1_495_MIRIFUSHORT_cal.fits'
rtdata.get_data(f"miri/mrs/{input_file}")
args = [
'jwst.residual_fringe.ResidualFringeStep',
input_file,
'--save_results=true',
]
Step.from_cmdline(args)
output = input_file.replace('cal', 'residual_fringe')
rtdata.output = output
rtdata.get_truth(f"truth/test_miri_residual_fringe/{output}")
diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs)
assert diff.identical, diff.report()
def test_pipeline_commandline_invalid_args():
from io import StringIO
args = [
'jwst.stpipe.tests.test_pipeline.MyPipeline',
# The file_name parameters are *required*, and one of them
# is missing, so we should get a message to that effect
# followed by the commandline usage message.
'--flat_filename={0}'.format(
abspath(join(dirname(__file__), 'data', 'flat.fits'))),
'--steps.flat_field.threshold=47'
]
sys.stdout = buffer = StringIO()
with pytest.raises(ValueError):
Step.from_cmdline(args)
help = buffer.getvalue()
assert "Multiply by this number" in help
def test_nrs_fs_brightobj_spec2(self):
"""
Regression test of calwebb_spec2 pipeline performed on NIRSpec
fixed-slit data that uses the NRS_BRIGHTOBJ mode (S1600A1 slit).
"""
input_file = self.get_data(self.test_dir,
'jw84600042001_02101_00001_nrs2_rateints.fits')
collect_pipeline_cfgs()
args = [
'calwebb_tso-spec2.cfg',
input_file
]
Step.from_cmdline(args)
outputs = [('jw84600042001_02101_00001_nrs2_calints.fits',
'jw84600042001_02101_00001_nrs2_calints_ref.fits'),
('jw84600042001_02101_00001_nrs2_x1dints.fits',
'jw84600042001_02101_00001_nrs2_x1dints_ref.fits')
]
self.compare_outputs(outputs)
def test_nirspec_assignwcs_skip(_jail, rtdata, fitsdiff_default_kwargs,
caplog):
"""
Test of calwebb_spec2 pipeline performed on NIRSpec MSA exposure
with the AssignWcs step skipped. The pipeline should abort.
"""
# Get the input file
rtdata.get_data(
'nirspec/mos/f170lp-g235m_mos_observation-6-c0e0_001_dn_nrs1_mod.fits')
# Run the calwebb_spec2 pipeline
collect_pipeline_cfgs("config")
args = [
"config/calwebb_spec2.cfg", rtdata.input,
'--steps.assign_wcs.skip=True'
]
Step.from_cmdline(args)
assert 'Aborting remaining processing for this exposure.' in caplog.text
def test_default_input_dir(mk_tmp_dirs):
"""Test defaults"""
input_file = t_path('data/flat.fits')
step = Step.from_cmdline([
'jwst.stpipe.tests.steps.StepWithModel',
input_file
])
# Check that `input_dir` is set.
input_path = path.split(input_file)[0]
assert step.input_dir == input_path
def run_pipeline(jail, rtdata_module):
"""Run calwebb_detector1 pipeline on MIRI imaging data."""
rtdata = rtdata_module
rtdata.get_data("miri/image/jw00001001001_01101_00001_MIRIMAGE_uncal.fits")
collect_pipeline_cfgs("config")
args = [
"config/calwebb_detector1.cfg", rtdata.input,
"--steps.dq_init.save_results=True",
"--steps.lastframe.save_results=True",
"--steps.firstframe.save_results=True",
"--steps.saturation.save_results=True",
"--steps.rscd.save_results=True",
"--steps.linearity.save_results=True",
"--steps.dark_current.save_results=True",
"--steps.refpix.save_results=True",
"--steps.jump.rejection_threshold=25", "--steps.jump.save_results=True"
]
Step.from_cmdline(args)
return rtdata
def test_use_input_dir(mk_tmp_dirs):
"""Test with a specified path"""
input_dir = t_path('data')
input_file = 'flat.fits'
step = Step.from_cmdline([
'jwst.stpipe.tests.steps.StepWithModel', input_file, '--input_dir',
input_dir
])
# Check that `input_dir` is set.
assert step.input_dir == input_dir
def test_step_from_commandline_class_alias(mock_stpipe_entry_points):
args = ['stpipe_dummy', '--par1=58', '--par2=hij klm']
step = Step.from_cmdline(args)
assert isinstance(step, AnotherDummyStep)
assert step.par1 == 58.
assert step.par2 == 'hij klm'
assert step.par3 is False
step.run(1, 2)
def test_miri_mrs_extract1d_center(rtdata, fitsdiff_default_kwargs):
"""Test running extract_1d on an s3d cube containing a point source with user-supplied center"""
rtdata.get_data(
"miri/mrs/miri_003_det_image_seq1_MIRIFUSHORT_12SHORTexp1_s3d.fits")
args = [
'jwst.extract_1d.Extract1dStep', rtdata.input,
'--output_file=miri_003_det_image_seq1_MIRIFUSHORT_12SHORTexp1_center',
'--center_xy=28,28'
]
Step.from_cmdline(args)
rtdata.output = "miri_003_det_image_seq1_MIRIFUSHORT_12SHORTexp1_center_extract1dstep.fits"
# Get the truth file
rtdata.get_truth(
'truth/test_miri_mrs_extract1d/miri_003_det_image_seq1_MIRIFUSHORT_12SHORTexp1_center_extract1dstep.fits'
)
# Compare the results
diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs)
assert diff.identical, diff.report()
def test_pipeline_commandline(_jail):
args = [
abspath(join(dirname(__file__), 'steps', 'python_pipeline.cfg')),
'--steps.flat_field.threshold=47'
]
pipe = Step.from_cmdline(args)
assert pipe.flat_field.threshold == 47.0
assert pipe.flat_field.multiplier == 2.0
pipe.run()
def run_detector1pipeline(rtdata_module, jail):
"""Run calwebb_detector1 pipeline on NIRSpec data with IRS2 readout mode."""
rtdata = rtdata_module
rtdata.get_data("nirspec/irs2/jw0010010_11010_nrs1_chimera_uncal.fits")
collect_pipeline_cfgs("config")
Step.from_cmdline([
"config/calwebb_detector1.cfg",
rtdata.input,
"--steps.dq_init.save_results=True",
"--steps.saturation.save_results=True",
"--steps.superbias.save_results=True",
"--steps.refpix.save_results=True",
"--steps.rscd.save_results=True",
"--steps.linearity.save_results=True",
"--steps.dark_current.save_results=True",
"--steps.jump.save_results=True",
"--steps.ramp_fit.save_results=True",
"--steps.gain_scale.save_results=True",
"--steps.jump.rejection_threshold=200",
])
def run_tso_spec2(jail, rtdata_module):
"""Run stage 2 pipeline on NIRISS SOSS data."""
rtdata = rtdata_module
# Run tso-spec2 pipeline on the first _rateints file, saving intermediate products
rtdata.get_data(
"niriss/soss/jw00625023001_03101_00001-seg001_nis_rateints.fits")
args = [
"calwebb_spec2",
rtdata.input,
"--steps.flat_field.save_results=True",
"--steps.srctype.save_results=True",
]
Step.from_cmdline(args)
# Run tso-spec2 pipeline on the second _rateints file, without saving or
# checking any results (simply create a fresh input for level-3 test)
rtdata.get_data(
"niriss/soss/jw00625023001_03101_00001-seg002_nis_rateints.fits")
args = ["calwebb_spec2", rtdata.input]
Step.from_cmdline(args)
def run_detector1pipeline(jail, rtdata_module):
"""Run calwebb_detector1 on NIRCam imaging long data"""
rtdata = rtdata_module
rtdata.get_data("nircam/image/jw42424001001_01101_00001_nrca5_uncal.fits")
collect_pipeline_cfgs("config")
# Run detector1 pipeline only on one of the _uncal files
args = [
"config/calwebb_detector1.cfg",
rtdata.input,
"--steps.dq_init.save_results=True",
"--steps.saturation.save_results=True",
"--steps.superbias.save_results=True",
"--steps.refpix.save_results=True",
"--steps.linearity.save_results=True",
"--steps.dark_current.save_results=True",
"--steps.jump.save_results=True",
"--steps.jump.rejection_threshold=50.0",
]
Step.from_cmdline(args)
def run_tso_spec2_pipeline(jail, rtdata_module, request):
"""Run the calwebb_spec2 pipeline performed on NIRSpec
fixed-slit data that uses the NRS_BRIGHTOBJ mode (S1600A1 slit)
"""
rtdata = rtdata_module
# Get the input exposure
rtdata.get_data('nirspec/tso/jw84600042001_02101_00001_nrs2_rateints.fits')
# Run the calwebb_spec2 pipeline;
collect_pipeline_cfgs("config")
args = ["config/calwebb_tso-spec2.cfg", rtdata.input,
"--steps.assign_wcs.save_results=True",
"--steps.extract_2d.save_results=True",
"--steps.wavecorr.save_results=True",
"--steps.flat_field.save_results=True",
"--steps.photom.save_results=True"]
Step.from_cmdline(args)
return rtdata
def run_pipeline(jail, rtdata_module):
"""
Run the calwebb_spec3 pipeline on NIRSpec Fixed-Slit exposures.
"""
rtdata = rtdata_module
# Get the cfg files
collect_pipeline_cfgs("config")
# Get the ASN file and input exposures
rtdata.get_asn('nirspec/fs/jw93045-o010_20180725t035735_spec3_001_asn.json')
# Run the calwebb_spec3 pipeline; save results from intermediate steps
args = ["config/calwebb_spec3.cfg", rtdata.input,
"--steps.outlier_detection.save_results=true",
"--steps.resample_spec.save_results=true",
"--steps.extract_1d.save_results=true"]
Step.from_cmdline(args)
return rtdata
def run_spec2(jail, rtdata_module):
"""Run the Spec2Pipeline on a single exposure"""
rtdata = rtdata_module
collect_pipeline_cfgs("config")
# Setup the inputs
file_name = 'jw80600018001_02101_00003_mirifushort_rateints.fits'
rtdata.get_data(INPUT_PATH + '/' + file_name)
# Run the pipeline
args = [
"config/calwebb_tso-spec2.cfg",
rtdata.input,
'--steps.assign_wcs.save_results=true',
'--steps.flat_field.save_results=true',
'--steps.srctype.save_results=true',
'--steps.fringe.save_results=true',
'--steps.photom.save_results=true',
]
Step.from_cmdline(args)
def test_saving_pars(tmpdir):
"""Save the step parameters from the commandline"""
cfg_path = t_path(join('steps',
'jwst_generic_pars-makeliststep_0002.asdf'))
saved_path = tmpdir.join('savepars.asdf')
step = Step.from_cmdline([cfg_path, '--save-parameters', str(saved_path)])
assert saved_path.check()
with datamodels.StepParsModel(str(saved_path)) as saved:
assert saved.parameters == ParsModelWithPar3.parameters
step.closeout()
def run_pipelines(jail, rtdata_module):
"""Run stage 2-3 pipelines on NIRISS SOSS data."""
rtdata = rtdata_module
collect_pipeline_cfgs("config")
# Run tso-spec2 pipeline on the first _rateints file, saving intermediate products
rtdata.get_data(
"niriss/soss/jw00625023001_03101_00001-seg001_nis_rateints.fits")
args = [
"config/calwebb_tso-spec2.cfg",
rtdata.input,
"--steps.flat_field.save_results=True",
"--steps.srctype.save_results=True",
]
Step.from_cmdline(args)
# Run tso-spec2 pipeline on the second _rateints file, without saving or
# checking any results (simply create a fresh input for level-3 test)
rtdata.get_data(
"niriss/soss/jw00625023001_03101_00001-seg002_nis_rateints.fits")
args = ["config/calwebb_tso-spec2.cfg", rtdata.input]
Step.from_cmdline(args)
# Get the level3 assocation json file (though not its members) and run
# the tso3 pipeline on all _calints files listed in association
rtdata.get_data(
"niriss/soss/jw00625-o023_20191210t204036_tso3_001_asn.json")
args = ["config/calwebb_tso3.cfg", rtdata.input]
Step.from_cmdline(args)
return rtdata
def test_nirisssoss2pipeline1(_bigdata):
"""
Regression test of calwebb_tso_spec2 pipeline performed on NIRISS SOSS data.
"""
collect_pipeline_cfgs()
args = [
'calwebb_tso_spec2.cfg',
op.join(
_bigdata,
'pipelines/jw10003001002_03101_00001-seg003_nis_rateints.fits')
]
Step.from_cmdline(args)
# Compare the _calints products
n_cr = 'jw10003001002_03101_00001-seg003_nis_calints.fits'
n_ref = _bigdata + '/pipelines/jw10003001002_03101_00001-seg003_nis_calints_ref.fits'
h = pf.open(n_cr)
href = pf.open(n_ref)
result = pf.diff.FITSDiff(
h,
href,
ignore_hdus=['INT_TIMES', 'VAR_POISSON', 'VAR_RNOISE', 'ASDF'],
ignore_keywords=['DATE', 'CAL_VER', 'CAL_VCS', 'CRDS_VER', 'CRDS_CTX'],
rtol=0.00001)
assert result.identical, result.report()
# Compare the _x1dints products
n_cr = 'jw10003001002_03101_00001-seg003_nis_x1dints.fits'
n_ref = _bigdata + '/pipelines/jw10003001002_03101_00001-seg003_nis_x1dints_ref.fits'
h = pf.open(n_cr)
href = pf.open(n_ref)
result = pf.diff.FITSDiff(
h,
href,
ignore_hdus=['INT_TIMES', 'ASDF'],
ignore_keywords=['DATE', 'CAL_VER', 'CAL_VCS', 'CRDS_VER', 'CRDS_CTX'],
rtol=0.00001)
assert result.identical, result.report()
def test_step_from_commandline_par_precedence(command_line_pars, command_line_config_pars,
reference_pars, expected_pars, tmp_path, monkeypatch):
args = []
class_name = "jwst.stpipe.tests.steps.WithDefaultsStep"
config_name = "WithDefaultsStep"
reference_type = f"pars-{config_name.lower()}"
input_path = join(dirname(__file__), "data", "science.fits")
if command_line_config_pars:
command_line_config_path = tmp_path/"with_defaults_step.cfg"
config = ConfigObj(str(command_line_config_path))
config["class"] = class_name
config["name"] = config_name
for key, value in command_line_config_pars.items():
config[key] = value
config.write()
args.append(str(command_line_config_path.absolute()))
else:
args.append(class_name)
args.append(input_path)
if command_line_pars:
for key, value in command_line_pars.items():
args.append(f"--{key}={value}")
reference_file_map = {}
if reference_pars:
reference_path = tmp_path/f"{reference_type}.asdf"
parameters = {
"class": class_name,
"name": config_name,
}
parameters.update(reference_pars)
model = datamodels.StepParsModel({"parameters": parameters})
model.save(reference_path)
reference_file_map[reference_type] = str(reference_path)
def mock_get_reference_file(dataset, reference_file_type, observatory=None, asn_exptypes=None):
if reference_file_type in reference_file_map:
return reference_file_map[reference_file_type]
else:
raise CrdsLookupError(f"Error determining best reference for '{reference_file_type}' = \
Unknown reference type '{reference_file_type}'")
monkeypatch.setattr(crds_client, "get_reference_file", mock_get_reference_file)
step = Step.from_cmdline(args)
for key, value in expected_pars.items():
assert getattr(step, key) == value
def test_nirisssoss2pipeline1(self):
"""
Regression test of calwebb_tso_spec2 pipeline performed on NIRISS SOSS data.
"""
input_file = self.get_data(self.test_dir,
'jw10003001002_03101_00001-seg003_nis_rateints.fits')
collect_pipeline_cfgs()
args = [
'calwebb_tso-spec2.cfg',
input_file
]
Step.from_cmdline(args)
outputs = [{'files':('jw10003001002_03101_00001-seg003_nis_calints.fits',
'jw10003001002_03101_00001-seg003_nis_calints_ref.fits'),
'pars':dict(ignore_hdus=['INT_TIMES', 'VAR_POISSON',
'VAR_RNOISE', 'ASDF'])},
{'files':('jw10003001002_03101_00001-seg003_nis_x1dints.fits',
'jw10003001002_03101_00001-seg003_nis_x1dints_ref.fits'),
'pars':dict(ignore_hdus=['INT_TIMES', 'ASDF'])}
]
self.compare_outputs(outputs)
def test_step_from_commandline():
args = [
abspath(join(dirname(__file__), 'steps', 'some_other_step.cfg')),
'--par1=58', '--par2=hij klm'
]
step = Step.from_cmdline(args)
assert step.par1 == 58.
assert step.par2 == 'hij klm'
assert step.par3 is True
step.run(1, 2)
def test_step_from_commandline_class():
args = [
'jwst.stpipe.tests.steps.AnotherDummyStep', '--par1=58',
'--par2=hij klm'
]
step = Step.from_cmdline(args)
assert step.par1 == 58.
assert step.par2 == 'hij klm'
assert step.par3 is False
step.run(1, 2)
def test_image3_pipeline2(self):
"""Regression test definitions for CALIMAGE3 pipeline.
Regression test of calwebb_image3 pipeline on NIRCam
simulated long-wave data with a 6-point dither.
"""
asn_file = self.get_data(
self.test_dir, "jw10002-o001_20171116t191235_image3_002_asn.json")
for file in raw_from_asn(asn_file):
self.get_data(self.test_dir, file)
collect_pipeline_cfgs('config')
args = [
'config/calwebb_image3.cfg', asn_file,
'--steps.tweakreg.kernel_fwhm=2',
'--steps.tweakreg.snr_threshold=5',
'--steps.tweakreg.enforce_user_order=True',
'--steps.tweakreg.searchrad=10',
'--steps.tweakreg.fitgeometry=rscale'
]
Step.from_cmdline(args)
outputs = [('jw10002001001_01101_00004_nrcblong_o001_crf.fits',
'jw10002001001_01101_00004_nrcblong_o001_crf_ref.fits'), {
'files':
('jw10002-o001_t002_nircam_f444w_i2d.fits',
'jw10002-o001_t002_nircam_f444w_i2d_ref.fits'),
'pars': {
'ignore_keywords':
self.ignore_keywords + ['NAXIS1', 'TFORM*'],
'ignore_fields':
self.ignore_keywords,
'rtol':
0.0001
}
}]
self.compare_outputs(outputs)
def test_save_proper_pipeline_container_substeps_withdir(mk_tmp_dirs):
"""Test how pipeline saving should work"""
tmp_current_path, tmp_data_path, tmp_config_path = mk_tmp_dirs
args = [
'jwst.stpipe.tests.steps.ProperPipeline',
data_fn_path,
'--output_dir=' + tmp_data_path,
'--steps.stepwithmodel.save_results=true',
'--steps.another_stepwithmodel.save_results=true',
'--steps.another_stepwithmodel.output_dir=' + tmp_config_path,
'--steps.stepwithcontainer.save_results=true',
]
Step.from_cmdline(args)
assert path.isfile(path.join(tmp_data_path, 'flat_0_pp.fits'))
assert path.isfile(path.join(tmp_data_path, 'flat_1_pp.fits'))
assert path.isfile(path.join(tmp_data_path, 'flat_swm.fits'))
assert path.isfile(path.join(tmp_config_path, 'flat_aswm.fits'))
assert path.isfile(path.join(tmp_data_path, 'flat_0_swc.fits'))
assert path.isfile(path.join(tmp_data_path, 'flat_1_swc.fits'))
def run_pipelines(jail, rtdata_module):
"""Run stage 2 and 3 pipelines on NIRCam TSO image data."""
rtdata = rtdata_module
# Run the calwebb_tso-image2 pipeline on each of the 2 inputs
rate_files = [
"nircam/tsimg/jw00312006001_02102_00001-seg001_nrcb1_rateints.fits",
"nircam/tsimg/jw00312006001_02102_00001-seg002_nrcb1_rateints.fits"
]
for rate_file in rate_files:
rtdata.get_data(rate_file)
args = ["calwebb_image2", rtdata.input]
Step.from_cmdline(args)
# Get the level3 association json file (though not its members) and run
# the tso3 pipeline on all _calints files listed in association
rtdata.get_data("nircam/tsimg/jw00312-o006_20191225t115310_tso3_001_asn.json")
args = ["calwebb_tso3", rtdata.input]
Step.from_cmdline(args)
return rtdata
def test_nirisssoss2pipeline1(self):
"""
Regression test of calwebb_tso_spec2 pipeline performed on NIRISS SOSS data.
"""
input_file = self.get_data(self.test_dir,
'jw10003001002_03101_00001-seg003_nis_rateints.fits')
collect_pipeline_cfgs()
args = [
'calwebb_tso-spec2.cfg',
input_file
]
Step.from_cmdline(args)
outputs = [{'files':('jw10003001002_03101_00001-seg003_nis_calints.fits',
'jw10003001002_03101_00001-seg003_nis_calints_ref.fits'),
'pars':dict(ignore_hdus=['INT_TIMES', 'VAR_POISSON',
'VAR_RNOISE', 'ASDF'])},
{'files':('jw10003001002_03101_00001-seg003_nis_x1dints.fits',
'jw10003001002_03101_00001-seg003_nis_x1dints_ref.fits'),
'pars':dict(ignore_hdus=['INT_TIMES', 'ASDF'])}
]
self.compare_outputs(outputs)
