def test_image3_pipeline1(self):
asn_name = "mosaic_long_asn.json"
asn_file = self.get_data('test_calimage3', asn_name)
for file in raw_from_asn(asn_file):
self.get_data('test_calimage3', file)
collect_pipeline_cfgs('config')
args = [
'config/calwebb_image3.cfg',
asn_file,
'--steps.tweakreg.skip=True',
]
Step.from_cmdline(args)
outputs = [(# Compare level-2c crf product
'nrca5_47Tuc_subpix_dither1_newpos_a3001_crf.fits',
'nrca5_47Tuc_subpix_dither1_newpos_cal-a3001_ref.fits'),
{'files':(# Compare i2d product
'mosaic_long_i2d.fits',
'mosaic_long_i2d_ref.fits'),
'pars': {'ignore_keywords':
self.ignore_keywords+['NAXIS1', 'TFORM*'],
'ignore_fields':self.ignore_keywords,
'rtol': 0.0001}
},
('mosaic_long_cat.ecsv', 'mosaic_long_cat_ref.ecsv'),
]
self.compare_outputs(outputs)
def test_wfs_image2(self):
"""
Regression test of the WFS&C `calwebb_wfs-image2.cfg` pipeline
"""
data_base = 'jw82600026001_02101_00001_nrca1_rate'
ext = '.fits'
input_name = '{}{}'.format(data_base, ext)
input_file = self.get_data(self.test_dir, 'sdp_jw82600_wfs', 'level2a',
input_name)
collect_pipeline_cfgs('cfgs')
Step.from_cmdline([os.path.join('cfgs', 'calwebb_wfs-image2.cfg'), input_file])
cal_name = input_name.replace('rate', 'cal')
output_name = input_name.replace('rate','cal_ref')
outputs = [(cal_name, output_name)]
self.compare_outputs(outputs)
output_files = glob.glob('*')
output_files.remove('cfgs')
# these would happen when docopy=True
if input_name in output_files:
output_files.remove(input_name)
if output_name in output_files:
output_files.remove(output_name)
assert cal_name in output_files
output_files.remove(cal_name)
assert not output_files, 'Unexpected output files {}'.format(output_files)
def test_nrs_fs_spec3(self):
"""
Regression test of calwebb_spec3 pipeline performed on
NIRSpec fixed-slit data.
"""
cfg_dir = './cfgs'
collect_pipeline_cfgs(cfg_dir)
datapath = ['test_datasets', 'fss', '93045', 'level2b']
asn_file = self.get_data(*datapath,
'jw93045-o010_20180725t035735_spec3_001_asn.json')
for file in raw_from_asn(asn_file):
self.get_data(*datapath, file)
args = [
path.join(cfg_dir, 'calwebb_spec3.cfg'),
asn_file
]
Step.from_cmdline(args)
# Compare results
outputs = [('jw00023001001_01101_00001_NRS1_cal.fits',
'jw00023001001_01101_00001_NRS1_cal_ref.fits'),
('jw00023001001_01101_00001_NRS1_s2d.fits',
'jw00023001001_01101_00001_NRS1_s2d_ref.fits'),
('jw00023001001_01101_00001_NRS1_x1d.fits',
'jw00023001001_01101_00001_NRS1_x1d_ref.fits')
]
self.compare_outputs(outputs)
def test_image3_pipeline2(self):
"""Regression test definitions for CALIMAGE3 pipeline.
Regression test of calwebb_image3 pipeline on NIRCam
simulated long-wave data with a 6-point dither.
"""
asn_file = self.get_data(self.test_dir,
"jw10002-o001_20171116t191235_image3_002_asn.json")
for file in raw_from_asn(asn_file):
self.get_data(self.test_dir, file)
collect_pipeline_cfgs('config')
args = [
'config/calwebb_image3.cfg',
asn_file,
'--steps.tweakreg.kernel_fwhm=2',
'--steps.tweakreg.snr_threshold=5',
'--steps.tweakreg.enforce_user_order=True',
'--steps.tweakreg.searchrad=10',
'--steps.tweakreg.fitgeometry=rscale'
]
Step.from_cmdline(args)
outputs = [('jw10002001001_01101_00004_nrcblong_o001_crf.fits',
'jw10002001001_01101_00004_nrcblong_o001_crf_ref.fits'),
{'files':('jw10002-o001_t002_nircam_f444w_i2d.fits',
'jw10002-o001_t002_nircam_f444w_i2d_ref.fits'),
'pars':{'ignore_keywords':self.ignore_keywords+['NAXIS1','TFORM*'],
'ignore_fields':self.ignore_keywords,
'rtol':0.0001}
}
]
self.compare_outputs(outputs)
def test_nrs_fs_brightobj_spec2(self):
"""
Regression test of calwebb_spec2 pipeline performed on NIRSpec
fixed-slit data that uses the NRS_BRIGHTOBJ mode (S1600A1 slit).
"""
input_file = self.get_data(self.test_dir,
'jw84600042001_02101_00001_nrs2_rateints.fits')
collect_pipeline_cfgs()
args = [
'calwebb_tso-spec2.cfg',
input_file
]
Step.from_cmdline(args)
outputs = [('jw84600042001_02101_00001_nrs2_calints.fits',
'jw84600042001_02101_00001_nrs2_calints_ref.fits'),
('jw84600042001_02101_00001_nrs2_x1dints.fits',
'jw84600042001_02101_00001_nrs2_x1dints_ref.fits')
]
self.compare_outputs(outputs)
def build_reffile_container_func(input_models, reftype):
"""
Return a ModelContainer of reference file models.
Parameters
----------
input_models: ModelContainer
the science data, ImageModels in a ModelContainer
reftype: string
type of reference file
Returns
-------
a ModelContainer with corresponding reference files for each input model
"""
"""
model_mapping = {'gain':'GainModel', 'readnoise':'ReadnoiseModel'}
module = importlib.import_module('jwst_lib.models')
model_name = model_mapping[reftype]
ref_model = getattr(module, model_name)
print("ref_model defined as: {} of type {}".format(ref_model, type(ref_model)))
"""
q = Step()
reffiles = [q.get_reference_file(im, reftype) for im in input_models]
# Check if all the ref files are the same. If so build it by reading
# the reference file just once.
if len(set(reffiles)) <= 1:
length = len(input_models)
ref_list = [models.open(reffiles[0])] * length
#ref_list = [ref_model(reffiles[0])] * length
else:
ref_list = [models.open(ref) for ref in reffiles]
#ref_list = [ref_model(ref) for ref in reffiles]
return models.ModelContainer(ref_list)
def test_nirisssoss2pipeline1(self):
"""
Regression test of calwebb_tso_spec2 pipeline performed on NIRISS SOSS data.
"""
input_file = self.get_data(self.test_dir,
'jw10003001002_03101_00001-seg003_nis_rateints.fits')
collect_pipeline_cfgs()
args = [
'calwebb_tso-spec2.cfg',
input_file
]
Step.from_cmdline(args)
outputs = [{'files':('jw10003001002_03101_00001-seg003_nis_calints.fits',
'jw10003001002_03101_00001-seg003_nis_calints_ref.fits'),
'pars':dict(ignore_hdus=['INT_TIMES', 'VAR_POISSON',
'VAR_RNOISE', 'ASDF'])},
{'files':('jw10003001002_03101_00001-seg003_nis_x1dints.fits',
'jw10003001002_03101_00001-seg003_nis_x1dints_ref.fits'),
'pars':dict(ignore_hdus=['INT_TIMES', 'ASDF'])}
]
self.compare_outputs(outputs)
def find_suffixes():
"""Find all possible suffixes from the jwst package
Returns
-------
suffixes: set
The set of all programmatically findable suffixes.
Notes
-----
This will load all of the `jwst` package. Consider if this
is worth doing dynamically or only as a utility to update
a static list.
"""
from jwst.stpipe import Step
suffixes = set()
jwst = import_module('jwst')
jwst_fpath = path.split(jwst.__file__)[0]
# First traverse the code base and find all
# `Step` classes. The default suffix is the
# class name.
for module in load_local_pkg(jwst_fpath):
for klass_name, klass in getmembers(
module,
lambda o: isclass(o) and issubclass(o, Step)
):
suffixes.add(klass_name.lower())
# Instantiate Steps/Pipelines from their configuration files.
# Different names and suffixes can be defined in this way.
# Note: Based on the `collect_pipeline_cfgs` script
config_path = path.join(jwst_fpath, 'pipeline')
for config_file in listdir(config_path):
if config_file.endswith('.cfg'):
try:
step = Step.from_config_file(
path.join(config_path, config_file)
)
except Exception as exception:
pass
else:
suffixes.add(step.name.lower())
if step.suffix is not None:
suffixes.add(step.suffix.lower())
# That's all folks
return list(suffixes)
def test_image3_pipeline2(self):
"""Regression test definitions for CALIMAGE3 pipeline.
Regression test of calwebb_image3 pipeline on NIRCam
simulated long-wave data with a 6-point dither.
"""
asn_file = self.get_data(self.test_dir,
"jw10002-o001_20171116t191235_image3_002_asn.json")
for file in raw_from_asn(asn_file):
self.get_data(self.test_dir, file)
collect_pipeline_cfgs('config')
args = [
'config/calwebb_image3.cfg',
asn_file,
'--steps.tweakreg.kernel_fwhm=2',
'--steps.tweakreg.snr_threshold=5',
'--steps.tweakreg.enforce_user_order=True',
'--steps.tweakreg.searchrad=10',
'--steps.tweakreg.fitgeometry=rscale'
]
Step.from_cmdline(args)
outputs = [('jw10002001001_01101_00004_nrcblong_o001_crf.fits',
'jw10002001001_01101_00004_nrcblong_o001_crf_ref.fits'),
{'files':('jw10002-o001_t002_nircam_f444w_i2d.fits',
'jw10002-o001_t002_nircam_f444w_i2d_ref.fits'),
'pars':{'ignore_keywords':self.ignore_keywords+['NAXIS1','TFORM*'],
'ignore_fields':self.ignore_keywords,
'rtol':0.0001}
},
('jw10002-o001_t002_nircam_f444w_cat.ecsv',
'jw10002-o001_t002_nircam_f444w_cat_ref.ecsv'),
]
self.compare_outputs(outputs)
def test_nrc_wfss_background(filters, pupils, detectors, make_wfss_datamodel):
"""Test background subtraction for NIRCAM WFSS modes."""
data = make_wfss_datamodel
data.meta.instrument.filter = filters
data.meta.instrument.pupil = pupils
data.meta.instrument.detector = detectors
data.meta.instrument.channel = 'LONG'
data.meta.instrument.name = 'NIRCAM'
data.meta.exposure.type = 'NRC_WFSS'
if data.meta.instrument.detector == 'NRCALONG':
data.meta.instrument.module = 'A'
elif data.meta.instrument.detector == 'NRCBLONG':
data.meta.instrument.module = 'B'
wcs_corrected = AssignWcsStep.call(data)
# Get References
wavelenrange = Step().get_reference_file(wcs_corrected, "wavelengthrange")
bkg_file = Step().get_reference_file(wcs_corrected, 'wfssbkg')
mask = mask_from_source_cat(wcs_corrected, wavelenrange)
with datamodels.open(bkg_file) as bkg_ref:
bkg_ref = no_NaN(bkg_ref)
# calculate backgrounds
pipeline_data_mean = robust_mean(wcs_corrected.data[mask])
test_data_mean, _, _ = sigma_clipped_stats(wcs_corrected.data, sigma=2)
pipeline_reference_mean = robust_mean(bkg_ref.data[mask])
test_reference_mean, _, _ = sigma_clipped_stats(bkg_ref.data, sigma=2)
assert np.isclose([pipeline_data_mean], [test_data_mean], rtol=1e-3)
assert np.isclose([pipeline_reference_mean], [test_reference_mean],
rtol=1e-1)
def run_pipelines(jail, rtdata_module):
"""Run stage 2 and 3 pipelines on NIRCam TSO image data."""
rtdata = rtdata_module
collect_pipeline_cfgs("config")
# Run the calwebb_tso-image2 pipeline on each of the 2 inputs
rate_files = [
"nircam/tsimg/jw00312006001_02102_00001-seg001_nrcb1_rateints.fits",
"nircam/tsimg/jw00312006001_02102_00001-seg002_nrcb1_rateints.fits"
]
for rate_file in rate_files:
rtdata.get_data(rate_file)
args = ["config/calwebb_tso-image2.cfg", rtdata.input]
Step.from_cmdline(args)
# Get the level3 assocation json file (though not its members) and run
# the tso3 pipeline on all _calints files listed in association
rtdata.get_data(
"nircam/tsimg/jw00312-o006_20191225t115310_tso3_001_asn.json")
args = ["config/calwebb_tso3.cfg", rtdata.input]
Step.from_cmdline(args)
return rtdata
def run_pipeline(rtdata_module, jail):
"""Run calwebb_Detector1 pipeline on NIRSpec uncal data. This test is for
data taken with the IRS2 mode. The steps are similar to the NIRCam level
2a test but the IRS2 mode tests different paths in the same modules."""
rtdata = rtdata_module
rtdata.get_data(
"nirspec/test_detector1Pipeline/jw0010010_11010_nrs1_chimera_uncal.fits"
)
Step.from_cmdline([
"jwst.pipeline.Detector1Pipeline", rtdata.input,
"--save_calibrated_ramp=True", "--steps.group_scale.save_results=True",
"--steps.dq_init.save_results=True",
"--steps.saturation.save_results=True",
"--steps.ipc.save_results=True", "--steps.superbias.save_results=True",
"--steps.refpix.save_results=True", "--steps.rscd.save_results=True",
"--steps.linearity.save_results=True",
"--steps.dark_current.save_results=True",
"--steps.jump.save_results=True", "--steps.ramp_fit.save_results=True",
"--steps.gain_scale.save_results=True",
"--steps.jump.rejection_threshold=20.0"
])
return rtdata
def test_nirisssoss2pipeline1(self):
"""
Regression test of calwebb_tso_spec2 pipeline performed on NIRISS SOSS data.
"""
input_file = self.get_data(
self.test_dir,
'jw10003001002_03101_00001-seg003_nis_rateints.fits')
collect_pipeline_cfgs()
args = ['calwebb_tso-spec2.cfg', input_file]
Step.from_cmdline(args)
outputs = [{
'files': ('jw10003001002_03101_00001-seg003_nis_calints.fits',
'jw10003001002_03101_00001-seg003_nis_calints_ref.fits'),
'pars':
dict(
ignore_hdus=['INT_TIMES', 'VAR_POISSON', 'VAR_RNOISE', 'ASDF'])
}, {
'files': ('jw10003001002_03101_00001-seg003_nis_x1dints.fits',
'jw10003001002_03101_00001-seg003_nis_x1dints_ref.fits'),
'pars':
dict(ignore_hdus=['INT_TIMES', 'ASDF'])
}]
self.compare_outputs(outputs)
def test_nirspec_fs_mbkg_user(rtdata, fitsdiff_default_kwargs):
"""Run a test for NIRSpec FS data with a user-supplied background file."""
# Get user-supplied background
user_background = "v2_nrs_bkg_user_clean_x1d.fits"
rtdata.get_data(f"nirspec/fs/{user_background}")
# Get input data
rtdata.get_data("nirspec/fs/nrs_sci+bkg_cal.fits")
collect_pipeline_cfgs("config")
args = ["config/master_background.cfg", rtdata.input,
"--user_background", user_background]
Step.from_cmdline(args)
output = "nrs_sci+bkg_master_background.fits"
rtdata.output = output
# Get the truth file
rtdata.get_truth(f"truth/test_nirspec_fs_mbkg_user/{output}")
# Compare the results
diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs)
assert diff.identical, diff.report()
def run_detector1(rtdata_module):
"""Run detector1 pipeline on MIRI imaging data."""
rtdata = rtdata_module
rtdata.get_data(
"miri/image/det_image_1_MIRIMAGE_F770Wexp1_5stars_uncal.fits")
collect_pipeline_cfgs("config")
# Run detector1 pipeline only on one of the _uncal files
args = [
"config/calwebb_detector1.cfg",
rtdata.input,
"--save_calibrated_ramp=True",
"--steps.dq_init.save_results=True",
"--steps.saturation.save_results=True",
"--steps.refpix.save_results=True",
"--steps.rscd.save_results=True",
"--steps.lastframe.save_results=True",
"--steps.firstframe.save_results=True",
"--steps.linearity.save_results=True",
"--steps.dark_current.save_results=True",
"--steps.jump.rejection_threshold=200",
]
Step.from_cmdline(args)
def run_pipeline(jail, rtdata_module, request):
"""Run the calwebb_spec2 pipeline on NIRSpec Fixed-Slit exposures.
We currently test the following types of inputs:
1) Full-frame exposure (all slits will be extracted)
2) ALLSLITS subarray exposure (all slits will be extracted)
3) S400A1 subarray exposure (1 slit extracted)"""
rtdata = rtdata_module
# Get the input exposure
rtdata.get_data('nirspec/fs/' + request.param + 'rate.fits')
# Run the calwebb_spec2 pipeline; save results from intermediate steps
args = [
"calwebb_spec2", rtdata.input, "--steps.assign_wcs.save_results=true",
"--steps.extract_2d.save_results=true",
"--steps.wavecorr.save_results=true",
"--steps.srctype.save_results=true",
"--steps.flat_field.save_results=true",
"--steps.pathloss.save_results=true"
]
Step.from_cmdline(args)
return rtdata
def run_pipeline(jail, rtdata_module):
"""Run the calwebb_spec2 pipeline on an ASN of nodded MIRI LRS
fixedslit exposures."""
rtdata = rtdata_module
# Get the cfg files
collect_pipeline_cfgs("config")
# Get the spec2 ASN and its members
rtdata.get_asn("miri/lrs/jw00623-o032_20191210t195246_spec2_001_asn.json")
# Run the calwebb_spec2 pipeline; save results from intermediate steps
# NOTE: THE RESAMPLE_SPEC STEP IS SKIPPED FOR NOW, BECAUSE IT HAS A BUG
# (the s2d image is all zeros)
args = ["config/calwebb_spec2.cfg", rtdata.input,
"--steps.resample_spec.skip=true", # remove when bug fixed
"--save_bsub=true",
"--steps.assign_wcs.save_results=true",
"--steps.flat_field.save_results=true",
"--steps.srctype.save_results=true"]
Step.from_cmdline(args)
return rtdata
def run_pipeline(rtdata_module):
"""Run the calwebb_spec2 pipeline on a single NIRSpec MOS exposure."""
rtdata = rtdata_module
# Get the MSA metadata file referenced in the input exposure
rtdata.get_data("nirspec/mos/jw95065006001_0_short_msa.fits")
# Get the input exposure
rtdata.get_data("nirspec/mos/f170lp-g235m_mos_observation-6-c0e0_001_dn_nrs1_mod.fits")
# Run the calwebb_spec2 pipeline; save results from intermediate steps
args = ["calwebb_spec2", rtdata.input,
"--steps.assign_wcs.save_results=true",
"--steps.msa_flagging.save_results=true",
"--steps.extract_2d.save_results=true",
"--steps.srctype.save_results=true",
"--steps.wavecorr.save_results=true",
"--steps.flat_field.save_results=true",
"--steps.pathloss.save_results=true",
"--steps.barshadow.save_results=true"]
Step.from_cmdline(args)
return rtdata
def test_step_from_commandline_par_precedence(command_line_pars, command_line_config_pars,
reference_pars, expected_pars, tmp_path, monkeypatch):
args = []
class_name = "jwst.stpipe.tests.steps.WithDefaultsStep"
config_name = "WithDefaultsStep"
reference_type = f"pars-{config_name.lower()}"
input_path = join(dirname(__file__), "data", "science.fits")
if command_line_config_pars:
command_line_config_path = tmp_path / "with_defaults_step.cfg"
config = ConfigObj(str(command_line_config_path))
config["class"] = class_name
config["name"] = config_name
for key, value in command_line_config_pars.items():
config[key] = value
config.write()
args.append(str(command_line_config_path.absolute()))
else:
args.append(class_name)
args.append(input_path)
if command_line_pars:
for key, value in command_line_pars.items():
args.append(f"--{key}={value}")
reference_file_map = {}
if reference_pars:
reference_path = tmp_path / f"{reference_type}.asdf"
reference_config = StepConfig(class_name, config_name, reference_pars, [])
with reference_config.to_asdf() as af:
af.write_to(reference_path)
reference_file_map[reference_type] = str(reference_path)
def mock_get_reference_file(dataset, reference_file_type, observatory=None, asn_exptypes=None):
if reference_file_type in reference_file_map:
return reference_file_map[reference_file_type]
else:
raise CrdsLookupError(f"Error determining best reference for '{reference_file_type}' = \
Unknown reference type '{reference_file_type}'")
monkeypatch.setattr(crds_client, "get_reference_file", mock_get_reference_file)
step = Step.from_cmdline(args)
for key, value in expected_pars.items():
assert getattr(step, key) == value
def test_save_container_withfile(mk_tmp_dirs):
"""Step with output_use_model"""
tmp_current_path, tmp_data_path, tmp_config_path = mk_tmp_dirs
args = [
'jwst.stpipe.tests.steps.StepWithContainer',
data_fn_path,
'--output_file=tscwf.fits',
]
step = Step.from_cmdline(args)
assert path.isfile('tscwf_0_stepwithcontainer.fits')
assert path.isfile('tscwf_1_stepwithcontainer.fits')
step.closeout()
def tmtrun():
if '--version' in sys.argv:
sys.stdout.write("%s\n" % iris_pipeline.__version__)
sys.exit(0)
try:
step = Step.from_cmdline(sys.argv[1:])
except NoDataOnDetectorError:
import traceback
traceback.print_exc()
sys.exit(64)
except Exception as e:
import traceback
traceback.print_exc()
sys.exit(1)
def test_save_step_withoutputsuffix(mk_tmp_dirs):
"""Default save should be current working directory"""
tmp_current_path, tmp_data_path, tmp_config_path = mk_tmp_dirs
output_file = 'junk_rate.fits'
actual_output_file = 'junk_stepwithmodel.fits'
args = [
'jwst.stpipe.tests.steps.StepWithModel', data_fn_path,
'--output_file=' + output_file
]
step = Step.from_cmdline(args)
step.closeout()
assert path.isfile(actual_output_file)
def test_save_step_withoutput(mk_tmp_dirs):
"""Default save should be current working directory"""
tmp_current_path, tmp_data_path, tmp_config_path = mk_tmp_dirs
output_file = 'junk.fits'
args = [
'jwst.stpipe.tests.steps.StepWithModel', data_fn_path,
'--output_file=' + output_file
]
step = Step.from_cmdline(args)
step.closeout()
output_path, output_ext = path.splitext(output_file)
assert path.isfile(output_path + '_stepwithmodel' + output_ext)
def find_suffixes():
"""Find all possible suffixes from the jwst package
Returns
-------
suffixes: set
The set of all programmatically findable suffixes.
Notes
-----
This will load all of the `jwst` package. Consider if this
is worth doing dynamically or only as a utility to update
a static list.
"""
from jwst.stpipe import Step
suffixes = set()
jwst = import_module('jwst')
jwst_fpath = path.split(jwst.__file__)[0]
# First traverse the code base and find all
# `Step` classes. The default suffix is the
# class name.
for module in load_local_pkg(jwst_fpath):
for klass_name, klass in getmembers(
module, lambda o: isclass(o) and issubclass(o, Step)):
suffixes.add(klass_name.lower())
# Instantiate Steps/Pipelines from their configuration files.
# Different names and suffixes can be defined in this way.
# Note: Based on the `collect_pipeline_cfgs` script
config_path = path.join(jwst_fpath, 'pipeline')
for config_file in listdir(config_path):
if config_file.endswith('.cfg'):
try:
step = Step.from_config_file(
path.join(config_path, config_file))
except Exception:
pass
else:
suffixes.add(step.name.lower())
if step.suffix is not None:
suffixes.add(step.suffix.lower())
# That's all folks
return list(suffixes)
def test_save_step_withdir(mk_tmp_dirs):
"""Save to specified folder"""
tmp_current_path, tmp_data_path, tmp_config_path = mk_tmp_dirs
args = [
'jwst.stpipe.tests.steps.StepWithModel', data_fn_path,
'--output_dir=' + tmp_data_path
]
step = Step.from_cmdline(args)
step.closeout()
output_fn_path = path.join(
tmp_data_path,
data_name + '_stepwithmodel' + data_ext,
)
assert path.isfile(output_fn_path)
def test_pipeline_commandline_class(_jail):
args = [
'jwst.stpipe.tests.test_pipeline.MyPipeline',
'--logcfg={0}'.format(
abspath(join(dirname(__file__), 'steps', 'log.cfg'))),
# The file_name parameters are *required*
'--science_filename={0}'.format(
abspath(join(dirname(__file__), 'data', 'science.fits'))),
'--output_filename={0}'.format('output.fits'),
'--steps.flat_field.threshold=47'
]
pipe = Step.from_cmdline(args)
assert pipe.flat_field.threshold == 47.0
assert pipe.flat_field.multiplier == 1.0
pipe.run()
def test_save_step_withdir_withoutput(mk_tmp_dirs):
"""Save to specified folder"""
tmp_current_path, tmp_data_path, tmp_config_path = mk_tmp_dirs
output_file = 'junk.fits'
args = [
'jwst.stpipe.tests.steps.StepWithModel', data_fn_path,
'--output_dir=' + tmp_data_path, '--output_file=' + output_file
]
step = Step.from_cmdline(args)
step.closeout()
output_path, output_ext = path.splitext(output_file)
output_fn_path = path.join(tmp_data_path,
output_path + '_stepwithmodel' + output_ext)
assert path.isfile(output_fn_path)
def test_create_slitlets():
"""Test that slitlets are Slit type and have all the necessary fields"""
dm = ImageModel()
dm.meta.instrument.name = 'NIRSPEC'
dm.meta.observation.date = '2016-09-05'
dm.meta.observation.time = '8:59:37'
msa_oper = Step().get_reference_file(dm, 'msaoper')
result = create_slitlets(dm, msa_oper)
slit_fields = ('name', 'shutter_id', 'dither_position', 'xcen', 'ycen',
'ymin', 'ymax', 'quadrant', 'source_id', 'shutter_state',
'source_name', 'source_alias', 'stellarity', 'source_xpos',
'source_ypos', 'source_ra', 'source_dec')
for slit in result:
# Test the returned data type and fields.
assert type(slit) == Slit
assert slit._fields == slit_fields
def test_get_failed_open_shutters():
"""test that failed open shutters are returned from reference file"""
# Set up data model to retrieve reference file
dm = ImageModel()
dm.meta.instrument.name = 'NIRSPEC'
dm.meta.observation.date = '2016-09-05'
dm.meta.observation.time = '8:59:37'
# Get reference file and return all failed open shutters
msa_oper = Step().get_reference_file(dm, 'msaoper')
result = get_failed_open_shutters(msa_oper)
# get_failed_open_shutters returns 3 flaggable states
# state, Internal state, and TA state.
for shutter in result:
assert shutter['state'] == 'open' \
or shutter['Internal state'] == 'open' \
or shutter['TA state'] == 'open'
def test_saving_pars(tmpdir):
"""Save the step parameters from the commandline"""
cfg_path = t_path(join('steps', 'jwst_generic_pars-makeliststep_0002.asdf'))
saved_path = tmpdir.join('savepars.asdf')
step = Step.from_cmdline([
cfg_path,
'--save-parameters',
str(saved_path)
])
assert saved_path.check()
with asdf.open(t_path(join('steps', 'jwst_generic_pars-makeliststep_0002.asdf'))) as af:
original_config = StepConfig.from_asdf(af)
original_config.parameters["par3"] = False
with asdf.open(str(saved_path)) as af:
config = StepConfig.from_asdf(af)
assert config.parameters == original_config.parameters
step.closeout()
def run_pipelines(jail, rtdata_module):
"""Run the calwebb_wfs-image2 and calwebb_wfs-image3 pipelines
on NIRCam WFS&C images. The calwebb_wfs-image3 pipeline is
run twice: once with default params and a second time with
the do_refine option turned on."""
rtdata = rtdata_module
# Get the cfg files
collect_pipeline_cfgs("config")
# Get the image2 ASN and members
rtdata.get_asn(
"nircam/wavefront/jw00632-o003_20191210t194815_wfs-image2_001_asn.json"
)
# Run the calwebb_wfs-image2 pipeline
args = ["jwst.pipeline.Image2Pipeline", rtdata.input]
Step.from_cmdline(args)
# Get the image3 ASN file only, not the members. The members were just
# produced by the calwebb_wfs-image2 pipeline run.
rtdata.get_data(
"nircam/wavefront/jw00632-o003_20191210t194815_wfs-image3_001_asn.json"
)
# Run the calwebb_wfs-image3 pipeline with no refinement (default)
args = ["config/calwebb_wfs-image3.cfg", rtdata.input]
Step.from_cmdline(args)
# Get the second version of the image3 ASN, which has a unique output
# product name. Only get the ASN file, not the members.
rtdata.get_data(
"nircam/wavefront/jw00632-o003_20191210t194815_wfs-image3_002_asn.json"
)
# Run the calwebb_wfs-image3 pipeline with refinement
args = ["config/calwebb_wfs-image3.cfg", rtdata.input, "--do_refine=True"]
Step.from_cmdline(args)
return rtdata
def test_reftype(cfg_file, expected):
"""Test that reftype is produce as expected"""
step = Step.from_config_file(t_path(join('steps', cfg_file)))
assert step.get_pars_model().meta.reftype == 'pars-' + expected.lower()
def run_image3_closedfile(rtdata, jail):
"""Run calwebb_image3 on NIRCam imaging with data that had a closed file issue."""
rtdata.get_asn("nircam/image/fail_short_image3_asn.json")
args = ["calwebb_image3", rtdata.input]
Step.from_cmdline(args)
def test_print_configspec():
step = Step()
step.print_configspec()
def test_step_from_commandline_invalid4():
args = ['sys.argv']
with pytest.raises(ValueError):
Step.from_cmdline(args)
def test_step_from_commandline_invalid2():
args = ['__foo__.__bar__']
with pytest.raises(ValueError):
Step.from_cmdline(args)
def run_pipelines(jail, rtdata_module):
"""Run stage 1-3 pipelines on MIRI imaging data."""
rtdata = rtdata_module
rtdata.get_data(
"miri/image/det_image_1_MIRIMAGE_F770Wexp1_5stars_uncal.fits")
collect_pipeline_cfgs("config")
# Run detector1 pipeline only on one of the _uncal files
args = [
"config/calwebb_detector1.cfg",
rtdata.input,
"--save_calibrated_ramp=True",
"--steps.dq_init.save_results=True",
"--steps.saturation.save_results=True",
"--steps.refpix.save_results=True",
"--steps.rscd.save_results=True",
"--steps.lastframe.save_results=True",
"--steps.firstframe.save_results=True",
"--steps.linearity.save_results=True",
"--steps.dark_current.save_results=True",
"--steps.jump.rejection_threshold=10.0",
]
Step.from_cmdline(args)
# Now run image2 pipeline on the _rate file, saving intermediate products
rtdata.input = 'det_image_1_MIRIMAGE_F770Wexp1_5stars_rate.fits'
args = [
"config/calwebb_image2.cfg", rtdata.input,
"--steps.assign_wcs.save_results=True",
"--steps.flat_field.save_results=True"
]
Step.from_cmdline(args)
# Grab rest of _rate files for the asn and run image2 pipeline on each to
# produce fresh _cal files for the image3 pipeline. We won't check these
# or look at intermediate products, and skip resample (don't need i2d image)
rate_files = [
"miri/image/det_image_1_MIRIMAGE_F770Wexp2_5stars_rate.fits",
"miri/image/det_image_2_MIRIMAGE_F770Wexp1_5stars_rate.fits",
"miri/image/det_image_2_MIRIMAGE_F770Wexp2_5stars_rate.fits",
]
for rate_file in rate_files:
rtdata.get_data(rate_file)
args = [
"config/calwebb_image2.cfg", rtdata.input,
"--steps.resample.skip=True"
]
Step.from_cmdline(args)
# Get the level3 assocation json file (though not its members) and run
# image3 pipeline on all _cal files listed in association
rtdata.get_data("miri/image/det_dithered_5stars_image3_asn.json")
args = [
"config/calwebb_image3.cfg",
rtdata.input,
# Set some unique param values needed for these data
"--steps.tweakreg.snr_threshold=200",
"--steps.tweakreg.use2dhist=False",
"--steps.source_catalog.snr_threshold=20",
]
Step.from_cmdline(args)
return rtdata
def test_step_with_local_class():
step_fn = join(dirname(__file__), 'steps', 'local_class.cfg')
step = Step.from_config_file(step_fn)
step.run(datamodels.ImageModel((2, 2)))
