def test_fgs_image2pipeline1(_bigdata):
"""
Regression test of calwebb_image2 pipeline performed on FGS imaging mode data.
"""
Image2Pipeline.call(
_bigdata +
'/fgs/test_image2pipeline/jw86500007001_02101_00001_GUIDER2_rate.fits',
output_file='jw86500007001_02101_00001_GUIDER2_cal.fits')
h = pf.open('jw86500007001_02101_00001_GUIDER2_cal.fits')
href = pf.open(
_bigdata +
'/fgs/test_image2pipeline/jw86500007001_02101_00001_GUIDER2_cal_ref.fits'
)
newh = pf.HDUList([h['primary'], h['sci'], h['err'], h['dq'], h['area']])
newhref = pf.HDUList(
[href['primary'], href['sci'], href['err'], href['dq'], href['area']])
result = pf.diff.FITSDiff(
newh,
newhref,
ignore_keywords=['DATE', 'CAL_VER', 'CAL_VCS', 'CRDS_VER', 'CRDS_CTX'],
rtol=0.00001)
assert result.identical, result.report()
def run_calwebb_image2(filename, tso=False):
if not tso:
result2 = Image2Pipeline(
config_file='pipeline_config_files/calwebb_image2.cfg')
else:
result2 = Image2Pipeline(
config_file='pipeline_config_files/calwebb_tso-image2.cfg')
result2.save_results = True
result2.output_dir = '/ifs/jwst/wit/nircam/simulationWG/Imaging/CAR-19/Pipeline_Level2/'
result2.run(filename)
def test_fgs_image2pipeline1(self):
"""
Regression test of calwebb_image2 pipeline performed on FGS imaging mode data.
"""
input_file = self.get_data('test_image2pipeline',
'jw86500007001_02101_00001_GUIDER2_rate.fits')
output_file = 'jw86500007001_02101_00001_GUIDER2_cal.fits'
Image2Pipeline.call(input_file, save_results=True)
outputs = [(output_file, 'jw86500007001_02101_00001_GUIDER2_cal_ref.fits')]
self.compare_outputs(outputs)
def test_image2pipeline2b(self):
"""
Regression test of calwebb_image2 pipeline performed on NIRCam data,
using a multiple integration rate (rateints) file as input.
"""
input_file = self.get_data(
'test_image2pipeline',
'jw82500001003_02101_00001_NRCALONG_rateints.fits')
output_file = 'jw82500001003_02101_00001_NRCALONG_calints.fits'
Image2Pipeline.call(input_file, output_file=output_file)
outputs = [(output_file,
'jw82500001003_02101_00001_NRCALONG_calints_ref.fits',
['primary', 'sci', 'err', 'dq', 'area'])]
self.compare_outputs(outputs)
def run_pipeline(files, flat_field=True):
"""
Runs all stages of the JWST Pipeline on uncalibrated imaging data.
Parameters
----------
files : list
The files to run the pipeline on.
flat_field : bool
Option to run the flat field step in image2.
Returns
-------
outname : str
The name of the final i2d drizzled image.
"""
# Create a name for image3 association and drizzled files
detector = fits.getheader(files[0])['DETECTOR'].lower()
fltr = fits.getheader(files[0])['FILTER'].lower()
pupil = fits.getheader(files[0])['PUPIL'].lower()
target = fits.getheader(files[0])['TARGNAME'].lower()
name = '{}_{}_{}_{}'.format(detector, fltr, pupil, target)
# Run detector1
for f in files:
m = Detector1Pipeline()
#m.jump.override_readnoise = 'jwst_nircam_readnoise_0024_psub.fits'
m.refpix.odd_even_rows = False # only for MIRI
m.ipc.skip = True
m.persistence.skip = True
m.save_results = True
m.output_dir = os.getcwd()
m.run(f)
# Run image2
files = [f.replace('_uncal.fits', '_rate.fits') for f in files]
for f in files:
m = Image2Pipeline()
if not flat_field:
m.flat_field.skip = True
m.save_results = True
m.output_dir = os.getcwd()
m.run(f)
# Run image3
files = [f.replace('_rate.fits', '_cal.fits') for f in files]
asn = asn_from_list(files, rule=Asn_Lv3Image, product_name=name)
asn_file = '{}.json'.format(name)
with open(asn_file, 'w') as f:
f.write(asn.dump()[1])
m = Image3Pipeline()
m.save_results = True
m.output_dir = os.getcwd()
m.run(asn_file)
return '{}_i2d.fits'.format(name)
def test_image2pipeline2b(_bigdata):
"""
Regression test of calwebb_image2 pipeline performed on NIRCam data,
using a multiple integration rate (rateints) file as input.
"""
Image2Pipeline.call(_bigdata+'/pipelines/jw82500001003_02101_00001_NRCALONG_rateints.fits',
output_file='jw82500001003_02101_00001_NRCALONG_calints.fits')
h = fits.open('jw82500001003_02101_00001_NRCALONG_calints.fits')
href = fits.open(_bigdata+'/pipelines/jw82500001003_02101_00001_NRCALONG_calints_ref.fits')
newh = fits.HDUList([h['primary'],h['sci'],h['err'],h['dq'],h['area']])
newhref = fits.HDUList([href['primary'],href['sci'],href['err'],href['dq'],href['area']])
result = fits.diff.FITSDiff(newh,
newhref,
ignore_keywords = ['DATE','CAL_VER','CAL_VCS','CRDS_VER','CRDS_CTX'],
rtol = 0.00001
)
assert result.identical, result.report()
def test_image2pipeline1(_bigdata):
"""
Regression test of calwebb_image2 pipeline performed on MIRI data.
"""
Image2Pipeline.call(
_bigdata +
'/miri/test_image2pipeline/jw00001001001_01101_00001_mirimage_rate.fits'
)
h = pf.open('jw00001001001_01101_00001_mirimage_cal.fits')
href = pf.open(
_bigdata +
'/miri/test_image2pipeline/jw00001001001_01101_00001_mirimage_cal_ref.fits'
)
newh = pf.HDUList([h['primary'], h['sci'], h['err'], h['dq'], h['area']])
newhref = pf.HDUList(
[href['primary'], href['sci'], href['err'], href['dq'], href['area']])
result = pf.diff.FITSDiff(
newh,
newhref,
ignore_keywords=['DATE', 'CAL_VER', 'CAL_VCS', 'CRDS_VER', 'CRDS_CTX'],
rtol=0.00001)
assert result.identical, result.report()
h = pf.open('jw00001001001_01101_00001_mirimage_i2d.fits')
href = pf.open(
_bigdata +
'/miri/test_image2pipeline/jw00001001001_01101_00001_mirimage_i2d_ref.fits'
)
newh = pf.HDUList([h['primary'], h['sci'], h['wht'], h['con']])
newhref = pf.HDUList(
[href['primary'], href['sci'], href['wht'], href['con']])
result = pf.diff.FITSDiff(
newh,
newhref,
ignore_keywords=['DATE', 'CAL_VER', 'CAL_VCS', 'CRDS_VER', 'CRDS_CTX'],
rtol=0.00001)
assert result.identical, result.report()
def test_image2pipeline2_cal(_bigdata):
"""
Regression test of calwebb_image2 pipeline performed on NIRCam data.
"""
Image2Pipeline.call(
_bigdata +
'/pipelines/jw82500001003_02101_00001_NRCALONG_rate_ref.fits',
output_file='jw82500001003_02101_00001_NRCALONG_cal.fits')
na = 'jw82500001003_02101_00001_NRCALONG_cal.fits'
h = fits.open(na)
nb = _bigdata + '/pipelines/jw82500001003_02101_00001_NRCALONG_cal_ref.fits'
href = fits.open(nb)
newh = fits.HDUList([h['primary'], h['sci'], h['err'], h['dq'], h['area']])
newhref = fits.HDUList(
[href['primary'], href['sci'], href['err'], href['dq'], href['area']])
result = fits.diff.FITSDiff(
newh,
newhref,
ignore_keywords=['DATE', 'CAL_VER', 'CAL_VCS', 'CRDS_VER', 'CRDS_CTX'],
rtol=0.00001)
assert result.identical, result.report()
na = 'jw82500001003_02101_00001_NRCALONG_i2d.fits'
h = fits.open(na)
nb = _bigdata + '/pipelines/jw82500001003_02101_00001_NRCALONG_i2d_ref.fits'
href = fits.open(nb)
newh = fits.HDUList([h['primary'], h['sci'], h['con'], h['wht']])
newhref = fits.HDUList(
[href['primary'], href['sci'], href['con'], href['wht']])
result = fits.diff.FITSDiff(
newh,
newhref,
ignore_keywords=['DATE', 'CAL_VER', 'CAL_VCS', 'CRDS_VER', 'CRDS_CTX'],
rtol=0.00001)
assert result.identical, result.report()
def output_hdul(set_inandout_filenames, config):
set_inandout_filenames_info = core_utils.read_info4outputhdul(
config, set_inandout_filenames)
step, txt_name, step_input_file, step_output_file, run_calwebb_spec2, outstep_file_suffix = set_inandout_filenames_info
# start the timer to compute the step running time of PTT
PTT_start_time = time.time()
# check if the filter is to be changed
change_filter_opaque = config.getboolean("calwebb_spec2_input_file",
"change_filter_opaque")
if change_filter_opaque:
_, step_input_file = change_filter_opaque2science.change_filter_opaque(
step_input_file)
change_filter_opaque_msg = " * With FILTER=OPAQUE, the calwebb_spec2 will run up to the extract_2d step. Further steps will be skipped. \n"
print(change_filter_opaque_msg)
# determine if the pipeline is to be run in full
run_calwebb_spec2 = config.getboolean("run_calwebb_spec2_in_full",
"run_calwebb_spec2")
# determine which steps are to be run, if not run in full
run_pipe_step = config.getboolean("run_pipe_steps", step)
# determine which tests are to be run
assign_wcs_completion_tests = config.getboolean(
"run_pytest", "_".join((step, "completion", "tests")))
assign_wcs_reffile_tests = config.getboolean(
"run_pytest", "_".join((step, "reffile", "tests")))
assign_wcs_validation_tests = config.getboolean(
"run_pytest", "_".join((step, "validation", "tests")))
run_pytests = [
assign_wcs_completion_tests, assign_wcs_reffile_tests,
assign_wcs_validation_tests
]
# get other relevant info from PTT config file
esa_files_path = config.get("esa_intermediary_products", "esa_files_path")
wcs_threshold_diff = config.get("additional_arguments",
"wcs_threshold_diff")
save_wcs_plots = config.getboolean("additional_arguments",
"save_wcs_plots")
output_directory = config.get("calwebb_spec2_input_file",
"output_directory")
# Get the detector used
detector = fits.getval(step_input_file, "DETECTOR", 0)
# get main header from input file
inhdu = core_utils.read_hdrfits(step_input_file,
info=False,
show_hdr=False)
# if run_calwebb_spec2 is True calwebb_spec2 will be called, else individual steps will be ran
step_completed = False
end_time = '0.0'
# Check if data is IFU that the Image Model keyword is correct
mode_used = config.get("calwebb_spec2_input_file", "mode_used").lower()
if mode_used == "ifu":
DATAMODL = fits.getval(step_input_file, "DATAMODL", 0)
if DATAMODL != "IFUImageModel":
fits.setval(step_input_file, "DATAMODL", 0, value="IFUImageModel")
print("DATAMODL keyword changed to IFUImageModel.")
# get the shutter configuration file for MOS data only
msa_shutter_conf = "No shutter configuration file will be used."
if core_utils.check_MOS_true(inhdu):
msa_shutter_conf = config.get("esa_intermediary_products",
"msa_conf_name")
# check if the configuration shutter file name is in the header of the fits file and if not add it
msametfl = fits.getval(step_input_file, "MSAMETFL", 0)
if os.path.basename(msa_shutter_conf) != msametfl:
msametfl = os.path.basename(msa_shutter_conf)
fits.setval(step_input_file, "MSAMETFL", 0, value=msametfl)
# check if processing an image, then set proper variables
imaging_mode = False
if mode_used in ('image', 'confirm', 'taconfirm', 'wata', 'msata', 'bota',
'focus', 'mimf'):
run_calwebb_spec2 = True
imaging_mode = True
print(
'\n * Image processing will only be run in full with PTT. All intermediary products will be saved.'
)
print(
' -> For now, all pytests will be skipped since there are now image-specific routines yet. \n'
)
# TODO: add imaging tests
# get the name of the configuration file and run the pipeline
calwebb_spec2_cfg = config.get("run_calwebb_spec2_in_full",
"calwebb_spec2_cfg")
# copy the configuration file to create the pipeline log
stpipelogcfg = calwebb_spec2_cfg.replace("calwebb_spec2.cfg",
"stpipe-log.cfg")
subprocess.run(["cp", stpipelogcfg, os.getcwd()])
# run the pipeline
if run_calwebb_spec2:
# Create the logfile for PTT, but remove the previous log file
PTTcalspec2_log = os.path.join(output_directory,
'PTT_calspec2_' + detector + '.log')
if imaging_mode:
PTTcalspec2_log = PTTcalspec2_log.replace('calspec2', 'calimage2')
if os.path.isfile(PTTcalspec2_log):
os.remove(PTTcalspec2_log)
print(
"Information outputed to screen from PTT will be logged in file: ",
PTTcalspec2_log)
for handler in logging.root.handlers[:]:
logging.root.removeHandler(handler)
logging.basicConfig(filename=PTTcalspec2_log, level=logging.INFO)
logging.info(pipeline_version)
if change_filter_opaque:
logging.info(change_filter_opaque_msg)
run_calwebb_spec2_msg = " *** Will run pipeline in full ... "
print(run_calwebb_spec2_msg)
logging.info(run_calwebb_spec2_msg)
# create the map
txt_name = "full_run_map_" + detector + ".txt"
if os.path.isfile(txt_name):
os.remove(txt_name)
assign_wcs_utils.create_completed_steps_txtfile(
txt_name, step_input_file)
# start the timer to compute the step running time of PTT
core_utils.start_end_PTT_time(txt_name,
start_time=PTT_start_time,
end_time=None)
if mode_used == "bots":
calwebb_spec2_cfg = calwebb_spec2_cfg.replace(
"calwebb_spec2.cfg", "calwebb_tso-spec2.cfg")
print(
'\nUsing the following configuration file to run TSO pipeline:'
)
print(calwebb_spec2_cfg, '\n')
if imaging_mode:
calwebb_image2_cfg = calwebb_spec2_cfg.replace(
"calwebb_spec2.cfg", "calwebb_image2.cfg")
print(
'\nUsing the following configuration file to run imaging pipeline:'
)
print(calwebb_image2_cfg, '\n')
else:
print(
'\nUsing the following configuration file to run spectroscopic pipeline:'
)
print(calwebb_spec2_cfg, '\n')
input_file = config.get("calwebb_spec2_input_file", "input_file")
if "_uncal_rate" in input_file:
input_file = input_file.replace("_uncal_rate", "")
if "_uncal" in input_file:
input_file = input_file.replace("_uncal", "")
final_output_name = input_file.replace(".fits", "_cal.fits")
if core_utils.check_MOS_true(inhdu):
# copy the MSA shutter configuration file into the pytest directory
subprocess.run(["cp", msa_shutter_conf, "."])
# start the timer to compute the step running time
start_time = time.time()
# run the pipeline
print('Running pipeline... \n')
if not imaging_mode:
Spec2Pipeline.call(
step_input_file,
config_file=calwebb_spec2_cfg) #, logcfg=stpipelogcfg)
else:
Image2Pipeline.call(step_input_file,
config_file=calwebb_image2_cfg)
subprocess.run(["rm", "stpipe-log.cfg"])
# For the moment, the pipeline is using the wrong reference file for slit 400A1, so the
# file needs to be re-processed with the right reference file
if core_utils.check_FS_true(inhdu):
print(
"\n * WARNING: For the moment, the wrong reference file is being used for "
"processing slit 400A1. The file will be re-processed ")
# print(" $ jwst.pathloss.PathLossStep final_output_caldet1_NRS1_srctype.fits "
# "--override_pathloss=jwst-nirspec-a400.plrf.fits \n")
pathloss_400a1 = step_input_file.replace("srctype.fits",
"pathloss_400A1.fits")
reffile_400a1 = "jwst-nirspec-a400.plrf.fits"
print("Re-processing slit with new reference file: ",
reffile_400a1)
pl = PathLossStep()
pl.override_pathloss = reffile_400a1
pl.run(step_input_file)
subprocess.run([
"mv",
step_input_file.replace("srctype", "pathlossstep"),
pathloss_400a1
])
print("Saved pipeline re-processed file as: ", pathloss_400a1)
# end the timer to compute calwebb_spec2 running time
end_time = repr(time.time() - start_time) # this is in seconds
calspec2_time = " * Pipeline took " + end_time + " seconds to finish.\n"
print(calspec2_time)
logging.info(calspec2_time)
# remove the copy of the MSA shutter configuration file
if core_utils.check_MOS_true(inhdu):
subprocess.run(["rm", msametfl])
# add the detector string to the name of the files and move them to the working directory
core_utils.add_detector2filename(output_directory, step_input_file)
final_output_name_msg = "\nThe final pipeline product was saved in: " + final_output_name
print(final_output_name_msg)
logging.info(final_output_name_msg)
# read the assign wcs fits file
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
# scihdul = core_utils.read_hdrfits(step_output_file, info=False, show_hdr=False, ext=1)
# rename and move the pipeline log file
try:
calspec2_pipelog = "calspec2_pipeline_" + detector + ".log"
if imaging_mode:
calspec2_pipelog = calspec2_pipelog.replace(
'calspec2', 'calimage2')
path_where_pipeline_was_run = os.getcwd()
logfile = glob(path_where_pipeline_was_run + "/pipeline.log")[0]
print(logfile)
os.rename(logfile, os.path.join(output_directory,
calspec2_pipelog))
except IndexError:
print(
"\nWARNING: Something went wrong. Could not find a pipeline.log file \n"
)
# make sure we are able to find calspec2_pipelog either in the calwebb_spec2 directory or in the working dir
if not os.path.isfile(calspec2_pipelog):
calspec2_pipelog = os.path.join(output_directory, calspec2_pipelog)
# add the running time for all steps
step_running_times = core_utils.calculate_step_run_time(
calspec2_pipelog)
end_time_list = []
for stp in core_utils.step_string_dict:
if stp in step_running_times:
step_completed = True
step_time = step_running_times[stp]["run_time"]
out_suffix = core_utils.step_string_dict[stp]["suffix"]
core_utils.add_completed_steps(txt_name, stp, out_suffix,
step_completed, step_time)
end_time_list.append(step_time)
# print total running time in the text file and move it to the indicated directory
string2print = "pipeline_total_time"
if float(end_time) <= sum(end_time_list):
tot_time = repr(sum(end_time_list))
else:
tot_time = end_time
assign_wcs_utils.print_time2file(txt_name, tot_time, string2print)
PTT_runtimes_msg = "Pipeline and PTT run times written in file: " + os.path.basename(
txt_name) + " in working directory. \n"
print(PTT_runtimes_msg)
logging.info(PTT_runtimes_msg)
# move the final reporting text files to the working directory
core_utils.move_txt_files_2workdir(config, detector)
# end script for imaging case
if imaging_mode:
print('\nPTT finished processing imaging mode. \n')
pytest.exit(
"Skipping pytests for now because they need to be written for imaging mode."
)
return hdul, step_output_file, msa_shutter_conf, esa_files_path, wcs_threshold_diff, save_wcs_plots, run_pytests, mode_used
else:
# create the map but remove a previous one if it exists
if os.path.isfile(txt_name):
os.remove(txt_name)
assign_wcs_utils.create_completed_steps_txtfile(
txt_name, step_input_file)
# start the timer to compute the step running time of PTT
core_utils.start_end_PTT_time(txt_name,
start_time=PTT_start_time,
end_time=None)
msg = "\n Pipeline and PTT run times will be written in file: " + os.path.basename(
txt_name) + " in working directory. \n"
print(msg)
logging.info(msg)
if run_pipe_step:
# Create the logfile for PTT, but erase the previous one if it exists
PTTcalspec2_log = os.path.join(
output_directory,
'PTT_calspec2_' + detector + '_' + step + '.log')
if imaging_mode:
PTTcalspec2_log = PTTcalspec2_log.replace(
'calspec2', 'calimage2')
if os.path.isfile(PTTcalspec2_log):
os.remove(PTTcalspec2_log)
print(
"Information outputed to screen from PTT will be logged in file: ",
PTTcalspec2_log)
for handler in logging.root.handlers[:]:
logging.root.removeHandler(handler)
logging.basicConfig(filename=PTTcalspec2_log, level=logging.INFO)
logging.info(pipeline_version)
if change_filter_opaque:
logging.info(change_filter_opaque_msg)
# check that previous pipeline steps were run up to this point
core_utils.check_completed_steps(step, step_input_file)
if os.path.isfile(step_input_file):
msg = " *** Step " + step + " set to True"
print(msg)
logging.info(msg)
stp = AssignWcsStep()
if core_utils.check_MOS_true(inhdu):
# copy the MSA shutter configuration file into the pytest directory
subprocess.run(["cp", msa_shutter_conf, "."])
# get the right configuration files to run the step
local_pipe_cfg_path = config.get("calwebb_spec2_input_file",
"local_pipe_cfg_path")
# start the timer to compute the step running time
print("running pipeline...")
start_time = time.time()
if local_pipe_cfg_path == "pipe_source_tree_code":
result = stp.call(step_input_file)
else:
result = stp.call(step_input_file,
config_file=local_pipe_cfg_path +
'/assign_wcs.cfg')
result.save(step_output_file)
# end the timer to compute the step running time
end_time = repr(time.time() - start_time) # this is in seconds
msg = "Step " + step + " took " + end_time + " seconds to finish"
print(msg)
logging.info(msg)
if core_utils.check_MOS_true(inhdu):
# remove the copy of the MSA shutter configuration file
subprocess.run(["rm", msametfl])
# rename and move the pipeline log file
try:
calspec2_pipelog = "calspec2_pipeline_" + step + "_" + detector + ".log"
if imaging_mode:
calspec2_pipelog = calspec2_pipelog.replace(
'calspec2', 'calimage2')
pytest_workdir = TESTSDIR
logfile = glob(pytest_workdir + "/pipeline.log")[0]
os.rename(logfile,
os.path.join(output_directory, calspec2_pipelog))
except IndexError:
print(
"\n* WARNING: Something went wrong. Could not find a pipeline.log file \n"
)
else:
msg = "Skipping step. Input file " + step_input_file + " does not exit."
print(msg)
logging.info(msg)
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed, end_time)
pytest.skip("Skipping " + step +
" because the input file does not exist.")
else:
print("Skipping running pipeline step ", step)
# add the running time for this step
end_time = core_utils.get_stp_run_time_from_screenfile(
step, detector, output_directory)
if os.path.isfile(step_output_file):
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
step_completed = True
# add the running time for this step
core_utils.add_completed_steps(txt_name, step, outstep_file_suffix,
step_completed, end_time)
return hdul, step_output_file, msa_shutter_conf, esa_files_path, wcs_threshold_diff, save_wcs_plots, run_pytests, mode_used
else:
step_completed = False
# add the running time for this step
core_utils.add_completed_steps(txt_name, step, outstep_file_suffix,
step_completed, end_time)
pytest.skip("Test skipped because input file " + step_output_file +
" does not exist.")