def run_pipeline(filename, outdir='./'):
"""Run calwebb_detector1 on the input file
Parameters
----------
filename : str
Name of input raw file
Returns
-------
slope_filename : str
Name of fits file with slope image
"""
lin_file = filename.replace('_uncal', '_linearity')
rate_file = filename.replace('_uncal', '_rate')
if not os.path.isfile(lin_file) or not os.path.isfile(rate_file):
cal = Detector1Pipeline()
cal.dark_current.skip = True
cal.persistence.skip = True
cal.linearity.save_results = True
cal.jump.rejection_threshold = 9
cal.jump.maximum_cores = 'quarter'
cal.save_results = True
cal.output_dir = outdir
cal.run(filename)
def test_fgs_detector1_1(self):
"""
Regression test of calwebb_detector1 pipeline performed on FGS imaging mode data.
"""
input_file = self.get_data(
'test_sloperpipeline',
'jw86500007001_02101_00001_GUIDER2_uncal.fits')
pipe = Detector1Pipeline()
pipe.ipc.skip = True
pipe.refpix.odd_even_columns = True
pipe.refpix.use_side_ref_pixels = True
pipe.refpix.side_smoothing_length = 11
pipe.refpix.side_gain = 1.0
pipe.refpix.odd_even_rows = True
pipe.jump.rejection_threshold = 250.0
pipe.persistence.skip = True
pipe.ramp_fit.save_opt = False
pipe.save_calibrated_ramp = True
pipe.output_file = 'jw86500007001_02101_00001_GUIDER2_rate.fits'
pipe.run(input_file)
outputs = [('jw86500007001_02101_00001_GUIDER2_ramp.fits',
'jw86500007001_02101_00001_GUIDER2_ramp_ref.fits',
['primary', 'sci', 'err', 'groupdq', 'pixeldq']),
('jw86500007001_02101_00001_GUIDER2_rateints.fits',
'jw86500007001_02101_00001_GUIDER2_rateints_ref.fits',
['primary', 'sci', 'err', 'dq']),
('jw86500007001_02101_00001_GUIDER2_rate.fits',
'jw86500007001_02101_00001_GUIDER2_rate_ref.fits',
['primary', 'sci', 'err', 'dq'])]
self.compare_outputs(outputs)
def test_fgs_detector1_1(_bigdata):
"""
Regression test of calwebb_detector1 pipeline performed on FGS imaging mode data.
"""
pipe = Detector1Pipeline()
pipe.ipc.skip = True
pipe.refpix.odd_even_columns = True
pipe.refpix.use_side_ref_pixels = True
pipe.refpix.side_smoothing_length = 11
pipe.refpix.side_gain = 1.0
pipe.refpix.odd_even_rows = True
pipe.jump.rejection_threshold = 250.0
pipe.persistence.skip = True
pipe.ramp_fit.save_opt = False
pipe.save_calibrated_ramp = True
pipe.output_file = 'jw86500007001_02101_00001_GUIDER2_rate.fits'
pipe.run(_bigdata+'/fgs/test_sloperpipeline/jw86500007001_02101_00001_GUIDER2_uncal.fits')
# Compare calibrated ramp product
n_cr = 'jw86500007001_02101_00001_GUIDER2_ramp.fits'
h = pf.open(n_cr)
n_ref = _bigdata+'/fgs/test_sloperpipeline/jw86500007001_02101_00001_GUIDER2_ramp_ref.fits'
href = pf.open(n_ref)
newh = pf.HDUList([h['primary'],h['sci'],h['err'],h['groupdq'],h['pixeldq']])
newhref = pf.HDUList([href['primary'],href['sci'],href['err'],href['groupdq'],href['pixeldq']])
result = pf.diff.FITSDiff(newh,
newhref,
ignore_keywords = ['DATE','CAL_VER','CAL_VCS','CRDS_VER','CRDS_CTX'],
rtol = 0.00001
)
assert result.identical, result.report()
# Compare multi-integration countrate image product
n_int = 'jw86500007001_02101_00001_GUIDER2_rateints.fits'
h = pf.open(n_int)
n_ref = _bigdata+'/fgs/test_sloperpipeline/jw86500007001_02101_00001_GUIDER2_rateints_ref.fits'
href = pf.open(n_ref)
newh = pf.HDUList([h['primary'],h['sci'],h['err'],h['dq']])
newhref = pf.HDUList([href['primary'],href['sci'],href['err'],href['dq']])
result = pf.diff.FITSDiff(newh,
newhref,
ignore_keywords = ['DATE','CAL_VER','CAL_VCS','CRDS_VER','CRDS_CTX'],
rtol = 0.00001
)
assert result.identical, result.report()
# Compare countrate image product
n_rate = 'jw86500007001_02101_00001_GUIDER2_rate.fits'
h = pf.open(n_rate)
n_ref = _bigdata+'/fgs/test_sloperpipeline/jw86500007001_02101_00001_GUIDER2_rate_ref.fits'
href = pf.open(n_ref)
newh = pf.HDUList([h['primary'],h['sci'],h['err'],h['dq']])
newhref = pf.HDUList([href['primary'],href['sci'],href['err'],href['dq']])
result = pf.diff.FITSDiff(newh,
newhref,
ignore_keywords = ['DATE','CAL_VER','CAL_VCS','CRDS_VER','CRDS_CTX'],
rtol = 0.00001
)
assert result.identical, result.report()
def run_calwebb_detector1(filename):
m = Detector1Pipeline(
config_file='pipeline_config_files/calwebb_detector1.cfg')
# make changes to the parameters/reference files used
m.refpix.odd_even_rows = False
# jump step is way too sensitive
m.jump.rejection_threshold = 91
# skip steps you don't want to run
m.group_scale.skip = True
m.ipc.skip = True
m.rscd.skip = True
m.firstframe.skip = True
m.lastframe.skip = True
# name your output file
m.save_results = True
m.output_dir = '/ifs/jwst/wit/nircam/simulationWG/Imaging/CAR-19/Pipeline_Level1/'
m.output_file = os.path.basename(filename.replace('_uncal', '_rate'))
# run the pipeline with these paramters
m.run(filename)
print('')
print("Done running CALDETECTOR1 on {}".format(filename))
print("Output saved to {}".format(os.path.join(m.output_dir,
m.output_file)))
print('')
def run_pipeline(files, flat_field=True):
"""
Runs all stages of the JWST Pipeline on uncalibrated imaging data.
Parameters
----------
files : list
The files to run the pipeline on.
flat_field : bool
Option to run the flat field step in image2.
Returns
-------
outname : str
The name of the final i2d drizzled image.
"""
# Create a name for image3 association and drizzled files
detector = fits.getheader(files[0])['DETECTOR'].lower()
fltr = fits.getheader(files[0])['FILTER'].lower()
pupil = fits.getheader(files[0])['PUPIL'].lower()
target = fits.getheader(files[0])['TARGNAME'].lower()
name = '{}_{}_{}_{}'.format(detector, fltr, pupil, target)
# Run detector1
for f in files:
m = Detector1Pipeline()
#m.jump.override_readnoise = 'jwst_nircam_readnoise_0024_psub.fits'
m.refpix.odd_even_rows = False # only for MIRI
m.ipc.skip = True
m.persistence.skip = True
m.save_results = True
m.output_dir = os.getcwd()
m.run(f)
# Run image2
files = [f.replace('_uncal.fits', '_rate.fits') for f in files]
for f in files:
m = Image2Pipeline()
if not flat_field:
m.flat_field.skip = True
m.save_results = True
m.output_dir = os.getcwd()
m.run(f)
# Run image3
files = [f.replace('_rate.fits', '_cal.fits') for f in files]
asn = asn_from_list(files, rule=Asn_Lv3Image, product_name=name)
asn_file = '{}.json'.format(name)
with open(asn_file, 'w') as f:
f.write(asn.dump()[1])
m = Image3Pipeline()
m.save_results = True
m.output_dir = os.getcwd()
m.run(asn_file)
return '{}_i2d.fits'.format(name)
def test_niriss_detector1(_bigdata):
"""
Regression test of calwebb_detector1 pipeline performed on NIRISS data.
"""
step = Detector1Pipeline()
step.save_calibrated_ramp = True
step.ipc.skip = True
step.persistence.skip = True
step.refpix.odd_even_columns = True
step.refpix.use_side_ref_pixels = True
step.refpix.side_smoothing_length = 11
step.refpix.side_gain = 1.0
step.refpix.odd_even_rows = True
step.jump.rejection_threshold = 250.0
step.ramp_fit.save_opt = False
step.ramp_fit.suffix = 'ramp'
step.output_file = 'jw00034001001_01101_00001_NIRISS_rate.fits'
step.run(_bigdata +
'/pipelines/jw00034001001_01101_00001_NIRISS_uncal.fits')
# Compare ramp product
n_ramp = 'jw00034001001_01101_00001_NIRISS_ramp.fits'
h = pf.open(n_ramp)
n_ref = _bigdata + '/pipelines/jw00034001001_01101_00001_NIRISS_ramp_ref.fits'
href = pf.open(n_ref)
newh = pf.HDUList(
[h['primary'], h['sci'], h['err'], h['groupdq'], h['pixeldq']])
newhref = pf.HDUList([
href['primary'], href['sci'], href['err'], href['groupdq'],
href['pixeldq']
])
result = pf.diff.FITSDiff(
newh,
newhref,
ignore_keywords=['DATE', 'CAL_VER', 'CAL_VCS', 'CRDS_VER', 'CRDS_CTX'],
rtol=0.00001)
assert result.identical, result.report()
# Compare countrate image product
n_cr = 'jw00034001001_01101_00001_NIRISS_rate.fits'
h = pf.open(n_cr)
n_ref = _bigdata + '/pipelines/jw00034001001_01101_00001_NIRISS_rate_ref.fits'
href = pf.open(n_ref)
newh = pf.HDUList([h['primary'], h['sci'], h['err'], h['dq']])
newhref = pf.HDUList(
[href['primary'], href['sci'], href['err'], href['dq']])
result = pf.diff.FITSDiff(
newh,
newhref,
ignore_keywords=['DATE', 'CAL_VER', 'CAL_VCS', 'CRDS_VER', 'CRDS_CTX'],
rtol=0.00001)
assert result.identical, result.report()
def test_detector1pipeline4(_bigdata):
"""
Regression test of calwebb_detector1 pipeline performed on NIRSpec data.
"""
step = Detector1Pipeline()
step.save_calibrated_ramp = True
step.ipc.skip = True
step.persistence.skip = True
step.jump.rejection_threshold = 4.0
step.ramp_fit.save_opt = False
step.output_file = 'jw84600007001_02101_00001_nrs1_rate.fits'
step.run(_bigdata + '/pipelines/jw84600007001_02101_00001_nrs1_uncal.fits')
# Compare ramp product
n_ramp = 'jw84600007001_02101_00001_nrs1_ramp.fits'
h = fits.open(n_ramp)
n_ref = _bigdata + '/pipelines/jw84600007001_02101_00001_nrs1_ramp_ref.fits'
href = fits.open(n_ref)
newh = fits.HDUList(
[h['primary'], h['sci'], h['err'], h['groupdq'], h['pixeldq']])
newhref = fits.HDUList([
href['primary'], href['sci'], href['err'], href['groupdq'],
h['pixeldq']
])
result = fits.diff.FITSDiff(
newh,
newhref,
ignore_keywords=['DATE', 'CAL_VER', 'CAL_VCS', 'CRDS_VER', 'CRDS_CTX'],
rtol=0.00001)
assert result.identical, result.report()
# Compare countrate image product
n_cr = 'jw84600007001_02101_00001_nrs1_rate.fits'
h = fits.open(n_cr)
n_ref = _bigdata + '/pipelines/jw84600007001_02101_00001_nrs1_rate_ref.fits'
href = fits.open(n_ref)
newh = fits.HDUList([h['primary'], h['sci'], h['err'], h['dq']])
newhref = fits.HDUList(
[href['primary'], href['sci'], href['err'], href['dq']])
result = fits.diff.FITSDiff(
newh,
newhref,
ignore_keywords=['DATE', 'CAL_VER', 'CAL_VCS', 'CRDS_VER', 'CRDS_CTX'],
rtol=0.00001)
assert result.identical, result.report()
def test_detector1pipeline1(_bigdata):
"""
Regression test for gain_scale naming when results are requested to
be saved for the gain_scale step.
"""
step = Detector1Pipeline()
step.group_scale.skip = True
step.dq_init.skip = True
step.saturation.skip = True
step.ipc.skip = True
step.superbias.skip = True
step.refpix.skip = True
step.rscd.skip = True
step.firstframe.skip = True
step.lastframe.skip = True
step.linearity.skip = True
step.dark_current.skip = True
step.persistence.skip = True
step.jump.skip = True
step.ramp_fit.skip = False
step.gain_scale.skip = False
step.gain_scale.save_results = True
expfile = 'jw00001001001_01101_00001_MIRIMAGE'
step.run(_bigdata + '/miri/test_sloperpipeline/' + expfile + '_uncal.fits')
files = glob('*.fits')
output_file = expfile + '_gain_scale.fits'
assert output_file in files
files.remove(output_file)
output_file = expfile + '_gain_scaleints.fits'
assert output_file in files
files.remove(output_file)
assert not len(files)
tf.write(line1 + "\n")
final_out = "gain_scale.fits"
output_names = [
"group_scale.fits", "dq_init.fits", "saturation.fits", "superbias.fits",
"refpix.fits", "rscd.fits", "lastframe.fits", "linearity.fits",
"dark_current.fits", "jump.fits", "ramp_fit.fits", final_out
]
# Get and save the value of the raw data root name to add at the end of calwebb_detector1
#rawdatrt = fits.getval(fits_input_uncal_file, 'rawdatrt', 0)
if not step_by_step:
# start the timer to compute the step running time
start_time = time.time()
Detector1Pipeline.call(fits_input_uncal_file,
config_file=calwebb_detector1_cfg)
# end the timer to compute calwebb_spec2 running time
end_time = time.time() - start_time # this is in seconds
print(" * calwebb_detector1 took " + repr(end_time) +
" seconds to finish *")
total_time = "{:<18} {:<20} {:<20}".format("", "total time = ",
repr(end_time))
tot_time_min = round((end_time / 60.0), 2)
for outnm in output_names:
line2write = "{:<18} {:<20} {:<20}".format(outnm.replace(".fits", ""),
outnm, "")
with open(txt_outputs_summary, "a") as tf:
tf.write(line2write + "\n")
else:
# steps to be ran, in order
steps_to_run = [
def calwebb_detector1_save_jump(input_file,
output_dir,
ramp_fit=True,
save_fitopt=True):
"""Call ``calwebb_detector1`` on the provided file, running all
steps up to the ``ramp_fit`` step, and save the result. Optionally
run the ``ramp_fit`` step and save the resulting slope file as well.
Parameters
----------
input_file : str
Name of fits file to run on the pipeline
output_dir : str
Directory into which the pipeline outputs are saved
ramp_fit : bool
If ``False``, the ``ramp_fit`` step is not run. The output file
will be a ``*_jump.fits`` file.
If ``True``, the ``*jump.fits`` file will be produced and saved.
In addition, the ``ramp_fit`` step will be run and a
``*rate.fits`` or ``*_rateints.fits`` file will be saved.
(``rateints`` if the input file has >1 integration)
save_fitopt : bool
If ``True``, the file of optional outputs from the ramp fitting
step of the pipeline is saved.
Returns
-------
jump_output : str
Name of the saved file containing the output prior to the
``ramp_fit`` step.
pipe_output : str
Name of the saved file containing the output after ramp-fitting
is performed (if requested). Otherwise ``None``.
"""
input_file_only = os.path.basename(input_file)
# Find the instrument used to collect the data
instrument = fits.getheader(input_file)['INSTRUME'].lower()
# Switch to calling the pipeline rather than individual steps,
# and use the run() method so that we can set parameters
# progammatically.
model = Detector1Pipeline()
# Always true
if instrument == 'nircam':
model.refpix.odd_even_rows = False
# Default CR rejection threshold is too low
model.jump.rejection_threshold = 15
model.jump.save_results = True
model.jump.output_dir = output_dir
jump_output = os.path.join(output_dir,
input_file_only.replace('uncal', 'jump'))
# Check to see if the jump version of the requested file is already
# present
run_jump = not os.path.isfile(jump_output)
if ramp_fit:
model.ramp_fit.save_results = True
#model.save_results = True
model.output_dir = output_dir
#pipe_output = os.path.join(output_dir, input_file_only.replace('uncal', 'rate'))
pipe_output = os.path.join(
output_dir, input_file_only.replace('uncal', '0_ramp_fit'))
run_slope = not os.path.isfile(pipe_output)
if save_fitopt:
model.ramp_fit.save_opt = True
fitopt_output = os.path.join(
output_dir, input_file_only.replace('uncal', 'fitopt'))
run_fitopt = not os.path.isfile(fitopt_output)
else:
model.ramp_fit.save_opt = False
fitopt_output = None
run_fitopt = False
else:
model.ramp_fit.skip = True
pipe_output = None
fitopt_output = None
run_slope = False
run_fitopt = False
# Call the pipeline if any of the files at the requested calibration
# states are not present in the output directory
if run_jump or (ramp_fit and run_slope) or (save_fitopt and run_fitopt):
model.run(input_file)
else:
print((
"Files with all requested calibration states for {} already present in "
"output directory. Skipping pipeline call.".format(input_file)))
return jump_output, pipe_output, fitopt_output
def run_caldet1(fits_input_uncal_file, step_by_step=False):
if not os.path.isfile(fits_input_uncal_file):
print("Input file not found in the current directory. Unable to proceed, exiting script.")
exit()
# Get the detector used
detector = fits.getval(fits_input_uncal_file, "DETECTOR", 0)
# Get the cfg file
calwebb_detector1_cfg, calwebb_tso1_cfg, calwebb_dark_cfg, output_dir, mode_used, rawdatrt = get_caldet1cfg_and_workingdir()
if mode_used != "BOTS" and mode_used.lower() != "dark":
cfg_file = calwebb_detector1_cfg
elif mode_used == "BOTS":
cfg_file = calwebb_tso1_cfg
elif mode_used.lower() == "dark":
cfg_file = calwebb_dark_cfg
configfile_used = "Using this configuration file: "+cfg_file
# Initiate the PTT log file
PTTcaldetector1_log = os.path.join(output_dir, 'PTT_caldetector1_'+detector+'.log')
print("Information outputed to screen from this script will be logged in file: ", PTTcaldetector1_log)
for handler in logging.root.handlers[:]:
logging.root.removeHandler(handler)
logging.basicConfig(filename=PTTcaldetector1_log, level=logging.DEBUG)
print(configfile_used)
logging.info(configfile_used)
# create the text file to record the names of the output files and the time the pipeline took to run
txt_outputs_summary = "cal_detector1_outputs_and_times_"+detector+".txt"
end_time_total = []
line0 = "# {:<20}".format("Input file: "+fits_input_uncal_file)
line1 = "# {:<16} {:<19} {:<20}".format("Step", "Output file", "Time to run [s]")
with open(txt_outputs_summary, "w+") as tf:
tf.write(line0+"\n")
tf.write(line1+"\n")
# Name of the file containing all the pipeline output
caldetector1_pipeline_log = "pipeline.log"
# copy the configuration file to create the pipeline log
stpipelogcfg = calwebb_detector1_cfg.replace("calwebb_detector1.cfg", "stpipe-log.cfg")
subprocess.run(["cp", stpipelogcfg, "."])
#final_output_caldet1 = "gain_scale.fits"
final_output_caldet1 = "final_output_caldet1_"+detector+".fits"
output_names = ["group_scale.fits", "dq_init.fits", "saturation.fits", "superbias.fits", "refpix.fits",
"lastframe.fits", "linearity.fits", "dark_current.fits", "jump.fits", "ramp_fit.fits",
final_output_caldet1]
if not step_by_step:
print("Got arguments and will run the calwebb_detector1 pipeline in full. This may take a while...")
# start the timer to compute the step running time
start_time = time.time()
result = Detector1Pipeline.call(fits_input_uncal_file, config_file=cfg_file)
result.save(final_output_caldet1)
# end the timer to compute pipeline running time
end_time = time.time() - start_time # this is in seconds
time2finish_string = " * calwebb_detector1 took "+repr(end_time)+" seconds to finish *"
print(time2finish_string)
logging.info(time2finish_string)
if end_time > 60.0:
end_time_min = round(end_time / 60.0, 1) # in minutes
tot_time = repr(end_time_min)+"min"
if end_time_min > 60.0:
end_time_hr = round(end_time_min / 60.0, 1) # in hrs
tot_time = repr(end_time_hr)+"hr"
else:
tot_time = str(round(end_time, 1))+"sec"
total_time = "{:<18} {:<20} {:<20}".format("", "total_time = ", repr(end_time)+" ="+tot_time)
# get the running time for the individual steps
if os.path.isfile(caldetector1_pipeline_log):
step_running_times = calculate_step_run_time(caldetector1_pipeline_log, output_names)
# write step running times in the text file
end_time_list = []
for outnm in output_names:
stp = outnm.replace(".fits", "")
if stp in step_running_times:
step_time = step_running_times[stp]["run_time"]
else:
step_time = "N/A"
end_time_list.append(step_time)
line2write = "{:<18} {:<20} {:<20}".format(stp, outnm, step_time)
with open(txt_outputs_summary, "a") as tf:
tf.write(line2write+"\n")
else:
print("No pipeline.log found. Unable to record times per step.")
else:
print("Got arguments and will run the calwebb_detector1 pipeline step by step.")
# steps to be ran, in order
steps_to_run = [GroupScaleStep(), DQInitStep(), SaturationStep(), SuperBiasStep(), RefPixStep(),
LastFrameStep(), LinearityStep(), DarkCurrentStep(), JumpStep(), RampFitStep(), GainScaleStep()]
comp_keys = ["S_GRPSCL", "S_DQINIT", "S_SATURA", "S_SUPERB", "S_REFPIX", "S_LASTFR", "S_LINEAR",
"S_DARK", "S_JUMP", "S_RAMP", "S_GANSCL"]
# run the pipeline step by step
for i, stp_instance in enumerate(steps_to_run):
stp = stp_instance
if i == 0:
step_input_file = fits_input_uncal_file
else:
# check if step was completed and find the appropriate input file
j = 1
continue_while = True
while continue_while:
step_input_file = output_names[i-j]
if (i-j == 0):
step_input_file = fits_input_uncal_file
break
if i == len(output_names)-1:
step_input_file = glob("*ramp*fit.fits")[0]
break
if os.path.isfile(step_input_file):
completion_key_val = fits.getval(step_input_file, comp_keys[i-j])
msg = "Checking for next step... "
completion_keywd_msg = " * Completion keyword: "+comp_keys[i-j]+" and value: "+completion_key_val
print(msg)
print(completion_keywd_msg)
logging.info(msg)
logging.info(completion_keywd_msg)
if "SKIPPED" in completion_key_val:
j += 1
elif "COMPLETE" in completion_key_val:
continue_while = False
running_stp_msg = "\n-> Running step: "+str(stp)+" with input file: "+step_input_file
output_msg = " output will be saved as: "+output_names[i]
logging.info(running_stp_msg)
logging.info(output_msg)
# start the timer to compute the step running time
start_time = time.time()
if "ramp" not in output_names[i]:
result = stp.call(step_input_file)
result.save(output_names[i])
else:
# the pipeline works differently for the ramp_fit step because it has more than one output
# this step is also hanging from the command line
#subprocess.call(["strun", "jwst.ramp_fitting.RampFitStep", "jump.fits"])
(out_slope, int_slope) = stp.call(step_input_file)
out_slope.save(output_names[i])
try:
int_slope.save("ramp_fit_int.fits")
except AttributeError:
msg = "File has only 1 integration."
print(msg)
logging.info(msg)
# end the timer to compute cal_detector1 running time
et = time.time() - start_time # this is in seconds
end_time = repr(et)
end_time_total.append(et)
step = output_names[i].replace(".fits", "")
msg = " * calwebb_detector1 step "+step+" took "+end_time+" seconds to finish * \n"
print(msg)
logging.info(msg)
if et > 60.0:
end_time_min = round(et / 60.0, 1) # in minutes
end_time = repr(end_time_min)+"min"
if end_time_min > 60.0:
end_time_hr = round(end_time_min / 60.0, 1) # in hrs
end_time = repr(end_time_hr)+"hr"
else:
end_time = repr(round(et, 1))+"sec"
# record results in text file
line2write = "{:<18} {:<20} {:<20}".format(step, output_names[i], end_time)
with open(txt_outputs_summary, "a") as tf:
tf.write(line2write+"\n")
# record total time in text file
tot_time_sec = sum(end_time_total)
if tot_time_sec > 60.0:
tot_time_min = round((tot_time_sec/60.0), 1)
tot_time = repr(tot_time_min)+"min"
if tot_time_min > 60.0:
tot_time_hr = round((tot_time_min/60.0), 1)
tot_time = repr(tot_time_hr)+"hr"
else:
tot_time = round((tot_time_sec/60.0), 1)
total_time = "{:<18} {:>20} {:>20}".format("", "total_time ", repr(tot_time_sec)+" ="+tot_time)
# record total time in text file
with open(txt_outputs_summary, "a") as tf:
tf.write(total_time+"\n")
msg = "\n ** Calwebb_detector 1 took "+repr(tot_time)+" to complete **"
print(msg)
logging.info(msg)
# Move products to working dir
# rename and move the pipeline log file
new_name = "caldetector1_pipeline_"+detector+".log"
if os.path.isfile(caldetector1_pipeline_log):
os.rename(caldetector1_pipeline_log, os.path.join(output_dir, new_name))
# move the PTT log file and the fits intermediary product files
if os.getcwd() != output_dir:
fits_list = glob("*.fits")
if len(fits_list) >= 1:
msg = "Output fits files are located at: "+output_dir
print(msg)
logging.info(msg)
for ff in fits_list:
if "step_" in ff:
ff_newname = os.path.join(output_dir, ff.replace("step_", ""))
else:
ff_newname = os.path.join(output_dir, ff)
if detector.lower() not in ff.lower():
ff_newname = ff_newname.replace(".fits", "_"+detector+".fits")
subprocess.run(["mv", ff, ff_newname])
# move text files too
subprocess.run(["mv", txt_outputs_summary, output_dir])
else:
msg = "No fits files detected after calwbb_detector1 finished. Exiting script."
print(msg)
logging.info(msg)
msg = "Script run_cal_detector1.py finished."
print(msg)
logging.info(msg)
def calwebb_detector1_save_jump(input_file,
output_dir,
ramp_fit=True,
save_fitopt=True):
"""Call ``calwebb_detector1`` on the provided file, running all
steps up to the ``ramp_fit`` step, and save the result. Optionally
run the ``ramp_fit`` step and save the resulting slope file as well.
Parameters
----------
input_file : str
Name of fits file to run on the pipeline
output_dir : str
Directory into which the pipeline outputs are saved
ramp_fit : bool
If ``False``, the ``ramp_fit`` step is not run. The output file
will be a ``*_jump.fits`` file.
If ``True``, the ``*jump.fits`` file will be produced and saved.
In addition, the ``ramp_fit`` step will be run and a
``*rate.fits`` or ``*_rateints.fits`` file will be saved.
(``rateints`` if the input file has >1 integration)
save_fitopt : bool
If ``True``, the file of optional outputs from the ramp fitting
step of the pipeline is saved.
Returns
-------
jump_output : str
Name of the saved file containing the output prior to the
``ramp_fit`` step.
pipe_output : str
Name of the saved file containing the output after ramp-fitting
is performed (if requested). Otherwise ``None``.
"""
input_file_only = os.path.basename(input_file)
# Find the instrument used to collect the data
datamodel = datamodels.RampModel(input_file)
instrument = datamodel.meta.instrument.name.lower()
# If the data pre-date jwst version 1.2.1, then they will have
# the NUMDTHPT keyword (with string value of the number of dithers)
# rather than the newer NRIMDTPT keyword (with an integer value of
# the number of dithers). If so, we need to update the file here so
# that it doesn't cause the pipeline to crash later. Both old and
# new keywords are mapped to the model.meta.dither.dither_points
# metadata entry. So we should be able to focus on that.
if isinstance(datamodel.meta.dither.dither_points, str):
# If we have a string, change it to an integer
datamodel.meta.dither.dither_points = int(
datamodel.meta.dither.dither_points)
elif datamodel.meta.dither.dither_points is None:
# If the information is missing completely, put in a dummy value
datamodel.meta.dither.dither_points = 1
# Switch to calling the pipeline rather than individual steps,
# and use the run() method so that we can set parameters
# progammatically.
model = Detector1Pipeline()
# Always true
if instrument == 'nircam':
model.refpix.odd_even_rows = False
# Default CR rejection threshold is too low
model.jump.rejection_threshold = 15
# Turn off IPC step until it is put in the right place
model.ipc.skip = True
model.jump.save_results = True
model.jump.output_dir = output_dir
jump_output = os.path.join(output_dir,
input_file_only.replace('uncal', 'jump'))
# Check to see if the jump version of the requested file is already
# present
run_jump = not os.path.isfile(jump_output)
if ramp_fit:
model.ramp_fit.save_results = True
# model.save_results = True
model.output_dir = output_dir
# pipe_output = os.path.join(output_dir, input_file_only.replace('uncal', 'rate'))
pipe_output = os.path.join(
output_dir, input_file_only.replace('uncal', '0_ramp_fit'))
run_slope = not os.path.isfile(pipe_output)
if save_fitopt:
model.ramp_fit.save_opt = True
fitopt_output = os.path.join(
output_dir, input_file_only.replace('uncal', 'fitopt'))
run_fitopt = not os.path.isfile(fitopt_output)
else:
model.ramp_fit.save_opt = False
fitopt_output = None
run_fitopt = False
else:
model.ramp_fit.skip = True
pipe_output = None
fitopt_output = None
run_slope = False
run_fitopt = False
# Call the pipeline if any of the files at the requested calibration
# states are not present in the output directory
if run_jump or (ramp_fit and run_slope) or (save_fitopt and run_fitopt):
model.run(datamodel)
else:
print((
"Files with all requested calibration states for {} already present in "
"output directory. Skipping pipeline call.".format(input_file)))
return jump_output, pipe_output, fitopt_output
#final_output_caldet1 = "gain_scale.fits"
final_output_caldet1 = "final_output_caldet1_" + detector + ".fits"
output_names = [
"group_scale.fits", "dq_init.fits", "saturation.fits", "superbias.fits",
"refpix.fits", "lastframe.fits", "linearity.fits", "dark_current.fits",
"jump.fits", "ramp_fit.fits", final_output_caldet1
]
if not step_by_step:
print(
"Got arguments and will run the calwebb_detector1 pipeline in full. This may take a while..."
)
# start the timer to compute the step running time
start_time = time.time()
result = Detector1Pipeline.call(fits_input_uncal_file,
config_file=cfg_file)
result.save(final_output_caldet1)
# end the timer to compute pipeline running time
end_time = time.time() - start_time # this is in seconds
time2finish_string = " * calwebb_detector1 took " + repr(
end_time) + " seconds to finish *"
print(time2finish_string)
logging.info(time2finish_string)
if end_time > 60.0:
end_time_min = round(end_time / 60.0, 1) # in minutes
tot_time = repr(end_time_min) + "min"
if end_time_min > 60.0:
end_time_hr = round(end_time_min / 60.0, 1) # in hrs
tot_time = repr(end_time_hr) + "hr"
else:
def test_detector1pipeline1(_bigdata):
"""
Regression test of calwebb_detector1 pipeline performed on MIRI data.
"""
step = Detector1Pipeline()
step.save_calibrated_ramp = True
step.ipc.skip = True
step.refpix.odd_even_columns = True
step.refpix.use_side_ref_pixels = True
step.refpix.side_smoothing_length=11
step.refpix.side_gain=1.0
step.refpix.odd_even_rows = True
step.persistence.skip = True
step.jump.rejection_threshold = 250.0
step.ramp_fit.save_opt = False
step.output_file='jw00001001001_01101_00001_MIRIMAGE'
step.suffix='rate'
step.run(_bigdata+'/miri/test_sloperpipeline/jw00001001001_01101_00001_MIRIMAGE_uncal.fits'
)
# Compare calibrated ramp product
n_cr = 'jw00001001001_01101_00001_MIRIMAGE_ramp.fits'
h = fits.open( n_cr )
n_ref = _bigdata+'/miri/test_sloperpipeline/jw00001001001_01101_00001_MIRIMAGE_uncal_jump.fits'
href = fits.open( n_ref )
newh = fits.HDUList([h['primary'],h['sci'],h['err'],h['groupdq'],h['pixeldq']])
newhref = fits.HDUList([href['primary'],href['sci'],href['err'],href['groupdq'],href['pixeldq']])
result = fits.diff.FITSDiff(newh,
newhref,
ignore_keywords = ['DATE','CAL_VER','CAL_VCS','CRDS_VER','CRDS_CTX'],
rtol = 0.00001
)
assert result.identical, result.report()
# Compare multi-integration countrate image product
n_int = 'jw00001001001_01101_00001_MIRIMAGE_rateints.fits'
h = fits.open( n_int )
n_ref = _bigdata+'/miri/test_sloperpipeline/jw00001001001_01101_00001_MIRIMAGE_uncal_integ.fits'
href = fits.open( n_ref )
newh = fits.HDUList([h['primary'],h['sci'],h['err'],h['dq']])
newhref = fits.HDUList([href['primary'],href['sci'],href['err'],href['dq']])
result = fits.diff.FITSDiff(newh,
newhref,
ignore_keywords = ['DATE','CAL_VER','CAL_VCS','CRDS_VER','CRDS_CTX'],
rtol = 0.00001
)
assert result.identical, result.report()
# Compare countrate image product
n_rate = 'jw00001001001_01101_00001_MIRIMAGE_rate.fits'
h = fits.open( n_rate )
n_ref = _bigdata+'/miri/test_sloperpipeline/jw00001001001_01101_00001_MIRIMAGE_uncal_MiriSloperPipeline.fits'
href = fits.open( n_ref )
newh = fits.HDUList([h['primary'],h['sci'],h['err'],h['dq']])
newhref = fits.HDUList([href['primary'],href['sci'],href['err'],href['dq']])
result = fits.diff.FITSDiff(newh,
newhref,
ignore_keywords = ['DATE','CAL_VER','CAL_VCS','CRDS_VER','CRDS_CTX'],
rtol = 0.00001
)
assert result.identical, result.report()