def run_caldet1(fits_input_uncal_file, step_by_step=False):
if not os.path.isfile(fits_input_uncal_file):
print("Input file not found in the current directory. Unable to proceed, exiting script.")
exit()
# Get the detector used
detector = fits.getval(fits_input_uncal_file, "DETECTOR", 0)
# Get the cfg file
calwebb_detector1_cfg, calwebb_tso1_cfg, calwebb_dark_cfg, output_dir, mode_used, rawdatrt = get_caldet1cfg_and_workingdir()
if mode_used != "BOTS" and mode_used.lower() != "dark":
cfg_file = calwebb_detector1_cfg
elif mode_used == "BOTS":
cfg_file = calwebb_tso1_cfg
elif mode_used.lower() == "dark":
cfg_file = calwebb_dark_cfg
configfile_used = "Using this configuration file: "+cfg_file
# Initiate the PTT log file
PTTcaldetector1_log = os.path.join(output_dir, 'PTT_caldetector1_'+detector+'.log')
print("Information outputed to screen from this script will be logged in file: ", PTTcaldetector1_log)
for handler in logging.root.handlers[:]:
logging.root.removeHandler(handler)
logging.basicConfig(filename=PTTcaldetector1_log, level=logging.DEBUG)
print(configfile_used)
logging.info(configfile_used)
# create the text file to record the names of the output files and the time the pipeline took to run
txt_outputs_summary = "cal_detector1_outputs_and_times_"+detector+".txt"
end_time_total = []
line0 = "# {:<20}".format("Input file: "+fits_input_uncal_file)
line1 = "# {:<16} {:<19} {:<20}".format("Step", "Output file", "Time to run [s]")
with open(txt_outputs_summary, "w+") as tf:
tf.write(line0+"\n")
tf.write(line1+"\n")
# Name of the file containing all the pipeline output
caldetector1_pipeline_log = "pipeline.log"
# copy the configuration file to create the pipeline log
stpipelogcfg = calwebb_detector1_cfg.replace("calwebb_detector1.cfg", "stpipe-log.cfg")
subprocess.run(["cp", stpipelogcfg, "."])
#final_output_caldet1 = "gain_scale.fits"
final_output_caldet1 = "final_output_caldet1_"+detector+".fits"
output_names = ["group_scale.fits", "dq_init.fits", "saturation.fits", "superbias.fits", "refpix.fits",
"lastframe.fits", "linearity.fits", "dark_current.fits", "jump.fits", "ramp_fit.fits",
final_output_caldet1]
if not step_by_step:
print("Got arguments and will run the calwebb_detector1 pipeline in full. This may take a while...")
# start the timer to compute the step running time
start_time = time.time()
result = Detector1Pipeline.call(fits_input_uncal_file, config_file=cfg_file)
result.save(final_output_caldet1)
# end the timer to compute pipeline running time
end_time = time.time() - start_time # this is in seconds
time2finish_string = " * calwebb_detector1 took "+repr(end_time)+" seconds to finish *"
print(time2finish_string)
logging.info(time2finish_string)
if end_time > 60.0:
end_time_min = round(end_time / 60.0, 1) # in minutes
tot_time = repr(end_time_min)+"min"
if end_time_min > 60.0:
end_time_hr = round(end_time_min / 60.0, 1) # in hrs
tot_time = repr(end_time_hr)+"hr"
else:
tot_time = str(round(end_time, 1))+"sec"
total_time = "{:<18} {:<20} {:<20}".format("", "total_time = ", repr(end_time)+" ="+tot_time)
# get the running time for the individual steps
if os.path.isfile(caldetector1_pipeline_log):
step_running_times = calculate_step_run_time(caldetector1_pipeline_log, output_names)
# write step running times in the text file
end_time_list = []
for outnm in output_names:
stp = outnm.replace(".fits", "")
if stp in step_running_times:
step_time = step_running_times[stp]["run_time"]
else:
step_time = "N/A"
end_time_list.append(step_time)
line2write = "{:<18} {:<20} {:<20}".format(stp, outnm, step_time)
with open(txt_outputs_summary, "a") as tf:
tf.write(line2write+"\n")
else:
print("No pipeline.log found. Unable to record times per step.")
else:
print("Got arguments and will run the calwebb_detector1 pipeline step by step.")
# steps to be ran, in order
steps_to_run = [GroupScaleStep(), DQInitStep(), SaturationStep(), SuperBiasStep(), RefPixStep(),
LastFrameStep(), LinearityStep(), DarkCurrentStep(), JumpStep(), RampFitStep(), GainScaleStep()]
comp_keys = ["S_GRPSCL", "S_DQINIT", "S_SATURA", "S_SUPERB", "S_REFPIX", "S_LASTFR", "S_LINEAR",
"S_DARK", "S_JUMP", "S_RAMP", "S_GANSCL"]
# run the pipeline step by step
for i, stp_instance in enumerate(steps_to_run):
stp = stp_instance
if i == 0:
step_input_file = fits_input_uncal_file
else:
# check if step was completed and find the appropriate input file
j = 1
continue_while = True
while continue_while:
step_input_file = output_names[i-j]
if (i-j == 0):
step_input_file = fits_input_uncal_file
break
if i == len(output_names)-1:
step_input_file = glob("*ramp*fit.fits")[0]
break
if os.path.isfile(step_input_file):
completion_key_val = fits.getval(step_input_file, comp_keys[i-j])
msg = "Checking for next step... "
completion_keywd_msg = " * Completion keyword: "+comp_keys[i-j]+" and value: "+completion_key_val
print(msg)
print(completion_keywd_msg)
logging.info(msg)
logging.info(completion_keywd_msg)
if "SKIPPED" in completion_key_val:
j += 1
elif "COMPLETE" in completion_key_val:
continue_while = False
running_stp_msg = "\n-> Running step: "+str(stp)+" with input file: "+step_input_file
output_msg = " output will be saved as: "+output_names[i]
logging.info(running_stp_msg)
logging.info(output_msg)
# start the timer to compute the step running time
start_time = time.time()
if "ramp" not in output_names[i]:
result = stp.call(step_input_file)
result.save(output_names[i])
else:
# the pipeline works differently for the ramp_fit step because it has more than one output
# this step is also hanging from the command line
#subprocess.call(["strun", "jwst.ramp_fitting.RampFitStep", "jump.fits"])
(out_slope, int_slope) = stp.call(step_input_file)
out_slope.save(output_names[i])
try:
int_slope.save("ramp_fit_int.fits")
except AttributeError:
msg = "File has only 1 integration."
print(msg)
logging.info(msg)
# end the timer to compute cal_detector1 running time
et = time.time() - start_time # this is in seconds
end_time = repr(et)
end_time_total.append(et)
step = output_names[i].replace(".fits", "")
msg = " * calwebb_detector1 step "+step+" took "+end_time+" seconds to finish * \n"
print(msg)
logging.info(msg)
if et > 60.0:
end_time_min = round(et / 60.0, 1) # in minutes
end_time = repr(end_time_min)+"min"
if end_time_min > 60.0:
end_time_hr = round(end_time_min / 60.0, 1) # in hrs
end_time = repr(end_time_hr)+"hr"
else:
end_time = repr(round(et, 1))+"sec"
# record results in text file
line2write = "{:<18} {:<20} {:<20}".format(step, output_names[i], end_time)
with open(txt_outputs_summary, "a") as tf:
tf.write(line2write+"\n")
# record total time in text file
tot_time_sec = sum(end_time_total)
if tot_time_sec > 60.0:
tot_time_min = round((tot_time_sec/60.0), 1)
tot_time = repr(tot_time_min)+"min"
if tot_time_min > 60.0:
tot_time_hr = round((tot_time_min/60.0), 1)
tot_time = repr(tot_time_hr)+"hr"
else:
tot_time = round((tot_time_sec/60.0), 1)
total_time = "{:<18} {:>20} {:>20}".format("", "total_time ", repr(tot_time_sec)+" ="+tot_time)
# record total time in text file
with open(txt_outputs_summary, "a") as tf:
tf.write(total_time+"\n")
msg = "\n ** Calwebb_detector 1 took "+repr(tot_time)+" to complete **"
print(msg)
logging.info(msg)
# Move products to working dir
# rename and move the pipeline log file
new_name = "caldetector1_pipeline_"+detector+".log"
if os.path.isfile(caldetector1_pipeline_log):
os.rename(caldetector1_pipeline_log, os.path.join(output_dir, new_name))
# move the PTT log file and the fits intermediary product files
if os.getcwd() != output_dir:
fits_list = glob("*.fits")
if len(fits_list) >= 1:
msg = "Output fits files are located at: "+output_dir
print(msg)
logging.info(msg)
for ff in fits_list:
if "step_" in ff:
ff_newname = os.path.join(output_dir, ff.replace("step_", ""))
else:
ff_newname = os.path.join(output_dir, ff)
if detector.lower() not in ff.lower():
ff_newname = ff_newname.replace(".fits", "_"+detector+".fits")
subprocess.run(["mv", ff, ff_newname])
# move text files too
subprocess.run(["mv", txt_outputs_summary, output_dir])
else:
msg = "No fits files detected after calwbb_detector1 finished. Exiting script."
print(msg)
logging.info(msg)
msg = "Script run_cal_detector1.py finished."
print(msg)
logging.info(msg)
tf.write(line1 + "\n")
final_out = "gain_scale.fits"
output_names = [
"group_scale.fits", "dq_init.fits", "saturation.fits", "superbias.fits",
"refpix.fits", "rscd.fits", "lastframe.fits", "linearity.fits",
"dark_current.fits", "jump.fits", "ramp_fit.fits", final_out
]
# Get and save the value of the raw data root name to add at the end of calwebb_detector1
#rawdatrt = fits.getval(fits_input_uncal_file, 'rawdatrt', 0)
if not step_by_step:
# start the timer to compute the step running time
start_time = time.time()
Detector1Pipeline.call(fits_input_uncal_file,
config_file=calwebb_detector1_cfg)
# end the timer to compute calwebb_spec2 running time
end_time = time.time() - start_time # this is in seconds
print(" * calwebb_detector1 took " + repr(end_time) +
" seconds to finish *")
total_time = "{:<18} {:<20} {:<20}".format("", "total time = ",
repr(end_time))
tot_time_min = round((end_time / 60.0), 2)
for outnm in output_names:
line2write = "{:<18} {:<20} {:<20}".format(outnm.replace(".fits", ""),
outnm, "")
with open(txt_outputs_summary, "a") as tf:
tf.write(line2write + "\n")
else:
# steps to be ran, in order
steps_to_run = [
#final_output_caldet1 = "gain_scale.fits"
final_output_caldet1 = "final_output_caldet1_" + detector + ".fits"
output_names = [
"group_scale.fits", "dq_init.fits", "saturation.fits", "superbias.fits",
"refpix.fits", "lastframe.fits", "linearity.fits", "dark_current.fits",
"jump.fits", "ramp_fit.fits", final_output_caldet1
]
if not step_by_step:
print(
"Got arguments and will run the calwebb_detector1 pipeline in full. This may take a while..."
)
# start the timer to compute the step running time
start_time = time.time()
result = Detector1Pipeline.call(fits_input_uncal_file,
config_file=cfg_file)
result.save(final_output_caldet1)
# end the timer to compute pipeline running time
end_time = time.time() - start_time # this is in seconds
time2finish_string = " * calwebb_detector1 took " + repr(
end_time) + " seconds to finish *"
print(time2finish_string)
logging.info(time2finish_string)
if end_time > 60.0:
end_time_min = round(end_time / 60.0, 1) # in minutes
tot_time = repr(end_time_min) + "min"
if end_time_min > 60.0:
end_time_hr = round(end_time_min / 60.0, 1) # in hrs
tot_time = repr(end_time_hr) + "hr"
else: