def run_caldet1(fits_input_uncal_file, step_by_step=False): if not os.path.isfile(fits_input_uncal_file): print("Input file not found in the current directory. Unable to proceed, exiting script.") exit() # Get the detector used detector = fits.getval(fits_input_uncal_file, "DETECTOR", 0) # Get the cfg file calwebb_detector1_cfg, calwebb_tso1_cfg, calwebb_dark_cfg, output_dir, mode_used, rawdatrt = get_caldet1cfg_and_workingdir() if mode_used != "BOTS" and mode_used.lower() != "dark": cfg_file = calwebb_detector1_cfg elif mode_used == "BOTS": cfg_file = calwebb_tso1_cfg elif mode_used.lower() == "dark": cfg_file = calwebb_dark_cfg configfile_used = "Using this configuration file: "+cfg_file # Initiate the PTT log file PTTcaldetector1_log = os.path.join(output_dir, 'PTT_caldetector1_'+detector+'.log') print("Information outputed to screen from this script will be logged in file: ", PTTcaldetector1_log) for handler in logging.root.handlers[:]: logging.root.removeHandler(handler) logging.basicConfig(filename=PTTcaldetector1_log, level=logging.DEBUG) print(configfile_used) logging.info(configfile_used) # create the text file to record the names of the output files and the time the pipeline took to run txt_outputs_summary = "cal_detector1_outputs_and_times_"+detector+".txt" end_time_total = [] line0 = "# {:<20}".format("Input file: "+fits_input_uncal_file) line1 = "# {:<16} {:<19} {:<20}".format("Step", "Output file", "Time to run [s]") with open(txt_outputs_summary, "w+") as tf: tf.write(line0+"\n") tf.write(line1+"\n") # Name of the file containing all the pipeline output caldetector1_pipeline_log = "pipeline.log" # copy the configuration file to create the pipeline log stpipelogcfg = calwebb_detector1_cfg.replace("calwebb_detector1.cfg", "stpipe-log.cfg") subprocess.run(["cp", stpipelogcfg, "."]) #final_output_caldet1 = "gain_scale.fits" final_output_caldet1 = "final_output_caldet1_"+detector+".fits" output_names = ["group_scale.fits", "dq_init.fits", "saturation.fits", "superbias.fits", "refpix.fits", "lastframe.fits", "linearity.fits", "dark_current.fits", "jump.fits", "ramp_fit.fits", final_output_caldet1] if not step_by_step: print("Got arguments and will run the calwebb_detector1 pipeline in full. This may take a while...") # start the timer to compute the step running time start_time = time.time() result = Detector1Pipeline.call(fits_input_uncal_file, config_file=cfg_file) result.save(final_output_caldet1) # end the timer to compute pipeline running time end_time = time.time() - start_time # this is in seconds time2finish_string = " * calwebb_detector1 took "+repr(end_time)+" seconds to finish *" print(time2finish_string) logging.info(time2finish_string) if end_time > 60.0: end_time_min = round(end_time / 60.0, 1) # in minutes tot_time = repr(end_time_min)+"min" if end_time_min > 60.0: end_time_hr = round(end_time_min / 60.0, 1) # in hrs tot_time = repr(end_time_hr)+"hr" else: tot_time = str(round(end_time, 1))+"sec" total_time = "{:<18} {:<20} {:<20}".format("", "total_time = ", repr(end_time)+" ="+tot_time) # get the running time for the individual steps if os.path.isfile(caldetector1_pipeline_log): step_running_times = calculate_step_run_time(caldetector1_pipeline_log, output_names) # write step running times in the text file end_time_list = [] for outnm in output_names: stp = outnm.replace(".fits", "") if stp in step_running_times: step_time = step_running_times[stp]["run_time"] else: step_time = "N/A" end_time_list.append(step_time) line2write = "{:<18} {:<20} {:<20}".format(stp, outnm, step_time) with open(txt_outputs_summary, "a") as tf: tf.write(line2write+"\n") else: print("No pipeline.log found. Unable to record times per step.") else: print("Got arguments and will run the calwebb_detector1 pipeline step by step.") # steps to be ran, in order steps_to_run = [GroupScaleStep(), DQInitStep(), SaturationStep(), SuperBiasStep(), RefPixStep(), LastFrameStep(), LinearityStep(), DarkCurrentStep(), JumpStep(), RampFitStep(), GainScaleStep()] comp_keys = ["S_GRPSCL", "S_DQINIT", "S_SATURA", "S_SUPERB", "S_REFPIX", "S_LASTFR", "S_LINEAR", "S_DARK", "S_JUMP", "S_RAMP", "S_GANSCL"] # run the pipeline step by step for i, stp_instance in enumerate(steps_to_run): stp = stp_instance if i == 0: step_input_file = fits_input_uncal_file else: # check if step was completed and find the appropriate input file j = 1 continue_while = True while continue_while: step_input_file = output_names[i-j] if (i-j == 0): step_input_file = fits_input_uncal_file break if i == len(output_names)-1: step_input_file = glob("*ramp*fit.fits")[0] break if os.path.isfile(step_input_file): completion_key_val = fits.getval(step_input_file, comp_keys[i-j]) msg = "Checking for next step... " completion_keywd_msg = " * Completion keyword: "+comp_keys[i-j]+" and value: "+completion_key_val print(msg) print(completion_keywd_msg) logging.info(msg) logging.info(completion_keywd_msg) if "SKIPPED" in completion_key_val: j += 1 elif "COMPLETE" in completion_key_val: continue_while = False running_stp_msg = "\n-> Running step: "+str(stp)+" with input file: "+step_input_file output_msg = " output will be saved as: "+output_names[i] logging.info(running_stp_msg) logging.info(output_msg) # start the timer to compute the step running time start_time = time.time() if "ramp" not in output_names[i]: result = stp.call(step_input_file) result.save(output_names[i]) else: # the pipeline works differently for the ramp_fit step because it has more than one output # this step is also hanging from the command line #subprocess.call(["strun", "jwst.ramp_fitting.RampFitStep", "jump.fits"]) (out_slope, int_slope) = stp.call(step_input_file) out_slope.save(output_names[i]) try: int_slope.save("ramp_fit_int.fits") except AttributeError: msg = "File has only 1 integration." print(msg) logging.info(msg) # end the timer to compute cal_detector1 running time et = time.time() - start_time # this is in seconds end_time = repr(et) end_time_total.append(et) step = output_names[i].replace(".fits", "") msg = " * calwebb_detector1 step "+step+" took "+end_time+" seconds to finish * \n" print(msg) logging.info(msg) if et > 60.0: end_time_min = round(et / 60.0, 1) # in minutes end_time = repr(end_time_min)+"min" if end_time_min > 60.0: end_time_hr = round(end_time_min / 60.0, 1) # in hrs end_time = repr(end_time_hr)+"hr" else: end_time = repr(round(et, 1))+"sec" # record results in text file line2write = "{:<18} {:<20} {:<20}".format(step, output_names[i], end_time) with open(txt_outputs_summary, "a") as tf: tf.write(line2write+"\n") # record total time in text file tot_time_sec = sum(end_time_total) if tot_time_sec > 60.0: tot_time_min = round((tot_time_sec/60.0), 1) tot_time = repr(tot_time_min)+"min" if tot_time_min > 60.0: tot_time_hr = round((tot_time_min/60.0), 1) tot_time = repr(tot_time_hr)+"hr" else: tot_time = round((tot_time_sec/60.0), 1) total_time = "{:<18} {:>20} {:>20}".format("", "total_time ", repr(tot_time_sec)+" ="+tot_time) # record total time in text file with open(txt_outputs_summary, "a") as tf: tf.write(total_time+"\n") msg = "\n ** Calwebb_detector 1 took "+repr(tot_time)+" to complete **" print(msg) logging.info(msg) # Move products to working dir # rename and move the pipeline log file new_name = "caldetector1_pipeline_"+detector+".log" if os.path.isfile(caldetector1_pipeline_log): os.rename(caldetector1_pipeline_log, os.path.join(output_dir, new_name)) # move the PTT log file and the fits intermediary product files if os.getcwd() != output_dir: fits_list = glob("*.fits") if len(fits_list) >= 1: msg = "Output fits files are located at: "+output_dir print(msg) logging.info(msg) for ff in fits_list: if "step_" in ff: ff_newname = os.path.join(output_dir, ff.replace("step_", "")) else: ff_newname = os.path.join(output_dir, ff) if detector.lower() not in ff.lower(): ff_newname = ff_newname.replace(".fits", "_"+detector+".fits") subprocess.run(["mv", ff, ff_newname]) # move text files too subprocess.run(["mv", txt_outputs_summary, output_dir]) else: msg = "No fits files detected after calwbb_detector1 finished. Exiting script." print(msg) logging.info(msg) msg = "Script run_cal_detector1.py finished." print(msg) logging.info(msg)
tf.write(line1 + "\n") final_out = "gain_scale.fits" output_names = [ "group_scale.fits", "dq_init.fits", "saturation.fits", "superbias.fits", "refpix.fits", "rscd.fits", "lastframe.fits", "linearity.fits", "dark_current.fits", "jump.fits", "ramp_fit.fits", final_out ] # Get and save the value of the raw data root name to add at the end of calwebb_detector1 #rawdatrt = fits.getval(fits_input_uncal_file, 'rawdatrt', 0) if not step_by_step: # start the timer to compute the step running time start_time = time.time() Detector1Pipeline.call(fits_input_uncal_file, config_file=calwebb_detector1_cfg) # end the timer to compute calwebb_spec2 running time end_time = time.time() - start_time # this is in seconds print(" * calwebb_detector1 took " + repr(end_time) + " seconds to finish *") total_time = "{:<18} {:<20} {:<20}".format("", "total time = ", repr(end_time)) tot_time_min = round((end_time / 60.0), 2) for outnm in output_names: line2write = "{:<18} {:<20} {:<20}".format(outnm.replace(".fits", ""), outnm, "") with open(txt_outputs_summary, "a") as tf: tf.write(line2write + "\n") else: # steps to be ran, in order steps_to_run = [
#final_output_caldet1 = "gain_scale.fits" final_output_caldet1 = "final_output_caldet1_" + detector + ".fits" output_names = [ "group_scale.fits", "dq_init.fits", "saturation.fits", "superbias.fits", "refpix.fits", "lastframe.fits", "linearity.fits", "dark_current.fits", "jump.fits", "ramp_fit.fits", final_output_caldet1 ] if not step_by_step: print( "Got arguments and will run the calwebb_detector1 pipeline in full. This may take a while..." ) # start the timer to compute the step running time start_time = time.time() result = Detector1Pipeline.call(fits_input_uncal_file, config_file=cfg_file) result.save(final_output_caldet1) # end the timer to compute pipeline running time end_time = time.time() - start_time # this is in seconds time2finish_string = " * calwebb_detector1 took " + repr( end_time) + " seconds to finish *" print(time2finish_string) logging.info(time2finish_string) if end_time > 60.0: end_time_min = round(end_time / 60.0, 1) # in minutes tot_time = repr(end_time_min) + "min" if end_time_min > 60.0: end_time_hr = round(end_time_min / 60.0, 1) # in hrs tot_time = repr(end_time_hr) + "hr" else: