def output_hdul(set_inandout_filenames, config): set_inandout_filenames_info = core_utils.read_info4outputhdul(config, set_inandout_filenames) step, txt_name, step_input_file, step_output_file, run_calwebb_spec2, outstep_file_suffix = set_inandout_filenames_info skip_runing_pipe_step = config.getboolean("tests_only", "_".join((step, "tests"))) # Only run step if data is not IFU inhdu = core_utils.read_hdrfits(step_input_file, info=False, show_hdr=False) end_time = '0.0' print("Is the input file IFU? = ", core_utils.check_IFU_true(inhdu)) if not core_utils.check_IFU_true(inhdu): stp = ResampleSpecStep() # if run_calwebb_spec2 is True calwebb_spec2 will be called, else individual steps will be ran step_completed = False if run_calwebb_spec2: # read the assign wcs fits file local_step_output_file = core_utils.read_completion_to_full_run_map("full_run_map.txt", step) hdul = core_utils.read_hdrfits(local_step_output_file, info=False, show_hdr=False) # move the output file into the working directory working_directory = config.get("calwebb_spec2_input_file", "working_directory") step_output_file = os.path.join(working_directory, local_step_output_file) print ("Step product was saved as: ", step_output_file) subprocess.run(["mv", local_step_output_file, step_output_file]) return hdul else: if config.getboolean("steps", step): print ("*** Step "+step+" set to True") if os.path.isfile(step_input_file): if not skip_runing_pipe_step: # get the right configuration files to run the step local_pipe_cfg_path = config.get("calwebb_spec2_input_file", "local_pipe_cfg_path") # start the timer to compute the step running time start_time = time.time() if local_pipe_cfg_path == "pipe_source_tree_code": result = stp.call(step_input_file) else: result = stp.call(step_input_file, config_file=local_pipe_cfg_path+'/resample_spec.cfg') result.save(step_output_file) # end the timer to compute the step running time end_time = repr(time.time() - start_time) # this is in seconds print("Step "+step+" took "+end_time+" seconds to finish") step_completed = True core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time) hdul = core_utils.read_hdrfits(step_output_file, info=False, show_hdr=False) return hdul else: core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time) pytest.skip("Skipping "+step+" because the input file does not exist.") else: core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time) pytest.skip("Skipping "+step+". Step set to False in configuration file.") else: pytest.skip("Skipping "+step+" because data is IFU data and the resample will be done in cube_build.")
def output_hdul(set_inandout_filenames, config): set_inandout_filenames_info = core_utils.read_info4outputhdul( config, set_inandout_filenames) step, txt_name, step_input_file, step_output_file, run_calwebb_spec2, outstep_file_suffix = set_inandout_filenames_info run_pipe_step = config.getboolean("run_pipe_steps", step) # determine which tests are to be run resample_spec_completion_tests = config.getboolean( "run_pytest", "_".join((step, "completion", "tests"))) #resample_spec_reffile_tests = config.getboolean("run_pytest", "_".join((step, "reffile", "tests"))) #resample_spec_validation_tests = config.getboolean("run_pytest", "_".join((step, "validation", "tests"))) run_pytests = [ resample_spec_completion_tests ] #, resample_spec_reffile_tests, resample_spec_validation_tests] end_time = '0.0' # Only run step if data is not IFU or BOTS mode_used = config.get("calwebb_spec2_input_file", "mode_used").lower() working_directory = config.get("calwebb_spec2_input_file", "working_directory") initial_input_file = config.get("calwebb_spec2_input_file", "input_file") initial_input_file = os.path.join(working_directory, initial_input_file) detector = fits.getval(initial_input_file, "DETECTOR", 0) if not os.path.isfile(initial_input_file): pytest.skip( "Skipping " + step + " because the initial input file given in PTT_config.cfg does not exist." ) if mode_used != "bots" and mode_used != "ifu": # if run_calwebb_spec2 is True calwebb_spec2 will be called, else individual steps will be ran step_completed = False # check if the filter is to be changed change_filter_opaque = config.getboolean("calwebb_spec2_input_file", "change_filter_opaque") if change_filter_opaque: is_filter_opaque, step_input_filename = change_filter_opaque2science.change_filter_opaque( step_input_file, step=step) if is_filter_opaque: filter_opaque_msg = "With FILTER=OPAQUE, the calwebb_spec2 will run up to the extract_2d step. Resample pytest now set to Skip." print(filter_opaque_msg) core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time) pytest.skip("Skipping " + step + " because the input file does not exist.") if run_calwebb_spec2: hdul = core_utils.read_hdrfits(step_output_file, info=False, show_hdr=False) return hdul, step_output_file, run_pytests else: if run_pipe_step: # Create the logfile for PTT, but erase the previous one if it exists PTTcalspec2_log = os.path.join( working_directory, 'PTT_calspec2_' + detector + '_' + step + '.log') if os.path.isfile(PTTcalspec2_log): os.remove(PTTcalspec2_log) print( "Information outputed to screen from PTT will be logged in file: ", PTTcalspec2_log) for handler in logging.root.handlers[:]: logging.root.removeHandler(handler) logging.basicConfig(filename=PTTcalspec2_log, level=logging.INFO) # print pipeline version import jwst pipeline_version = "\n *** Using jwst pipeline version: " + jwst.__version__ + " *** \n" print(pipeline_version) logging.info(pipeline_version) if change_filter_opaque: logging.info(filter_opaque_msg) if os.path.isfile(step_input_file): msg = " *** Step " + step + " set to True" print(msg) logging.info(msg) stp = ResampleSpecStep() # check that previous pipeline steps were run up to this point core_utils.check_completed_steps(step, step_input_file) # get the right configuration files to run the step local_pipe_cfg_path = config.get( "calwebb_spec2_input_file", "local_pipe_cfg_path") # start the timer to compute the step running time start_time = time.time() if local_pipe_cfg_path == "pipe_source_tree_code": result = stp.call(step_input_file) else: result = stp.call(step_input_file, config_file=local_pipe_cfg_path + '/resample_spec.cfg') result.save(step_output_file) # end the timer to compute the step running time end_time = repr(time.time() - start_time) # this is in seconds msg = "Step " + step + " took " + end_time + " seconds to finish" print(msg) logging.info(msg) step_completed = True hdul = core_utils.read_hdrfits(step_output_file, info=False, show_hdr=False) # rename and move the pipeline log file try: calspec2_pilelog = "calspec2_pipeline_" + step + "_" + detector + ".log" pytest_workdir = os.getcwd() logfile = glob(pytest_workdir + "/pipeline.log")[0] os.rename( logfile, os.path.join(working_directory, calspec2_pilelog)) except: IndexError # add the running time for this step core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time) return hdul, step_output_file, run_pytests else: msg = " The input file does not exist. Skipping step." print(msg) logging.info(msg) core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time) pytest.skip("Skipping " + step + " because the input file does not exist.") else: msg = "Skipping running pipeline step " + step print(msg) logging.info(msg) end_time = core_utils.get_stp_run_time_from_screenfile( step, detector, working_directory) if os.path.isfile(step_output_file): hdul = core_utils.read_hdrfits(step_output_file, info=False, show_hdr=False) step_completed = True # add the running time for this step core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time) return hdul, step_output_file, run_pytests else: step_completed = False # add the running time for this step core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time) pytest.skip() else: pytest.skip("Skipping " + step + " because data is either IFU or BOTS.")