def output_hdul(set_inandout_filenames, config): # determine if the pipeline is to be run in full, per steps, or skipped run_calwebb_spec2 = config.get("run_calwebb_spec2_in_full", "run_calwebb_spec2") if run_calwebb_spec2 == "skip": print( '\n * PTT finished processing run_calwebb_spec2 is set to skip. \n' ) pytest.exit( "Skipping pipeline run and tests for spec2, run_calwebb_spec2 is set to skip in PTT_config file." ) elif "T" in run_calwebb_spec2: run_calwebb_spec2 = True else: run_calwebb_spec2 = False # get the general info set_inandout_filenames_info = core_utils.read_info4outputhdul( config, set_inandout_filenames) step, txt_name, step_input_file, step_output_file, outstep_file_suffix = set_inandout_filenames_info run_pipe_step = config.getboolean("run_pipe_steps", step) # determine which tests are to be run extract_2d_completion_tests = config.getboolean( "run_pytest", "_".join((step, "completion", "tests"))) extract_2d_validation_tests = config.getboolean( "run_pytest", "_".join((step, "validation", "tests"))) assign_wcs_validation_tests = config.getboolean( "run_pytest", "_".join((step, "validation", "tests"))) run_pytests = [ extract_2d_completion_tests, extract_2d_validation_tests, assign_wcs_validation_tests ] # get other relevant info from PTT config file compare_assign_wcs_and_extract_2d_with_esa = config.getboolean( "benchmark_intermediary_products", "compare_assign_wcs_and_extract_2d_with_esa") esa_files_path = config.get("benchmark_intermediary_products", "esa_files_path") data_directory = config.get("calwebb_spec2_input_file", "data_directory") truth_file = os.path.join( data_directory, config.get("benchmark_intermediary_products", "truth_file_assign_wcs")) if compare_assign_wcs_and_extract_2d_with_esa: truth_file = esa_files_path print("Will use this 'truth' file to compare result of extract_2d: ") print(truth_file) msa_conf_name = config.get("benchmark_intermediary_products", "msa_conf_name") extract_2d_threshold_diff = int( config.get("additional_arguments", "extract_2d_threshold_diff")) # Check if the mode used is MOS_sim and get the threshold for the assign_wcs test mode_used = config.get("calwebb_spec2_input_file", "mode_used").lower() wcs_threshold_diff = config.get("additional_arguments", "wcs_threshold_diff") save_wcs_plots = config.getboolean("additional_arguments", "save_wcs_plots") # if run_calwebb_spec2 is True calwebb_spec2 will be called, else individual steps will be ran step_completed = False end_time = '0.0' # only do this step if data is NOT IFU output_directory = config.get("calwebb_spec2_input_file", "output_directory") initial_input_file = config.get("calwebb_spec2_input_file", "input_file") initial_input_file = os.path.join(output_directory, initial_input_file) if os.path.isfile(initial_input_file): inhdu = core_utils.read_hdrfits(initial_input_file, info=False, show_hdr=False) detector = fits.getval(initial_input_file, "DETECTOR", 0) else: pytest.skip( "Skipping " + step + " because the initial input file given in PTT_config.cfg does not exist." ) if not core_utils.check_IFU_true(inhdu): if run_calwebb_spec2: hdul = core_utils.read_hdrfits(step_output_file, info=False, show_hdr=False) return hdul, step_output_file, msa_conf_name, truth_file, run_pytests, mode_used, wcs_threshold_diff, \ save_wcs_plots, extract_2d_threshold_diff, compare_assign_wcs_and_extract_2d_with_esa else: if run_pipe_step: # Create the logfile for PTT, but erase the previous one if it exists PTTcalspec2_log = os.path.join( output_directory, 'PTT_calspec2_' + detector + '_' + step + '.log') if os.path.isfile(PTTcalspec2_log): os.remove(PTTcalspec2_log) print( "Information outputed to screen from PTT will be logged in file: ", PTTcalspec2_log) for handler in logging.root.handlers[:]: logging.root.removeHandler(handler) logging.basicConfig(filename=PTTcalspec2_log, level=logging.INFO) # print pipeline version import jwst pipeline_version = "\n *** Using jwst pipeline version: " + jwst.__version__ + " *** \n" print(pipeline_version) logging.info(pipeline_version) if os.path.isfile(step_input_file): msg = " The input file " + step_input_file + " exists... will run step " + step print(msg) logging.info(msg) stp = Extract2dStep() # check that previous pipeline steps were run up to this point core_utils.check_completed_steps(step, step_input_file) # get the right configuration files to run the step local_pipe_cfg_path = config.get( "calwebb_spec2_input_file", "local_pipe_cfg_path") # start the timer to compute the step running time start_time = time.time() if local_pipe_cfg_path == "pipe_source_tree_code": result = stp.call(step_input_file) else: result = stp.call(step_input_file, config_file=local_pipe_cfg_path + '/extract_2d.cfg') result.save(step_output_file) step_completed = True hdul = core_utils.read_hdrfits(step_output_file, info=False, show_hdr=False) # end the timer to compute the step running time end_time = repr(time.time() - start_time) # this is in seconds msg = "Step " + step + " took " + end_time + " seconds to finish" print(msg) logging.info(msg) # rename and move the pipeline log file pipelog = "pipeline_" + detector + ".log" try: calspec2_pilelog = "calspec2_pipeline_" + step + "_" + detector + ".log" pytest_workdir = TESTSDIR logfile = glob(pytest_workdir + "/" + pipelog)[0] os.rename( logfile, os.path.join(output_directory, calspec2_pilelog)) except IndexError: print( "\n* WARNING: Something went wrong. Could not find a ", pipelog, " file \n") # add the running time for this step core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time) return hdul, step_output_file, msa_conf_name, truth_file, run_pytests, mode_used, \ wcs_threshold_diff, save_wcs_plots, extract_2d_threshold_diff, compare_assign_wcs_and_extract_2d_with_esa else: msg = " The input file does not exist. Skipping step." print(msg) logging.info(msg) core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time) pytest.skip("Skiping " + step + " because the input file does not exist.") else: msg = "Skipping running pipeline step " + step print(msg) logging.info(msg) end_time = core_utils.get_stp_run_time_from_screenfile( step, detector, output_directory) if os.path.isfile(step_output_file): hdul = core_utils.read_hdrfits(step_output_file, info=False, show_hdr=False) step_completed = True # add the running time for this step core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time) return hdul, step_output_file, msa_conf_name, truth_file, run_pytests, mode_used, \ wcs_threshold_diff, save_wcs_plots, extract_2d_threshold_diff, compare_assign_wcs_and_extract_2d_with_esa else: step_completed = False # add the running time for this step core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time) pytest.skip("Test skipped because input file " + step_output_file + " does not exist.") else: core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time) pytest.skip("Skipping " + step + " because data is IFU.")
def output_hdul(set_inandout_filenames, config): # determine if the pipeline is to be run in full, per steps, or skipped run_calwebb_spec2 = config.get("run_calwebb_spec2_in_full", "run_calwebb_spec2") if run_calwebb_spec2 == "skip": print( '\n * PTT finished processing run_calwebb_spec2 is set to skip. \n' ) pytest.exit( "Skipping pipeline run and tests for spec2, run_calwebb_spec2 is set to skip in PTT_config file." ) elif "T" in run_calwebb_spec2: run_calwebb_spec2 = True else: run_calwebb_spec2 = False # get the general info set_inandout_filenames_info = core_utils.read_info4outputhdul( config, set_inandout_filenames) step, txt_name, step_input_file, step_output_file, outstep_file_suffix = set_inandout_filenames_info run_pipe_step = config.getboolean("run_pipe_steps", step) # determine which tests are to be run cube_build_completion_tests = config.getboolean( "run_pytest", "_".join((step, "completion", "tests"))) #cube_build_reffile_tests = config.getboolean("run_pytest", "_".join((step, "reffile", "tests"))) #cube_build_validation_tests = config.getboolean("run_pytest", "_".join((step, "validation", "tests"))) run_pytests = [cube_build_completion_tests ] #, cube_build_reffile_tests, cube_build_validation_tests] # Only run step if data is IFU output_directory = config.get("calwebb_spec2_input_file", "output_directory") initial_input_file = config.get("calwebb_spec2_input_file", "input_file") initial_input_file = os.path.join(output_directory, initial_input_file) if os.path.isfile(initial_input_file): inhdu = core_utils.read_hdrfits(initial_input_file, info=False, show_hdr=False) detector = fits.getval(initial_input_file, "DETECTOR", 0) filt = fits.getval(initial_input_file, 'filter') grat = fits.getval(initial_input_file, 'grating') gratfilt = grat + "-" + filt + "_s3d" else: pytest.skip( "Skipping " + step + " because the initial input file given in PTT_config.cfg does not exist." ) end_time = '0.0' if core_utils.check_IFU_true(inhdu): # if run_calwebb_spec2 is True calwebb_spec2 will be called, else individual steps will be ran step_completed = False # check if the filter is to be changed change_filter_opaque = config.getboolean("calwebb_spec2_input_file", "change_filter_opaque") if change_filter_opaque: is_filter_opaque, step_input_filename = change_filter_opaque2science.change_filter_opaque( step_input_file, step=step) if is_filter_opaque: filter_opaque_msg = "With FILTER=OPAQUE, the calwebb_spec2 will run up to the extract_2d step. " \ "Cube build pytest now set to Skip." print(filter_opaque_msg) core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time) pytest.skip("Skipping " + step + " because FILTER=OPAQUE.") if run_calwebb_spec2: hdul = core_utils.read_hdrfits(step_output_file, info=False, show_hdr=False) return hdul, step_output_file, run_pytests else: if run_pipe_step: # Create the logfile for PTT, but erase the previous one if it exists PTTcalspec2_log = os.path.join( output_directory, 'PTT_calspec2_' + detector + '_' + step + '.log') if os.path.isfile(PTTcalspec2_log): os.remove(PTTcalspec2_log) print( "Information outputed to screen from PTT will be logged in file: ", PTTcalspec2_log) for handler in logging.root.handlers[:]: logging.root.removeHandler(handler) logging.basicConfig(filename=PTTcalspec2_log, level=logging.INFO) # print pipeline version import jwst pipeline_version = "\n *** Using jwst pipeline version: " + jwst.__version__ + " *** \n" print(pipeline_version) logging.info(pipeline_version) if change_filter_opaque: logging.info(filter_opaque_msg) if os.path.isfile(step_input_file): msg = " *** Step " + step + " set to True" print(msg) logging.info(msg) stp = CubeBuildStep() # check that previous pipeline steps were run up to this point core_utils.check_completed_steps(step, step_input_file) # get the right configuration files to run the step local_pipe_cfg_path = config.get( "calwebb_spec2_input_file", "local_pipe_cfg_path") # start the timer to compute the step running time start_time = time.time() if local_pipe_cfg_path == "pipe_source_tree_code": result = stp.call(step_input_file) else: result = stp.call(step_input_file, config_file=local_pipe_cfg_path + '/cube_build.cfg') result.save(step_output_file) # end the timer to compute the step running time end_time = repr(time.time() - start_time) # this is in seconds msg = "Step " + step + " took " + end_time + " seconds to finish" print(msg) logging.info(msg) # determine the specific output of the cube step specific_output_file = glob( step_output_file.replace( 'cube.fits', (gratfilt + '*.fits').lower()))[0] cube_suffix = specific_output_file.split( 'cube_build_')[-1].replace('.fits', '') # record info step_completed = True hdul = core_utils.read_hdrfits(specific_output_file, info=False, show_hdr=False) # rename and move the pipeline log file pipelog = "pipeline_" + detector + ".log" try: calspec2_pilelog = "calspec2_pipeline_" + step + "_" + detector + ".log" pytest_workdir = TESTSDIR logfile = glob(pytest_workdir + "/" + pipelog)[0] os.rename( logfile, os.path.join(output_directory, calspec2_pilelog)) except IndexError: print( "\n* WARNING: Something went wrong. Could not find a ", pipelog, " file \n") # add the running time for this step core_utils.add_completed_steps(txt_name, step, "_" + cube_suffix, step_completed, end_time) return hdul, step_output_file, run_pytests else: msg = " The input file does not exist. Skipping step." print(msg) logging.info(msg) core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time) pytest.skip("Skipping " + step + " because the input file does not exist.") else: msg = "Skipping running pipeline step " + step print(msg) logging.info(msg) end_time = core_utils.get_stp_run_time_from_screenfile( step, detector, output_directory) # record info # specific cube step suffix cube_suffix = "_s3d" if os.path.isfile(step_output_file): hdul = core_utils.read_hdrfits(step_output_file, info=False, show_hdr=False) step_completed = True # add the running time for this step core_utils.add_completed_steps(txt_name, step, cube_suffix, step_completed, end_time) return hdul, step_output_file, run_pytests else: step_completed = False # add the running time for this step core_utils.add_completed_steps(txt_name, step, cube_suffix, step_completed, end_time) pytest.skip("Test skipped because input file " + step_output_file + " does not exist.") else: pytest.skip("Skipping " + step + " because data is not IFU.")
def output_hdul(set_inandout_filenames, config): # determine if the pipeline is to be run in full, per steps, or skipped run_calwebb_spec2 = config.get("run_calwebb_spec2_in_full", "run_calwebb_spec2") if run_calwebb_spec2 == "skip": print('\n * PTT finished processing run_calwebb_spec2 is set to skip. \n') pytest.exit("Skipping pipeline run and tests for spec2, run_calwebb_spec2 is set to skip in PTT_config file.") elif "T" in run_calwebb_spec2: run_calwebb_spec2 = True else: run_calwebb_spec2 = False # get the general info set_inandout_filenames_info = core_utils.read_info4outputhdul(config, set_inandout_filenames) step, txt_name, step_input_file, step_output_file, outstep_file_suffix = set_inandout_filenames_info # determine which steps are to be run, if not run in full run_pipe_step = config.getboolean("run_pipe_steps", step) # determine which tests are to be run bkg_subtract_completion_tests = config.getboolean("run_pytest", "_".join((step, "completion", "tests"))) bkg_subtract_numerical_tests = config.getboolean("run_pytest", "_".join((step, "numerical", "tests"))) #bkg_subtract_validation_tests = config.getboolean("run_pytest", "_".join((step, "validation", "tests"))) run_pytests = [bkg_subtract_completion_tests, bkg_subtract_numerical_tests]#, bkg_subtract_validation_tests] # if run_calwebb_spec2 is True calwebb_spec2 will be called, else individual steps will be ran step_completed = False end_time = '0.0' # skip if BOTS data, else perform step output_directory = config.get("calwebb_spec2_input_file", "output_directory") initial_input_file = config.get("calwebb_spec2_input_file", "input_file") initial_input_file = os.path.join(output_directory, initial_input_file) detector = fits.getval(initial_input_file, "DETECTOR", 0) calspec2_pilelog = "calspec2_pipeline_" + step + "_" + detector + ".log" pytest_workdir = TESTSDIR if os.path.isfile(initial_input_file): inhdu = core_utils.read_hdrfits(initial_input_file, info=False, show_hdr=False) else: pytest.skip("Skipping "+step+" because the initial input file given in PTT_config.cfg does not exist.") if not core_utils.check_BOTS_true(inhdu): if run_calwebb_spec2: if os.path.isfile(step_output_file): hdul = core_utils.read_hdrfits(step_output_file, info=False, show_hdr=False) else: pytest.skip("Skipping "+step+" because the output file does not exist.") return hdul, step_output_file, step_input_file, run_pytests else: if run_pipe_step: # Create the logfile for PTT, but erase the previous one if it exists PTTcalspec2_log = os.path.join(output_directory, 'PTT_calspec2_'+detector+'_'+step+'.log') if os.path.isfile(PTTcalspec2_log): os.remove(PTTcalspec2_log) print("Information outputed to screen from PTT will be logged in file: ", PTTcalspec2_log) for handler in logging.root.handlers[:]: logging.root.removeHandler(handler) logging.basicConfig(filename=PTTcalspec2_log, level=logging.INFO) # print pipeline version import jwst pipeline_version = "\n *** Using jwst pipeline version: "+jwst.__version__+" *** \n" print(pipeline_version) logging.info(pipeline_version) if os.path.isfile(step_input_file): msg = " The input file "+step_input_file+" exists... will run step "+step print(msg) logging.info(msg) bkg_list = core_utils.getlist("additional_arguments", "bkg_list") existing_bgfiles = 0 for bg_file in bkg_list: if os.path.isfile(bg_file): existing_bgfiles += 1 if existing_bgfiles == 0: msg = " Need at least one background file to continue. Step will be skipped." print(msg) logging.info(msg) core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time) pytest.skip("Skipping "+step+" because files listed on bkg_list in the configuration " "file do not exist.") else: # continue since the file(s) exist msg = "*** Step "+step+" set to True" print(msg) logging.info(msg) stp = BackgroundStep() # check that previous pipeline steps were run up to this point core_utils.check_completed_steps(step, step_input_file) # get the right configuration files to run the step local_pipe_cfg_path = config.get("calwebb_spec2_input_file", "local_pipe_cfg_path") # start the timer to compute the step running time start_time = time.time() print("running pipeline...") if local_pipe_cfg_path == "pipe_source_tree_code": result = stp.call(step_input_file, bkg_list) else: result = stp.call(step_input_file, bkg_list, config_file=local_pipe_cfg_path+'/background.cfg') if result is not None: result.save(step_output_file) # end the timer to compute the step running time end_time = repr(time.time() - start_time) # this is in seconds msg = "Step "+step+" took "+end_time+" seconds to finish" print(msg) logging.info(msg) hdul = core_utils.read_hdrfits(step_output_file, info=False, show_hdr=False) step_completed = True # rename and move the pipeline log file pipelog = "pipeline_" + detector + ".log" try: calspec2_pilelog = "calspec2_pipeline_" + step + "_" + detector + ".log" pytest_workdir = TESTSDIR logfile = glob(pytest_workdir + "/" + pipelog)[0] os.rename(logfile, os.path.join(output_directory, calspec2_pilelog)) except IndexError: print("\n* WARNING: Something went wrong. Could not find a ", pipelog, " file \n") # add the running time for this step core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time) return hdul, step_output_file, step_input_file, run_pytests else: print(" The input file does not exist. Skipping step.") core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time) pytest.skip("Skipping "+step+" because the input file does not exist.") else: msg = "Skipping running pipeline step "+step print(msg) logging.info(msg) end_time = core_utils.get_stp_run_time_from_screenfile(step, detector, output_directory) if os.path.isfile(step_output_file): hdul = core_utils.read_hdrfits(step_output_file, info=False, show_hdr=False) step_completed = True # add the running time for this step core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time) return hdul, step_output_file, step_input_file, run_pytests else: # add the running time for this step core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time) pytest.skip("Test skipped because input file "+step_output_file+" does not exist.") else: msg = "Skipping "+step+" because data is BOTS." print(msg) logging.info(msg) core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time) pytest.skip(msg)
def output_hdul(set_inandout_filenames, config): # determine if the pipeline is to be run in full, per steps, or skipped run_calwebb_spec2 = config.get("run_calwebb_spec2_in_full", "run_calwebb_spec2") if run_calwebb_spec2 == "skip": print( '\n * PTT finished processing run_calwebb_spec2 is set to skip. \n' ) pytest.exit( "Skipping pipeline run and tests for spec2, run_calwebb_spec2 is set to skip in PTT_config file." ) elif "T" in run_calwebb_spec2: run_calwebb_spec2 = True else: run_calwebb_spec2 = False # get the general info set_inandout_filenames_info = core_utils.read_info4outputhdul( config, set_inandout_filenames) step, txt_name, step_input_file, step_output_file, outstep_file_suffix = set_inandout_filenames_info msa_shutter_conf = config.get("benchmark_intermediary_products", "msa_conf_name") msametfl = os.path.basename(msa_shutter_conf) dflat_path = config.get("benchmark_intermediary_products", "dflat_path") sflat_path = config.get("benchmark_intermediary_products", "sflat_path") fflat_path = config.get("benchmark_intermediary_products", "fflat_path") flattest_threshold_diff = config.get("additional_arguments", "flattest_threshold_diff") save_flattest_plot = config.getboolean("additional_arguments", "save_flattest_plot") write_flattest_files = config.getboolean("additional_arguments", "write_flattest_files") flattest_paths = [ step_output_file, msa_shutter_conf, dflat_path, sflat_path, fflat_path ] flattest_switches = [ flattest_threshold_diff, save_flattest_plot, write_flattest_files ] run_pipe_step = config.getboolean("run_pipe_steps", step) # determine which tests are to be run flat_field_completion_tests = config.getboolean( "run_pytest", "_".join((step, "completion", "tests"))) flat_field_reffile_tests = config.getboolean( "run_pytest", "_".join((step, "reffile", "tests"))) flat_field_validation_tests = config.getboolean( "run_pytest", "_".join((step, "validation", "tests"))) run_pytests = [ flat_field_completion_tests, flat_field_reffile_tests, flat_field_validation_tests ] # if run_calwebb_spec2 is True calwebb_spec2 will be called, else individual steps will be ran step_completed = False end_time = '0.0' # check if the filter is to be changed change_filter_opaque = config.getboolean("calwebb_spec2_input_file", "change_filter_opaque") if change_filter_opaque: is_filter_opaque, step_input_filename = change_filter_opaque2science.change_filter_opaque( step_input_file, step=step) if is_filter_opaque: filter_opaque_msg = "With FILTER=OPAQUE, the calwebb_spec2 will run up to the extract_2d step. Flat " \ "Field pytest now set to Skip." print(filter_opaque_msg) core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time) pytest.skip("Skipping " + step + " because FILTER=OPAQUE.") # get the MSA shutter configuration file full path only for MOS data output_directory = config.get("calwebb_spec2_input_file", "output_directory") initial_input_file = config.get("calwebb_spec2_input_file", "input_file") initial_input_file = os.path.join(output_directory, initial_input_file) if os.path.isfile(initial_input_file): inhdu = core_utils.read_hdrfits(initial_input_file, info=False, show_hdr=False) detector = fits.getval(initial_input_file, "DETECTOR", 0) else: pytest.skip( "Skipping " + step + " because the initial input file given in PTT_config.cfg does not exist." ) if run_calwebb_spec2: hdul = core_utils.read_hdrfits(step_output_file, info=False, show_hdr=False) flattest_paths = [ step_output_file, msa_shutter_conf, dflat_path, sflat_path, fflat_path ] return hdul, step_output_file, flattest_paths, flattest_switches, run_pytests else: if run_pipe_step: # Create the logfile for PTT, but erase the previous one if it exists PTTcalspec2_log = os.path.join( output_directory, 'PTT_calspec2_' + detector + '_' + step + '.log') if os.path.isfile(PTTcalspec2_log): os.remove(PTTcalspec2_log) print( "Information outputed to screen from PTT will be logged in file: ", PTTcalspec2_log) for handler in logging.root.handlers[:]: logging.root.removeHandler(handler) logging.basicConfig(filename=PTTcalspec2_log, level=logging.INFO) # print pipeline version import jwst pipeline_version = "\n *** Using jwst pipeline version: " + jwst.__version__ + " *** \n" print(pipeline_version) logging.info(pipeline_version) if change_filter_opaque: logging.info(filter_opaque_msg) if os.path.isfile(step_input_file): msg = " *** Step " + step + " set to True" print(msg) logging.info(msg) stp = FlatFieldStep() # check that previous pipeline steps were run up to this point core_utils.check_completed_steps(step, step_input_file) if core_utils.check_MOS_true(inhdu): # copy the MSA shutter configuration file into the pytest directory subprocess.run(["cp", msa_shutter_conf, "."]) # start the timer to compute the step running time ontheflyflat = step_output_file.replace( "flat_field.fits", "interpolatedflat.fits") msg1 = "Step product will be saved as: " + step_output_file msg2 = "on-the-fly flat will be saved as: " + ontheflyflat # get the right configuration files to run the step print(msg1) print(msg2) logging.info(msg1) logging.info(msg2) local_pipe_cfg_path = config.get("calwebb_spec2_input_file", "local_pipe_cfg_path") # start the timer to compute the step running time start_time = time.time() if local_pipe_cfg_path == "pipe_source_tree_code": stp.call(step_input_file, output_file=step_output_file, save_interpolated_flat=True) else: stp.call(step_input_file, output_file=step_output_file, save_interpolated_flat=True, config_file=local_pipe_cfg_path + '/flat_field.cfg') # end the timer to compute the step running time end_time = repr(time.time() - start_time) # this is in seconds msg = "Step " + step + " took " + end_time + " seconds to finish" print(msg) logging.info(msg) # move the on-the-fly flat to the working directory subprocess.run( ["mv", os.path.basename(ontheflyflat), ontheflyflat]) # raname and move the flat_field output subprocess.run([ "mv", os.path.basename(step_output_file).replace( "_flat_field.fits", "_flatfieldstep.fits"), step_output_file ]) if core_utils.check_MOS_true(inhdu): if TESTSDIR == os.path.dirname(msametfl): # remove the copy of the MSA shutter configuration file print("Removing MSA config file from: ", TESTSDIR) subprocess.run(["rm", msametfl]) step_completed = True hdul = core_utils.read_hdrfits(step_output_file, info=False, show_hdr=False) # rename and move the pipeline log file pipelog = "pipeline_" + detector + ".log" try: calspec2_pilelog = "calspec2_pipeline_" + step + "_" + detector + ".log" pytest_workdir = TESTSDIR logfile = glob(pytest_workdir + "/" + pipelog)[0] os.rename(logfile, os.path.join(output_directory, calspec2_pilelog)) except IndexError: print( "\n* WARNING: Something went wrong. Could not find a ", pipelog, " file \n") # add the running time for this step core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time) return hdul, step_output_file, flattest_paths, flattest_switches, run_pytests else: msg = " The input file does not exist. Skipping step." print(msg) logging.info(msg) core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time) pytest.skip("Skipping " + step + " because the input file does not exist.") else: msg = "Skipping running pipeline step " + step print(msg) logging.info(msg) end_time = core_utils.get_stp_run_time_from_screenfile( step, detector, output_directory) if os.path.isfile(step_output_file): hdul = core_utils.read_hdrfits(step_output_file, info=False, show_hdr=False) step_completed = True # add the running time for this step core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time) return hdul, step_output_file, flattest_paths, flattest_switches, run_pytests else: step_completed = False # add the running time for this step core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time) pytest.skip("Test skipped because input file " + step_output_file + " does not exist.")
def output_hdul(set_inandout_filenames, config): # determine if the pipeline is to be run in full, per steps, or skipped run_calwebb_spec2 = config.get("run_calwebb_spec2_in_full", "run_calwebb_spec2") if run_calwebb_spec2 == "skip": print( '\n * PTT finished processing run_calwebb_spec2 is set to skip. \n' ) pytest.exit( "Skipping pipeline run and tests for spec2, run_calwebb_spec2 is set to skip in PTT_config file." ) elif "T" in run_calwebb_spec2: run_calwebb_spec2 = True else: run_calwebb_spec2 = False # get the general info set_inandout_filenames_info = core_utils.read_info4outputhdul( config, set_inandout_filenames) step, txt_name, step_input_file, step_output_file, outstep_file_suffix = set_inandout_filenames_info run_pipe_step = config.getboolean("run_pipe_steps", step) # determine which tests are to be run extract_1d_completion_tests = config.getboolean( "run_pytest", "_".join((step, "completion", "tests"))) extract_1d_reffile_tests = config.getboolean( "run_pytest", "_".join((step, "reffile", "tests"))) # extract_1d_validation_tests = config.getboolean("run_pytest", "_".join((step, "validation", "tests"))) run_pytests = [extract_1d_completion_tests, extract_1d_reffile_tests] #, extract_1d_validation_tests] # Make sure at this point the MSA shutter configuration file is removed from the calwebb_spec2_pytests directory msa_shutter_conf = config.get("benchmark_intermediary_products", "msa_conf_name") msametfl = os.path.basename(msa_shutter_conf) # remove the copy of the MSA shutter configuration file inhdu = core_utils.read_hdrfits(step_input_file, info=False, show_hdr=False) if core_utils.check_MOS_true(inhdu): if TESTSDIR == os.path.dirname(msametfl): print("Removing MSA config file from: ", TESTSDIR) os.remove(msametfl) # check if processing an image, then set proper variables imaging_mode = False mode_used = config.get("calwebb_spec2_input_file", "mode_used").lower() if mode_used in ('image', 'confirm', 'taconfirm', 'wata', 'msata', 'bota', 'focus', 'mimf'): run_calwebb_spec2 = True imaging_mode = True # if run_calwebb_spec2 is True calwebb_spec2 will be called, else individual steps will be ran step_completed = False end_time = '0.0' # Get the detector used output_directory = config.get("calwebb_spec2_input_file", "output_directory") initial_input_file = config.get("calwebb_spec2_input_file", "input_file") initial_input_file = os.path.join(output_directory, initial_input_file) if os.path.isfile(initial_input_file): detector = fits.getval(initial_input_file, "DETECTOR", 0) else: pytest.skip( "Skipping " + step + " because the initial input file given in PTT_config.cfg does not exist." ) # set the name of the pipeline log to be read for times and renamed at the end for book-keeping pipelog = "pipeline_" + detector + ".log" # check if the filter is to be changed change_filter_opaque = config.getboolean("calwebb_spec2_input_file", "change_filter_opaque") if change_filter_opaque: is_filter_opaque, step_input_filename = change_filter_opaque2science.change_filter_opaque( step_input_file, step=step) if is_filter_opaque: filter_opaque_msg = "With FILTER=OPAQUE, the calwebb_spec2 will run up to the extract_2d step. " \ "Extract_1d pytest now set to Skip." print(filter_opaque_msg) core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time) #core_utils.convert_html2pdf() # convert the html report into a pdf file # move the final reporting files to the working directory core_utils.move_txt_files_2workdir(config, detector) pytest.skip("Skipping " + step + " because FILTER=OPAQUE.") if run_calwebb_spec2: # read the output header hdul = core_utils.read_hdrfits(step_output_file, info=False, show_hdr=False) # end the timer to compute the step running time of PTT PTT_end_time = time.time() core_utils.start_end_PTT_time(txt_name, start_time=None, end_time=PTT_end_time) # move the final reporting files to the working directory core_utils.move_txt_files_2workdir(config, detector) return hdul, step_output_file, run_pytests else: if run_pipe_step: # Create the logfile for PTT, but erase the previous one if it exists PTTcalspec2_log = os.path.join( output_directory, 'PTT_calspec2_' + detector + '_' + step + '.log') if imaging_mode: PTTcalspec2_log = PTTcalspec2_log.replace( 'calspec2', 'calimage2') if os.path.isfile(PTTcalspec2_log): os.remove(PTTcalspec2_log) print( "Information outputed to screen from PTT will be logged in file: ", PTTcalspec2_log) for handler in logging.root.handlers[:]: logging.root.removeHandler(handler) logging.basicConfig(filename=PTTcalspec2_log, level=logging.INFO) # print pipeline version import jwst pipeline_version = "\n *** Using jwst pipeline version: " + jwst.__version__ + " *** \n" print(pipeline_version) logging.info(pipeline_version) if change_filter_opaque: logging.info(filter_opaque_msg) if os.path.isfile(step_input_file): msg = " *** Step " + step + " set to True" print(msg) logging.info(msg) stp = Extract1dStep() # check that previous pipeline steps were run up to this point core_utils.check_completed_steps(step, step_input_file) # get the right configuration files to run the step local_pipe_cfg_path = config.get("calwebb_spec2_input_file", "local_pipe_cfg_path") # start the timer to compute the step running time start_time = time.time() if local_pipe_cfg_path == "pipe_source_tree_code": result = stp.call(step_input_file) else: result = stp.call(step_input_file, config_file=local_pipe_cfg_path + '/extract_1d.cfg') result.save(step_output_file) # end the timer to compute the step running time end_time = repr(time.time() - start_time) # this is in seconds msg = "Step " + step + " took " + end_time + " seconds to finish" print(msg) logging.info(msg) # rename and move the pipeline log file calspec2_pilelog = "calspec2_pipeline_" + step + "_" + detector + ".log" if imaging_mode: calspec2_pilelog = calspec2_pilelog.replace( 'calspec2', 'calimage2') os.rename(pipelog, os.path.join(output_directory, calspec2_pilelog)) else: msg = " The input file does not exist. Skipping step." print(msg) logging.info(msg) core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time) #core_utils.convert_html2pdf() # convert the html report into a pdf file # end the timer to compute the step running time of PTT PTT_end_time = time.time() core_utils.start_end_PTT_time(txt_name, start_time=None, end_time=PTT_end_time) # move the final reporting files to the working directory core_utils.move_txt_files_2workdir(config, detector) # skip the test if input file does not exist pytest.skip("Skipping " + step + " because the input file does not exist.") else: msg = "Skipping running pipeline step " + step print(msg) logging.info(msg) # get the running time for this step end_time = core_utils.get_stp_run_time_from_screenfile( step, detector, output_directory) # add the running time for this step if os.path.isfile(step_output_file): hdul = core_utils.read_hdrfits(step_output_file, info=False, show_hdr=False) step_completed = True core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time) else: step_completed = False core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time) # move the final reporting files to the working directory core_utils.move_txt_files_2workdir(config, detector) pytest.skip("Test skipped because input file " + step_output_file + " does not exist.") # get the total running time and print it in the file total_time = repr(core_utils.get_time_to_run_pipeline(txt_name)) total_time_min = repr(round(float(total_time) / 60.0, 2)) msg = "\n\n **** The total time for the pipeline to run was " + total_time + " seconds." print(msg) logging.info(msg) line2write = "{:<20} {:<20} {:<20} {:<20}".format( '', '', 'total_time ', total_time + ' =' + total_time_min + 'min') print(line2write) logging.info(line2write) with open(txt_name, "a") as tf: tf.write(line2write + "\n") # convert the html report into a pdf file #core_utils.convert_html2pdf() # end the timer to compute the step running time of PTT PTT_end_time = time.time() core_utils.start_end_PTT_time(txt_name, start_time=None, end_time=PTT_end_time) # move the final reporting text files to the working directory core_utils.move_txt_files_2workdir(config, detector) # rename and move the PTT log file try: calspec2_pipelog = "calspec2_pipeline_" + step + "_" + detector + ".log" if imaging_mode: calspec2_pipelog = calspec2_pipelog.replace( 'calspec2', 'calimage2') pytest_workdir = TESTSDIR logfile = glob(pytest_workdir + "/" + pipelog)[0] os.rename(logfile, os.path.join(output_directory, calspec2_pipelog)) except IndexError: print("\n* WARNING: Something went wrong. Could not find a ", pipelog, " file \n") return hdul, step_output_file, run_pytests
def output_hdul(set_inandout_filenames, config): # determine if the pipeline is to be run in full, per steps, or skipped run_calwebb_spec3 = config.get("calwebb_spec3", "run_calwebb_spec3") print("run_calwebb_spec3 = ", run_calwebb_spec3) if run_calwebb_spec3 == "skip": print( '\n * PTT finished processing run_calwebb_spec3 is set to skip. \n' ) pytest.exit( "Finished processing file, run_calwebb_spec3 is set to skip in configuration file." ) else: run_calwebb_spec3 = bool(run_calwebb_spec3) # get the general info step, step_input_filename, output_file, in_file_suffix, outstep_file_suffix, True_steps_suffix_map = set_inandout_filenames output_directory = config.get("calwebb_spec2_input_file", "output_directory") txt_name = os.path.join(output_directory, True_steps_suffix_map) step_input_file = os.path.join(output_directory, step_input_filename) step_output_file = os.path.join(output_directory, output_file) mode_used = config.get("calwebb_spec2_input_file", "mode_used").lower() # start the timer to compute the step running time of NPTT nptt_start_time = time.time() # determine which steps are to be run, if not run in full run_pipe_step = config.getboolean("run_pipe_steps", step) # determine which tests are to be run master_background_completion_tests = config.getboolean( "run_pytest", "_".join((step, "completion", "tests"))) master_background_reffile_tests = config.getboolean( "run_pytest", "_".join((step, "reffile", "tests"))) master_background_validation_tests = config.getboolean( "run_pytest", "_".join((step, "validation", "tests"))) run_pytests = [ master_background_completion_tests, master_background_reffile_tests, master_background_validation_tests ] # Get the detector used detector = fits.getval(step_input_file, "DETECTOR", 0) # get main header from input file inhdu = core_utils.read_hdrfits(step_input_file, info=False, show_hdr=False) # if run_calwebb_spec3 is True, calwebb_spec3 will be called, else individual steps will be ran step_completed = False end_time = '0.0' # get the shutter configuration file for MOS data only msa_shutter_conf = "No shutter configuration file will be used." if core_utils.check_MOS_true(inhdu): msa_shutter_conf = config.get("esa_intermediary_products", "msa_conf_name") # check if the configuration shutter file name is in the header of the fits file and if not add it msametfl = fits.getval(step_input_file, "MSAMETFL", 0) if os.path.basename(msa_shutter_conf) != msametfl: msametfl = os.path.basename(msa_shutter_conf) fits.setval(step_input_file, "MSAMETFL", 0, value=msametfl) # copy the MSA shutter configuration file to this directory if not the working directory if os.getcwd() != os.path.dirname(msa_shutter_conf): # copy the MSA shutter configuration file into the pytest directory print("Removing MSA config file from: ", os.getcwd()) subprocess.run(["cp", msa_shutter_conf, "."]) # check if processing an image, then set proper variables if mode_used in ('image', 'confirm', 'taconfirm', 'wata', 'msata', 'bota', 'focus', 'mimf'): run_calwebb_spec3 = True imaging_mode = True print('\n * Image processing stops after calwebb_spec2. ') # end script for imaging case if imaging_mode: print('\n * NPTT finished processing imaging mode. \n') pytest.exit( "Imaging does not get processed through calwebb_spec3.") # get the name of the configuration file and run the pipeline calwebb_spec3_cfg = config.get("calwebb_spec3", "calwebb_spec3_cfg") # copy the configuration file to create the pipeline log if not os.path.isfile(os.path.join(output_directory, "stpipe-log.cfg")): stpipelogcfg = calwebb_spec3_cfg.replace("calwebb_spec3.cfg", "stpipe-log.cfg") subprocess.run(["cp", stpipelogcfg, os.getcwd()]) # run the pipeline if run_calwebb_spec3: # Create the logfile for NPTT, but remove the previous log file npttcalspec3_log = os.path.join(output_directory, 'NPTT_calspec3_' + detector + '.log') if os.path.isfile(npttcalspec3_log): os.remove(npttcalspec3_log) print( "Spec3 screen information output from NPTT will be logged in file: ", npttcalspec3_log) for handler in logging.root.handlers[:]: logging.root.removeHandler(handler) logging.basicConfig(filename=npttcalspec3_log, level=logging.INFO) logging.info(pipeline_version) run_calwebb_spec3_msg = " *** Will run pipeline in full ... " print(run_calwebb_spec3_msg) logging.info(run_calwebb_spec3_msg) # create the map txt_name = "spec3_full_run_map_" + detector + ".txt" if os.path.isfile(txt_name): os.remove(txt_name) master_background_utils.create_completed_steps_txtfile( txt_name, step_input_file) # start the timer to compute the step running time of NPTT core_utils.start_end_PTT_time(txt_name, start_time=nptt_start_time, end_time=None) if mode_used == "bots": calwebb_spec3_cfg = calwebb_spec3_cfg.replace( "calwebb_spec3.cfg", "calwebb_tso-spec3.cfg") print( '\nUsing the following configuration file to run TSO pipeline:' ) else: print( '\nUsing the following configuration file to run spectroscopic pipeline:' ) print(calwebb_spec3_cfg, '\n') # start the timer to compute the step running time start_time = time.time() # run the pipeline print('Running pipeline... \n') Spec3Pipeline.call(step_input_file, config_file=calwebb_spec3_cfg) # end the timer to compute calwebb_spec3 running time end_time = repr(time.time() - start_time) # this is in seconds calspec3_time = " * Pipeline took " + end_time + " seconds to finish.\n" print(calspec3_time) logging.info(calspec3_time) # add the detector string to the name of the files and move them to the working directory core_utils.add_detector2filename(output_directory, step_input_file) # state name of the final spec3 _cal file if "spec2" in step_input_file: final_output_name = step_input_file.replace("spec2", "spec3") else: final_output_name = "final_output_spec3_" + detector + "_cal.fits" final_output_name_msg = "\nThe final pipeline product was saved in: " + final_output_name print(final_output_name_msg) logging.info(final_output_name_msg) # read the assign wcs fits file hdul = core_utils.read_hdrfits(step_output_file, info=False, show_hdr=False) # scihdul = core_utils.read_hdrfits(step_output_file, info=False, show_hdr=False, ext=1) if core_utils.check_MOS_true(inhdu): if os.getcwd() != os.path.dirname(msa_shutter_conf): # remove the copy of the MSA shutter configuration file print("Removing MSA config file from: ", os.getcwd()) subprocess.run(["rm", msametfl]) # rename and move the pipeline log file calspec3_pipelog = "calspec3_pipeline_" + detector + ".log" try: path_where_pipeline_was_run = os.getcwd() logfile = glob(path_where_pipeline_was_run + "/pipeline.log")[0] print(logfile) os.rename(logfile, os.path.join(output_directory, calspec3_pipelog)) except IndexError: print( "\nWARNING: Something went wrong. Could not find a pipeline.log file \n" ) # make sure we are able to find calspec3_pipelog either in the calwebb_spec3 directory or in the working dir if not os.path.isfile(calspec3_pipelog): calspec3_pipelog = os.path.join(output_directory, calspec3_pipelog) # add the running time for all steps step_running_times = core_utils.calculate_step_run_time( calspec3_pipelog) end_time_list = [] for stp in core_utils.step_string_dict: if stp in step_running_times: step_completed = True step_time = step_running_times[stp]["run_time"] out_suffix = core_utils.step_string_dict[stp]["suffix"] core_utils.add_completed_steps(txt_name, stp, out_suffix, step_completed, step_time) end_time_list.append(step_time) # print total running time in the text file and move it to the indicated directory string2print = "pipeline_total_time" if float(end_time) <= sum(end_time_list): tot_time = repr(sum(end_time_list)) else: tot_time = end_time master_background_utils.print_time2file(txt_name, tot_time, string2print) nptt_runtimes_msg = "Pipeline and NPTT run times written in file: " + os.path.basename( txt_name) + " in working directory. \n" print(nptt_runtimes_msg) logging.info(nptt_runtimes_msg) # move the final reporting text files to the working directory if os.getcwd() != output_directory: core_utils.move_txt_files_2workdir(config, detector) return hdul, step_output_file, msa_shutter_conf, run_pytests, mode_used else: # create the map but remove a previous one if it exists if os.path.isfile(txt_name): os.remove(txt_name) master_background_utils.create_completed_steps_txtfile( txt_name, step_input_file) # start the timer to compute the step running time of NPTT core_utils.start_end_PTT_time(txt_name, start_time=nptt_start_time, end_time=None) msg = "\n Pipeline and NPTT run times will be written in file: " + os.path.basename( txt_name) + " in working directory. \n" print(msg) logging.info(msg) if run_pipe_step: # Create the logfile for NPTT, but erase the previous one if it exists npttcalspec3_log = os.path.join( output_directory, 'NPTT_calspec3_' + detector + '_' + step + '.log') if os.path.isfile(npttcalspec3_log): os.remove(npttcalspec3_log) print( "Output information on screen from NPTT will be logged in file: ", npttcalspec3_log) for handler in logging.root.handlers[:]: logging.root.removeHandler(handler) logging.basicConfig(filename=npttcalspec3_log, level=logging.INFO) logging.info(pipeline_version) # check that previous pipeline steps were run up to this point core_utils.check_completed_steps(step, step_input_file) if os.path.isfile(step_input_file): msg = " *** Step " + step + " set to True" print(msg) logging.info(msg) stp = MasterBackgroundStep() # get the right configuration files to run the step local_pipe_cfg_path = config.get("calwebb_spec2_input_file", "local_pipe_cfg_path") # start the timer to compute the step running time print("running pipeline...") start_time = time.time() if local_pipe_cfg_path == "pipe_source_tree_code": result = stp.call(step_input_file) else: result = stp.call(step_input_file, config_file=local_pipe_cfg_path + '/master_background.cfg') result.save(step_output_file) # end the timer to compute the step running time end_time = repr(time.time() - start_time) # this is in seconds msg = "Step " + step + " took " + end_time + " seconds to finish" print(msg) logging.info(msg) if core_utils.check_MOS_true(inhdu): # remove the copy of the MSA shutter configuration file if os.getcwd() != os.path.dirname(msa_shutter_conf): print("Removing MSA config file from: ", os.getcwd()) subprocess.run(["rm", msametfl]) # rename and move the pipeline log file pipelog = "pipeline_" + detector + ".log" try: calspec3_pipelog = "calspec3_pipeline_" + step + "_" + detector + ".log" pytest_workdir = TESTSDIR logfile = glob(pytest_workdir + "/" + pipelog)[0] os.rename(logfile, os.path.join(output_directory, calspec3_pipelog)) except IndexError: print( "\n* WARNING: Something went wrong. Could not find a ", pipelog, " file \n") else: msg = "Skipping step. Input file " + step_input_file + " does not exit." print(msg) logging.info(msg) core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time) pytest.skip("Skipping " + step + " because the input file does not exist.") else: print("Skipping running pipeline step ", step) # add the running time for this step end_time = core_utils.get_stp_run_time_from_screenfile( step, detector, output_directory) if os.path.isfile(step_output_file): hdul = core_utils.read_hdrfits(step_output_file, info=False, show_hdr=False) step_completed = True # add the running time for this step core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time) return hdul, step_output_file, msa_shutter_conf, run_pytests, mode_used else: step_completed = False # add the running time for this step core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time) pytest.skip("Test skipped because input file " + step_output_file + " does not exist.")
def output_hdul(set_inandout_filenames, config): # determine if the pipeline is to be run in full, per steps, or skipped run_calwebb_spec3 = config.get("calwebb_spec3", "run_calwebb_spec3") print("run_calwebb_spec3 = ", run_calwebb_spec3) if run_calwebb_spec3 == "skip": print('\n * PTT finished processing run_calwebb_spec3 is set to skip. \n') pytest.exit("Finished processing file, run_calwebb_spec3 is set to skip in configuration file.") else: run_calwebb_spec3 = bool(run_calwebb_spec3) # get the general info step, step_input_filename, output_file, in_file_suffix, outstep_file_suffix, True_steps_suffix_map = set_inandout_filenames output_directory = config.get("calwebb_spec2_input_file", "output_directory") txt_name = os.path.join(output_directory, True_steps_suffix_map) step_input_file = os.path.join(output_directory, step_input_filename) step_output_file = os.path.join(output_directory, output_file) mode_used = config.get("calwebb_spec2_input_file", "mode_used").lower() # start the timer to compute the step running time of NPTT nptt_start_time = time.time() # determine if steps is to be run run_pipe_step = config.getboolean("run_pipe_steps", step) # determine which tests are to be run #completion_tests = config.getboolean("run_pytest", "_".join((step, "completion", "tests"))) #reffile_tests = config.getboolean("run_pytest", "_".join((step, "reffile", "tests"))) #validation_tests = config.getboolean("run_pytest", "_".join((step, "validation", "tests"))) #run_pytests = [completion_tests, reffile_tests, validation_tests] # TODO - booleans for different tests have not been implemented in the configuration file yet run_pytests = [False] # Get the detector used detector = fits.getval(step_input_file, "DETECTOR", 0) # get main header from input file inhdu = core_utils.read_hdrfits(step_input_file, info=False, show_hdr=False) # if run_calwebb_spec3 is True, calwebb_spec3 will be called, else individual steps will be ran step_completed = False end_time = '0.0' # only do this step if data is NOT IFU if mode_used == 'ifu': pytest.skip("Skipping test for step "+step+" because data is IFU.") # run the pipeline if run_calwebb_spec3: hdul = core_utils.read_hdrfits(step_output_file, info=False, show_hdr=False) return hdul, step_output_file, run_pytests, mode_used else: if run_pipe_step: # Create the logfile for NPTT, but erase the previous one if it exists npttcalspec3_log = os.path.join(output_directory, 'NPTT_calspec3_' + detector + '_' + step + '.log') if os.path.isfile(npttcalspec3_log): os.remove(npttcalspec3_log) print("Output information on screen from NPTT will be logged in file: ", npttcalspec3_log) for handler in logging.root.handlers[:]: logging.root.removeHandler(handler) logging.basicConfig(filename=npttcalspec3_log, level=logging.INFO) logging.info(pipeline_version) # check that previous pipeline steps were run up to this point core_utils.check_completed_steps(step, step_input_file) # TODO - need a function to gather all the same source files into a list to feed into this pipeline step if os.path.isfile(step_input_file): msg = " *** Step " + step + " set to True" print(msg) logging.info(msg) print("Not running this individual step at the moment.") """ *** NEED THE COMMANDS TO RUN THIS STEP - requires a list of _cal.fits files stp = MasterBackgroundStep() # get the right configuration files to run the step local_pipe_cfg_path = config.get("calwebb_spec2_input_file", "local_pipe_cfg_path") # start the timer to compute the step running time print("running pipeline...") start_time = time.time() if local_pipe_cfg_path == "pipe_source_tree_code": result = stp.call(step_input_file) else: result = stp.call(step_input_file, config_file=local_pipe_cfg_path + '/master_background.cfg') result.save(step_output_file) # end the timer to compute the step running time end_time = repr(time.time() - start_time) # this is in seconds msg = "Step " + step + " took " + end_time + " seconds to finish" print(msg) logging.info(msg) # rename and move the pipeline log file pipelog = "pipeline_" + detector + ".log" try: calspec3_pipelog = "calspec3_pipeline_" + step + "_" + detector + ".log" pytest_workdir = TESTSDIR logfile = glob(pytest_workdir + "/" + pipelog)[0] os.rename(logfile, os.path.join(output_directory, calspec3_pipelog)) except IndexError: print("\n* WARNING: Something went wrong. Could not find a ", pipelog, " file \n") """ else: msg = "Skipping step. Input file " + step_input_file + " does not exit." print(msg) logging.info(msg) core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time) pytest.skip("Skipping " + step + " because the input file does not exist.") else: print("Skipping running pipeline step ", step) # add the running time for this step end_time = core_utils.get_stp_run_time_from_screenfile(step, detector, output_directory) if os.path.isfile(step_output_file): hdul = core_utils.read_hdrfits(step_output_file, info=False, show_hdr=False) step_completed = True # add the running time for this step core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time) return hdul, step_output_file, run_pytests, mode_used else: step_completed = False # add the running time for this step core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time) pytest.skip("Test skipped because input file "+step_output_file+" does not exist.")
def output_hdul(set_inandout_filenames, config): # determine if the pipeline is to be run in full, per steps, or skipped run_calwebb_spec2 = config.get("run_calwebb_spec2_in_full", "run_calwebb_spec2") if run_calwebb_spec2 == "skip": print( '\n * PTT finished processing run_calwebb_spec2 is set to skip. \n' ) pytest.exit( "Skipping pipeline run and tests for spec2, run_calwebb_spec2 is set to skip in PTT_config file." ) elif "T" in run_calwebb_spec2: run_calwebb_spec2 = True else: run_calwebb_spec2 = False # get the general info set_inandout_filenames_info = core_utils.read_info4outputhdul( config, set_inandout_filenames) step, txt_name, step_input_file, step_output_file, outstep_file_suffix = set_inandout_filenames_info # determine which steps are to be run, if not run in full run_pipe_step = config.getboolean("run_pipe_steps", step) end_time = '0.0' # Only run step if data is MOS or IFU output_directory = config.get("calwebb_spec2_input_file", "output_directory") initial_input_file = config.get("calwebb_spec2_input_file", "input_file") initial_input_file = os.path.join(output_directory, initial_input_file) if os.path.isfile(initial_input_file): inhdu = core_utils.read_hdrfits(initial_input_file, info=False, show_hdr=False) detector = fits.getval(initial_input_file, "DETECTOR", 0) else: pytest.skip( "Skipping " + step + " because the initial input file given in PTT_config.cfg does not exist." ) print("Is data MOS or IFU?", core_utils.check_MOS_true(inhdu), core_utils.check_IFU_true(inhdu)) if core_utils.check_MOS_true(inhdu) or core_utils.check_IFU_true(inhdu): # determine which tests are to be run msa_flagging_completion_tests = config.getboolean( "run_pytest", "_".join((step, "completion", "tests"))) # msa_flagging_reffile_tests = config.getboolean("run_pytest", "_".join((step, "reffile", "tests"))) msa_flagging_validation_tests = config.getboolean( "run_pytest", "_".join((step, "validation", "tests"))) run_pytests = [ msa_flagging_completion_tests, msa_flagging_validation_tests ] # get other info from the configuration file relevant only for this step msa_flagging_operability_ref = config.get( "benchmark_intermediary_products", "msa_flagging_operability_ref") msa_flagging_threshold = config.get("additional_arguments", "msa_flagging_threshold") stellarity = config.get("additional_arguments", "stellarity") save_msa_flagging_figs = config.getboolean("additional_arguments", "save_msa_flagging_plots") if stellarity == "source_type": stellarity = None testing_arguments = [ msa_flagging_operability_ref, msa_flagging_threshold, stellarity, save_msa_flagging_figs ] # if run_calwebb_spec2 is True calwebb_spec2 will be called, else individual steps will be ran step_completed = False if run_calwebb_spec2: hdul = core_utils.read_hdrfits(step_output_file, info=False, show_hdr=False) return hdul, step_output_file, step_input_file, run_pytests, testing_arguments else: if run_pipe_step: if os.path.isfile(step_input_file): msg = " The input file " + step_input_file + " exists... will run step " + step print(msg) logging.info(msg) stp = MSAFlagOpenStep() # check that previous pipeline steps were run up to this point core_utils.check_completed_steps(step, step_input_file) # Create the logfile for PTT, but erase the previous one if it exists PTTcalspec2_log = os.path.join( output_directory, 'PTT_calspec2_' + detector + '_' + step + '.log') if os.path.isfile(PTTcalspec2_log): os.remove(PTTcalspec2_log) print( "Information outputed to screen from PTT will be logged in file: ", PTTcalspec2_log) for handler in logging.root.handlers[:]: logging.root.removeHandler(handler) logging.basicConfig(filename=PTTcalspec2_log, level=logging.INFO) # print pipeline version import jwst pipeline_version = "\n *** Using jwst pipeline version: " + jwst.__version__ + " *** \n" print(pipeline_version) logging.info(pipeline_version) # get the right configuration files to run the step # local_pipe_cfg_path = config.get("calwebb_spec2_input_file", "local_pipe_cfg_path") # start the timer to compute the step running time start_time = time.time() # if local_pipe_cfg_path == "pipe_source_tree_code": result = stp.call(step_input_file) # else: # result = stp.call(step_input_file, config_file=local_pipe_cfg_path+'/NOCONFIGFI.cfg') result.save(step_output_file) # end the timer to compute the step running time end_time = repr(time.time() - start_time) # this is in seconds msg = "Step " + step + " took " + end_time + " seconds to finish" print(msg) logging.info(msg) # rename and move the pipeline log file pipelog = "pipeline_" + detector + ".log" try: calspec2_pilelog = "calspec2_pipeline_" + step + "_" + detector + ".log" pytest_workdir = TESTSDIR logfile = glob(pytest_workdir + "/" + pipelog)[0] os.rename( logfile, os.path.join(output_directory, calspec2_pilelog)) except IndexError: print( "\n* WARNING: Something went wrong. Could not find a ", pipelog, " file \n") # add the running time for this step step_completed = True hdul = core_utils.read_hdrfits(step_output_file, info=False, show_hdr=False) core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time) return hdul, step_output_file, step_input_file, run_pytests, testing_arguments else: msg = " The input file does not exist. Skipping step." print(msg) logging.info(msg) core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time) pytest.skip("Skipping " + step + " because the input file does not exist.") else: msg = "Skipping running pipeline step " + step print(msg) logging.info(msg) end_time = core_utils.get_stp_run_time_from_screenfile( step, detector, output_directory) if os.path.isfile(step_output_file): hdul = core_utils.read_hdrfits(step_output_file, info=False, show_hdr=False) step_completed = True # add the running time for this step core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time) return hdul, step_output_file, step_input_file, run_pytests, testing_arguments else: step_completed = False # add the running time for this step core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time) pytest.skip("Test skipped because input file " + step_output_file + " does not exist.") else: pytest.skip("Skipping " + step + " because data is neither MOS or IFU.")
def output_hdul(set_inandout_filenames, config): # determine if the pipeline is to be run in full, per steps, or skipped run_calwebb_spec2 = config.get("run_calwebb_spec2_in_full", "run_calwebb_spec2") if run_calwebb_spec2 == "skip": print( '\n * PTT finished processing run_calwebb_spec2 is set to skip. \n' ) pytest.exit( "Skipping pipeline run and tests for spec2, run_calwebb_spec2 is set to skip in PTT_config file." ) elif "T" in run_calwebb_spec2: run_calwebb_spec2 = True else: run_calwebb_spec2 = False # get the general info set_inandout_filenames_info = core_utils.read_info4outputhdul( config, set_inandout_filenames) step, txt_name, step_input_file, step_output_file, outstep_file_suffix = set_inandout_filenames_info run_pipe_step = config.getboolean("run_pipe_steps", step) # determine which tests are to be run pathloss_completion_tests = config.getboolean( "run_pytest", "_".join((step, "completion", "tests"))) pathloss_reffile_tests = config.getboolean( "run_pytest", "_".join((step, "reffile", "tests"))) pathloss_validation_tests = config.getboolean( "run_pytest", "_".join((step, "validation", "tests"))) run_pytests = [ pathloss_completion_tests, pathloss_reffile_tests, pathloss_validation_tests ] pathloss_threshold_diff = config.get("additional_arguments", "pathloss_threshold_diff") save_flattest_plot = config.getboolean("additional_arguments", "save_pathloss_plot") write_flattest_files = config.getboolean("additional_arguments", "write_pathloss_files") pathloss_switches = [ pathloss_threshold_diff, save_flattest_plot, write_flattest_files ] # if run_calwebb_spec2 is True calwebb_spec2 will be called, else individual steps will be ran step_completed = False end_time = '0.0' # check if the filter is to be changed change_filter_opaque = config.getboolean("calwebb_spec2_input_file", "change_filter_opaque") if change_filter_opaque: is_filter_opaque, step_input_filename = change_filter_opaque2science.change_filter_opaque( step_input_file, step=step) if is_filter_opaque: filter_opaque_msg = "With FILTER=OPAQUE, the calwebb_spec2 will run up to the extract_2d step. " \ "Pathloss pytest now set to Skip." print(filter_opaque_msg) core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time) pytest.skip("Skipping " + step + " because FILTER=OPAQUE.") # only run this step if data is not BOTS output_directory = config.get("calwebb_spec2_input_file", "output_directory") initial_input_file = config.get("calwebb_spec2_input_file", "input_file") initial_input_file = os.path.join(output_directory, initial_input_file) if os.path.isfile(initial_input_file): inhdu = core_utils.read_hdrfits(initial_input_file, info=False, show_hdr=False) detector = fits.getval(initial_input_file, "DETECTOR", 0) else: pytest.skip( "Skipping " + step + " because the initial input file given in PTT_config.cfg does not exist." ) if not core_utils.check_BOTS_true(inhdu): if run_calwebb_spec2: hdul = core_utils.read_hdrfits(step_output_file, info=False, show_hdr=False) return hdul, step_output_file, step_input_file, run_pytests, pathloss_switches else: if run_pipe_step: # Create the logfile for PTT, but erase the previous one if it exists PTTcalspec2_log = os.path.join( output_directory, 'PTT_calspec2_' + detector + '_' + step + '.log') if os.path.isfile(PTTcalspec2_log): os.remove(PTTcalspec2_log) print( "Information outputed to screen from PTT will be logged in file: ", PTTcalspec2_log) for handler in logging.root.handlers[:]: logging.root.removeHandler(handler) logging.basicConfig(filename=PTTcalspec2_log, level=logging.INFO) # print pipeline version import jwst pipeline_version = "\n *** Using jwst pipeline version: " + jwst.__version__ + " *** \n" print(pipeline_version) logging.info(pipeline_version) if change_filter_opaque: logging.info(filter_opaque_msg) if os.path.isfile(step_input_file): msg = " *** Step " + step + " set to True" print(msg) logging.info(msg) stp = PathLossStep() # check that previous pipeline steps were run up to this point core_utils.check_completed_steps(step, step_input_file) # get the right configuration files to run the step local_pipe_cfg_path = config.get( "calwebb_spec2_input_file", "local_pipe_cfg_path") # start the timer to compute the step running time start_time = time.time() if local_pipe_cfg_path == "pipe_source_tree_code": result = stp.call(step_input_file) else: result = stp.call(step_input_file, config_file=local_pipe_cfg_path + '/pathloss.cfg') result.save(step_output_file) # For the moment, the pipeline is using the wrong reference file for slit 400A1, so the # file needs to be re-processed with the right reference file BUT we are # skipping this slit """ if core_utils.check_FS_true(inhdu): print("\n * WARNING: For the moment, the wrong reference file is being used for " "processing slit 400A1. The file will be re-processed ") # The file can manually be re-run with # $ strun jwst.pathloss.PathLossStep final_output_caldet1_NRS1_srctype.fits # --override_pathloss=jwst-nirspec-a400.plrf.fits pathloss_400a1 = step_input_file.replace("srctype.fits", "pathloss_400A1.fits") reffile_400a1 = "jwst-nirspec-a400.plrf.fits" print("Re-processing slit with new reference file: ", reffile_400a1) pl = PathLossStep() pl.override_pathloss = reffile_400a1 pl_result = pl.run(step_input_file) pl_result.save(pathloss_400a1) #import subprocess #subprocess.run(["mv", step_input_file.replace("srctype", "pathlossstep"), pathloss_400a1]) print("Saved pipeline re-processed file as: ", pathloss_400a1) """ # end the timer to compute calwebb_spec2 running time end_time = repr(time.time() - start_time) # this is in seconds msg = " * calwebb_spec2 took " + end_time + " seconds to finish." print(msg) logging.info(msg) step_completed = True hdul = core_utils.read_hdrfits(step_output_file, info=False, show_hdr=False) # rename and move the pipeline log file pipelog = "pipeline_" + detector + ".log" try: calspec2_pilelog = "calspec2_pipeline_" + step + "_" + detector + ".log" pytest_workdir = TESTSDIR logfile = glob(pytest_workdir + "/" + pipelog)[0] os.rename( logfile, os.path.join(output_directory, calspec2_pilelog)) except IndexError: print( "\n* WARNING: Something went wrong. Could not find a ", pipelog, " file \n") # add the running time for this step core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time) return hdul, step_output_file, step_input_file, run_pytests, pathloss_switches else: msg = " The input file does not exist. Skipping step." print(msg) logging.info(msg) core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time) pytest.skip("Skipping " + step + " because the input file does not exist.") else: msg = "Skipping running pipeline step " + step print(msg) logging.info(msg) end_time = core_utils.get_stp_run_time_from_screenfile( step, detector, output_directory) if os.path.isfile(step_output_file): hdul = core_utils.read_hdrfits(step_output_file, info=False, show_hdr=False) step_completed = True # add the running time for this step core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time) return hdul, step_output_file, step_input_file, run_pytests, pathloss_switches else: step_completed = False # add the running time for this step core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time) pytest.skip("Test skipped because input file " + step_output_file + " does not exist.") else: pytest.skip("Skipping " + step + " because data is BOTS.")
def output_hdul(set_inandout_filenames, config): # determine if the pipeline is to be run in full, per steps, or skipped run_calwebb_spec2 = config.get("run_calwebb_spec2_in_full", "run_calwebb_spec2") if run_calwebb_spec2 == "skip": print( '\n * PTT finished processing run_calwebb_spec2 is set to skip. \n' ) pytest.exit( "Skipping pipeline run and tests for spec2, run_calwebb_spec2 is set to skip in PTT_config file." ) elif "T" in run_calwebb_spec2: run_calwebb_spec2 = True else: run_calwebb_spec2 = False # get the general info set_inandout_filenames_info = core_utils.read_info4outputhdul( config, set_inandout_filenames) step, txt_name, step_input_file, step_output_file, outstep_file_suffix = set_inandout_filenames_info # determine if the spec2 pipeline should be run or not - obtained from CRDS repo: # https://github.com/spacetelescope/crds/blob/7fc216e73bd81d334b5d98d165be2caa3b6175e1/crds/jwst/jwst_system_crdscfg_b7.5.yaml exp_type = fits.getval(step_input_file, 'EXP_TYPE') allowed_exp_type_values = [ 'NRS_AUTOFLAT', 'NRS_AUTOWAVE', 'NRS_BRIGHTOBJ', 'NRS_CONFIRM', 'NRS_FIXEDSLIT', 'NRS_FOCUS', 'NRS_IFU', 'NRS_IMAGE', 'NRS_LAMP', 'NRS_MIMF', 'NRS_MSASPEC', 'NRS_MSATA', 'NRS_TACONFIRM', 'NRS_TACQ', 'NRS_TASLIT', 'NRS_WATA' ] if exp_type not in allowed_exp_type_values: print( "EXP_TYPE=", exp_type, " is not in exposure types expected for running the stage 2 pipeline: ", allowed_exp_type_values) pytest.exit( "The EXP_TYPE value of this data is not expected to run in the stage 2 pipeline." ) # start the timer to compute the step running time of PTT PTT_start_time = time.time() # check if the filter is to be changed change_filter_opaque = config.getboolean("calwebb_spec2_input_file", "change_filter_opaque") if change_filter_opaque: _, step_input_file = change_filter_opaque2science.change_filter_opaque( step_input_file) change_filter_opaque_msg = " * With FILTER=OPAQUE, the calwebb_spec2 will run up to the extract_2d step. " \ "Further steps will be skipped. \n" print(change_filter_opaque_msg) # determine which steps are to be run, if not run in full run_pipe_step = config.getboolean("run_pipe_steps", step) # determine which tests are to be run assign_wcs_completion_tests = config.getboolean( "run_pytest", "_".join((step, "completion", "tests"))) assign_wcs_reffile_tests = config.getboolean( "run_pytest", "_".join((step, "reffile", "tests"))) assign_wcs_validation_tests = config.getboolean( "run_pytest", "_".join((step, "validation", "tests"))) run_pytests = [ assign_wcs_completion_tests, assign_wcs_reffile_tests, assign_wcs_validation_tests ] # get other relevant info from PTT config file compare_assign_wcs_and_extract_2d_with_esa = config.getboolean( "benchmark_intermediary_products", "compare_assign_wcs_and_extract_2d_with_esa") esa_files_path = config.get("benchmark_intermediary_products", "esa_files_path") data_directory = config.get("calwebb_spec2_input_file", "data_directory") truth_file = os.path.join( data_directory, config.get("benchmark_intermediary_products", "truth_file_assign_wcs")) if compare_assign_wcs_and_extract_2d_with_esa: truth_file = esa_files_path print("Will use this 'truth' file to compare result of assign_wcs: ") print(truth_file) wcs_threshold_diff = config.get("additional_arguments", "wcs_threshold_diff") save_wcs_plots = config.getboolean("additional_arguments", "save_wcs_plots") output_directory = config.get("calwebb_spec2_input_file", "output_directory") # Get the detector used detector = fits.getval(step_input_file, "DETECTOR", 0) # get main header from input file inhdu = core_utils.read_hdrfits(step_input_file, info=False, show_hdr=False) # if run_calwebb_spec2 is True calwebb_spec2 will be called, else individual steps will be ran step_completed = False end_time = '0.0' # Check if data is IFU that the Image Model keyword is correct mode_used = config.get("calwebb_spec2_input_file", "mode_used").lower() if mode_used == "ifu": DATAMODL = fits.getval(step_input_file, "DATAMODL", 0) if DATAMODL != "IFUImageModel": fits.setval(step_input_file, "DATAMODL", 0, value="IFUImageModel") print("DATAMODL keyword changed to IFUImageModel.") # get the shutter configuration file for MOS data only msa_shutter_conf = "No shutter configuration file will be used." if core_utils.check_MOS_true(inhdu): msa_shutter_conf = config.get("benchmark_intermediary_products", "msa_conf_name") # check if the configuration shutter file name is in the header of the fits file and if not add it msametfl = fits.getval(step_input_file, "MSAMETFL", 0) if os.path.basename(msa_shutter_conf) != msametfl: msametfl = os.path.basename(msa_shutter_conf) fits.setval(step_input_file, "MSAMETFL", 0, value=msametfl) # check if processing an image, then set proper variables imaging_mode = False if mode_used in ('image', 'confirm', 'taconfirm', 'wata', 'msata', 'bota', 'focus', 'mimf'): run_calwebb_spec2 = True imaging_mode = True print( '\n * Image processing will only be run in full with PTT. All intermediary products will be saved.' ) print( ' -> For now, all pytests will be skipped since there are now image-specific routines yet. \n' ) # TODO: add imaging tests # get the name of the configuration file and run the pipeline calwebb_spec2_cfg = config.get("run_calwebb_spec2_in_full", "calwebb_spec2_cfg") # copy the configuration file to create the pipeline log pipelog = assign_wcs_utils.set_pipe_log(calwebb_spec2_cfg, detector) # run the pipeline if run_calwebb_spec2: # Create the logfile for PTT, but remove the previous log file PTTcalspec2_log = os.path.join(output_directory, 'PTT_calspec2_' + detector + '.log') if imaging_mode: PTTcalspec2_log = PTTcalspec2_log.replace('calspec2', 'calimage2') if os.path.isfile(PTTcalspec2_log): os.remove(PTTcalspec2_log) print( "Information outputed to screen from PTT will be logged in file: ", PTTcalspec2_log) for handler in logging.root.handlers[:]: logging.root.removeHandler(handler) logging.basicConfig(filename=PTTcalspec2_log, level=logging.INFO) logging.info(pipeline_version) if change_filter_opaque: logging.info(change_filter_opaque_msg) run_calwebb_spec2_msg = " *** Will run pipeline in full ... " print(run_calwebb_spec2_msg) logging.info(run_calwebb_spec2_msg) # create the map txt_name = "spec2_full_run_map_" + detector + ".txt" if os.path.isfile(txt_name): os.remove(txt_name) assign_wcs_utils.create_completed_steps_txtfile( txt_name, step_input_file) # start the timer to compute the step running time of PTT core_utils.start_end_PTT_time(txt_name, start_time=PTT_start_time, end_time=None) if mode_used == "bots": calwebb_spec2_cfg = calwebb_spec2_cfg.replace( "calwebb_spec2.cfg", "calwebb_tso-spec2.cfg") print( '\nUsing the following configuration file to run TSO pipeline:' ) print(calwebb_spec2_cfg, '\n') if imaging_mode: calwebb_image2_cfg = calwebb_spec2_cfg.replace( "calwebb_spec2.cfg", "calwebb_image2.cfg") print( '\nUsing the following configuration file to run imaging pipeline:' ) print(calwebb_image2_cfg, '\n') else: print( '\nUsing the following configuration file to run spectroscopic pipeline:' ) print(calwebb_spec2_cfg, '\n') input_file = config.get("calwebb_spec2_input_file", "input_file") if "_uncal_rate" in input_file: input_file = input_file.replace("_uncal_rate", "") if "_uncal" in input_file: input_file = input_file.replace("_uncal", "") # final_output_name = "final_output_spec2_"+detector+"_cal.fits" if detector.lower() not in input_file.lower(): input_file = input_file.replace(".fits", "_" + detector + ".fits") final_output_name = input_file.replace(".fits", "_cal.fits") if '_rate' in final_output_name: final_output_name = final_output_name.replace('_rate', '') if core_utils.check_MOS_true(inhdu): # copy the MSA shutter configuration file into the pytest directory subprocess.run(["cp", msa_shutter_conf, "."]) # start the timer to compute the step running time start_time = time.time() # run the pipeline print('Running pipeline... \n') try: if not imaging_mode: Spec2Pipeline.call( step_input_file, config_file=calwebb_spec2_cfg) # , logcfg=stpipelogcfg) else: Image2Pipeline.call(step_input_file, config_file=calwebb_image2_cfg) except: pytest.exit( "\n * The pipeline crashed, probably due to no open slits fall on detector" ) """ # For the moment, the pipeline is using the wrong reference file for slit 400A1, so the # file needs to be re-processed with the right reference file if core_utils.check_FS_true(inhdu): print("\n * WARNING: For the moment, the wrong reference file is being used for " "processing slit 400A1. The file will be re-processed ") # print(" $ jwst.pathloss.PathLossStep final_output_caldet1_NRS1_srctype.fits " # "--override_pathloss=jwst-nirspec-a400.plrf.fits \n") pathloss_400a1 = step_input_file.replace("srctype.fits", "pathloss_400A1.fits") reffile_400a1 = "jwst-nirspec-a400.plrf.fits" print("Re-processing slit with new reference file: ", reffile_400a1) pl = PathLossStep() pl.override_pathloss = reffile_400a1 pl.run(step_input_file) subprocess.run(["mv", step_input_file.replace("srctype", "pathlossstep"), pathloss_400a1]) print("Saved pipeline re-processed file as: ", pathloss_400a1) """ # end the timer to compute calwebb_spec2 running time end_time = repr(time.time() - start_time) # this is in seconds calspec2_time = " * Pipeline took " + end_time + " seconds to finish.\n" print(calspec2_time) logging.info(calspec2_time) # remove the copy of the MSA shutter configuration file if core_utils.check_MOS_true(inhdu): if TESTSDIR == os.path.dirname(msametfl): print("Removing MSA config file from: ", TESTSDIR) subprocess.run(["rm", msametfl]) # add the detector string to the name of the files and move them to the working directory core_utils.add_detector2filename(output_directory, step_input_file) final_output_name_msg = "\nThe final pipeline product was saved as: " + final_output_name print(final_output_name_msg) logging.info(final_output_name_msg) # read the assign wcs fits file hdul = core_utils.read_hdrfits(step_output_file, info=False, show_hdr=False) # scihdul = core_utils.read_hdrfits(step_output_file, info=False, show_hdr=False, ext=1) # rename and move the pipeline log file try: calspec2_pipelog = "calspec2_pipeline_" + detector + ".log" if imaging_mode: calspec2_pipelog = calspec2_pipelog.replace( 'calspec2', 'calimage2') path_where_pipeline_was_run = os.getcwd() logfile = glob(path_where_pipeline_was_run + "/" + pipelog)[0] print(logfile) os.rename(logfile, os.path.join(output_directory, calspec2_pipelog)) except IndexError: print("\nWARNING: Something went wrong. Could not find a ", pipelog, " file \n") # make sure we are able to find calspec2_pipelog either in the calwebb_spec2 directory or in the working dir if not os.path.isfile(calspec2_pipelog): calspec2_pipelog = os.path.join(output_directory, calspec2_pipelog) # add the running time for all steps step_running_times = core_utils.calculate_step_run_time( calspec2_pipelog) end_time_list = [] for stp in core_utils.step_string_dict: if stp in step_running_times: step_completed = True step_time = step_running_times[stp]["run_time"] out_suffix = core_utils.step_string_dict[stp]["suffix"] core_utils.add_completed_steps(txt_name, stp, out_suffix, step_completed, step_time) end_time_list.append(step_time) # print total running time in the text file and move it to the indicated directory string2print = "pipeline_total_time" if float(end_time) <= sum(end_time_list): tot_time = repr(sum(end_time_list)) else: tot_time = end_time assign_wcs_utils.print_time2file(txt_name, tot_time, string2print) PTT_runtimes_msg = "Pipeline and PTT run times written in file: " + os.path.basename( txt_name) + " in working directory. \n" print(PTT_runtimes_msg) logging.info(PTT_runtimes_msg) # move the final reporting text files to the working directory core_utils.move_txt_files_2workdir(config, detector) # rename the _cal file last_suffix = os.path.basename(step_output_file).split( ".fits")[0].split("_")[-1] cal_file = step_output_file.replace(last_suffix, "_cal") if os.path.isfile(cal_file): if "caldet1" in cal_file: spec2_cal_file = cal_file.replace("caldet1", "spec2") os.rename(cal_file, spec2_cal_file) # end script for imaging case if imaging_mode: print('\nPTT finished processing imaging mode. \n') pytest.exit( "Skipping pytests for now because they need to be written for imaging mode." ) return hdul, step_output_file, msa_shutter_conf, truth_file, wcs_threshold_diff, save_wcs_plots, \ run_pytests, mode_used, compare_assign_wcs_and_extract_2d_with_esa else: # create the map but remove a previous one if it exists if os.path.isfile(txt_name): os.remove(txt_name) assign_wcs_utils.create_completed_steps_txtfile( txt_name, step_input_file) # start the timer to compute the step running time of PTT core_utils.start_end_PTT_time(txt_name, start_time=PTT_start_time, end_time=None) msg = "\n Pipeline and PTT run times will be written in file: " + os.path.basename( txt_name) + " in working directory. \n" print(msg) logging.info(msg) if run_pipe_step: # Create the logfile for PTT, but erase the previous one if it exists PTTcalspec2_log = os.path.join( output_directory, 'PTT_calspec2_' + detector + '_' + step + '.log') if imaging_mode: PTTcalspec2_log = PTTcalspec2_log.replace( 'calspec2', 'calimage2') if os.path.isfile(PTTcalspec2_log): os.remove(PTTcalspec2_log) print( "Information outputed to screen from PTT will be logged in file: ", PTTcalspec2_log) for handler in logging.root.handlers[:]: logging.root.removeHandler(handler) logging.basicConfig(filename=PTTcalspec2_log, level=logging.INFO) logging.info(pipeline_version) if change_filter_opaque: logging.info(change_filter_opaque_msg) # check that previous pipeline steps were run up to this point core_utils.check_completed_steps(step, step_input_file) if os.path.isfile(step_input_file): msg = " *** Step " + step + " set to True" print(msg) logging.info(msg) stp = AssignWcsStep() if core_utils.check_MOS_true(inhdu): if os.getcwd() != output_directory: # copy the MSA shutter configuration file into the pytest directory subprocess.run(["cp", msa_shutter_conf, "."]) # get the right configuration files to run the step local_pipe_cfg_path = config.get("calwebb_spec2_input_file", "local_pipe_cfg_path") # start the timer to compute the step running time print("running pipeline...") start_time = time.time() if local_pipe_cfg_path == "pipe_source_tree_code": result = stp.call(step_input_file) else: result = stp.call(step_input_file, config_file=local_pipe_cfg_path + '/assign_wcs.cfg') result.save(step_output_file) # end the timer to compute the step running time end_time = repr(time.time() - start_time) # this is in seconds msg = "Step " + step + " took " + end_time + " seconds to finish" print(msg) logging.info(msg) if core_utils.check_MOS_true(inhdu): if TESTSDIR == os.path.dirname(msametfl): # remove the copy of the MSA shutter configuration file print("Removing MSA config file from: ", TESTSDIR) subprocess.run(["rm", msametfl]) # rename and move the pipeline log file try: calspec2_pipelog = "calspec2_pipeline_" + step + "_" + detector + ".log" if imaging_mode: calspec2_pipelog = calspec2_pipelog.replace( 'calspec2', 'calimage2') pytest_workdir = TESTSDIR logfile = glob(pytest_workdir + "/" + pipelog)[0] os.rename(logfile, os.path.join(output_directory, calspec2_pipelog)) except IndexError: print( "\n* WARNING: Something went wrong. Could not find a ", pipelog, " file \n") else: msg = "Skipping step. Input file " + step_input_file + " does not exit." print(msg) logging.info(msg) core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time) pytest.skip("Skipping " + step + " because the input file does not exist.") else: print("Skipping running pipeline step ", step) # add the running time for this step end_time = core_utils.get_stp_run_time_from_screenfile( step, detector, output_directory) if os.path.isfile(step_output_file): hdul = core_utils.read_hdrfits(step_output_file, info=False, show_hdr=False) step_completed = True # add the running time for this step core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time) return hdul, step_output_file, msa_shutter_conf, truth_file, wcs_threshold_diff, save_wcs_plots, \ run_pytests, mode_used, compare_assign_wcs_and_extract_2d_with_esa else: step_completed = False # add the running time for this step core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time) pytest.skip("Test skipped because input file " + step_output_file + " does not exist.")