def check_output_is_zero(output_hdul):
"""
This test is a simple subtraction of the input file minus a copy of the input file (instead of the actual
MSA imprint file. The output is expected to be zero.
:param output_hdul: list
:return: result: boolean
"""
# output_hdul = hdul, step_output_file, step_input_file, run_pytests
step_input_file = output_hdul[2]
step_output_file = output_hdul[1]
# Only run test if data is IFU or MSA
inhdu = core_utils.read_hdrfits(step_input_file,
info=False,
show_hdr=False)
if core_utils.check_IFU_true(inhdu) or core_utils.check_MOS_true(inhdu):
# set specifics for the test
msa_imprint_structure = copy.deepcopy(step_input_file)
result_to_check = step_output_file.replace(".fits", "_zerotest.fits")
# run the step with the specifics
stp = ImprintStep()
res = stp.call(step_input_file, msa_imprint_structure)
res.save(result_to_check)
# check that the end product of image - image is zero
c = fits.getdata(result_to_check)
subtraction = sum(c.flatten())
result = False
if subtraction == 0.0:
result = True
# erase test output file
subprocess.run(["rm", result_to_check])
return result
def validate_barshadow(output_hdul):
hdu = output_hdul[0]
log_msgs = None
# show the figures
show_figs = False
# Determine if data is MOS
if core_utils.check_MOS_true(hdu):
# Determine if the source is point or extended. If extended, unknown, or not present, correction
# will be applied.
if 'SRCTYPE' in hdu:
if hdu['SRCTYPE'] == 'POINT':
pytest.skip(
"Skipping pytest: The SRCTYPE keyword is set to POINT so barshadow correction is not "
"applied.")
plfile = output_hdul[1].replace('_barshadow', '_pathloss')
bsfile = output_hdul[1]
barshadow_threshold_diff, save_barshadow_final_plot, save_barshadow_intermediary_plots, write_barshadow_files = output_hdul[
2]
barshadow_testresult, result_msg, log_msgs = barshadow_testing.run_barshadow_tests(
plfile,
bsfile,
barshadow_threshold_diff=float(barshadow_threshold_diff),
save_final_figs=save_barshadow_final_plot,
show_final_figs=show_figs,
save_intermediary_figs=save_barshadow_intermediary_plots,
show_intermediary_figs=show_figs,
write_barshadow_files=write_barshadow_files,
debug=False)
else:
pytest.skip("Skipping pytest: The input fits file is not MOS.")
if log_msgs is not None:
for msg in log_msgs:
logging.info(msg)
if barshadow_testresult == "skip":
logging.info(result_msg)
pytest.skip(result_msg)
else:
print(result_msg)
logging.info(result_msg)
return barshadow_testresult
def validate_extract2d(output_hdul):
# get the input information for the wcs routine
hdu = output_hdul[0]
infile_name = output_hdul[1]
msa_conf_name = output_hdul[2]
truth_file = output_hdul[3]
extract_2d_threshold_diff = output_hdul[8]
compare_assign_wcs_and_extract_2d_with_esa = output_hdul[9]
esa_files_path = None
if compare_assign_wcs_and_extract_2d_with_esa:
esa_files_path = output_hdul[3]
truth_file = None
print(
'Will be using this number of pixels as threshold for extract_2d test: ',
extract_2d_threshold_diff)
msg = "\n Performing extract_2d validation test... "
print(msg)
logging.info(msg)
if core_utils.check_FS_true(hdu):
result, log_msgs = check_corners_extract2d.find_FSwindowcorners(
infile_name,
truth_file=truth_file,
esa_files_path=esa_files_path,
extract_2d_threshold_diff=extract_2d_threshold_diff)
elif core_utils.check_MOS_true(hdu):
result, log_msgs = check_corners_extract2d.find_MOSwindowcorners(
infile_name,
msa_conf_name,
truth_file=truth_file,
esa_files_path=esa_files_path,
extract_2d_threshold_diff=extract_2d_threshold_diff)
else:
pytest.skip("Skipping pytest: The fits file is not FS or MOS.")
for msg in log_msgs:
logging.info(msg)
if result == "skip":
pytest.skip("Extract_2d validation will be skipped.")
return result
def validate_barshadow(output_hdul):
hdu = output_hdul[0]
log_msgs = None
# show the figures
show_figs = False
# Determine if data is MOS
if core_utils.check_MOS_true(hdu):
plfile = output_hdul[1].replace('_barshadow', '_pathloss')
bsfile = output_hdul[1]
(barshadow_threshold_diff, save_barshadow_final_plot,
save_barshadow_intermediary_plots,
write_barshadow_files) = output_hdul[2]
barshadow_testresult, result_msg, log_msgs = barshadow_testing.run_barshadow_tests(
plfile,
bsfile,
barshadow_threshold_diff=float(barshadow_threshold_diff),
save_final_figs=save_barshadow_final_plot,
show_final_figs=show_figs,
save_intermediary_figs=save_barshadow_intermediary_plots,
show_intermediary_figs=show_figs,
write_barshadow_files=write_barshadow_files,
debug=False)
else:
pytest.skip("Skipping pytest: The input fits file is not MOS.")
if log_msgs is not None:
for msg in log_msgs:
logging.info(msg)
if barshadow_testresult == "skip":
print(result_msg)
logging.info(result_msg)
pytest.skip(result_msg)
else:
print(result_msg)
logging.info(result_msg)
return barshadow_testresult
def output_hdul(set_inandout_filenames, config):
# determine if the pipeline is to be run in full, per steps, or skipped
run_calwebb_spec2 = config.get("run_calwebb_spec2_in_full",
"run_calwebb_spec2")
if run_calwebb_spec2 == "skip":
print(
'\n * PTT finished processing run_calwebb_spec2 is set to skip. \n'
)
pytest.exit(
"Skipping pipeline run and tests for spec2, run_calwebb_spec2 is set to skip in PTT_config file."
)
elif "T" in run_calwebb_spec2:
run_calwebb_spec2 = True
else:
run_calwebb_spec2 = False
# get the general info
set_inandout_filenames_info = core_utils.read_info4outputhdul(
config, set_inandout_filenames)
step, txt_name, step_input_file, step_output_file, outstep_file_suffix = set_inandout_filenames_info
run_pipe_step = config.getboolean("run_pipe_steps", step)
# determine which tests are to be run
barshadow_completion_tests = config.getboolean(
"run_pytest", "_".join((step, "completion", "tests")))
barshadow_validation_tests = config.getboolean(
"run_pytest", "_".join((step, "validation", "tests")))
run_pytests = [barshadow_completion_tests, barshadow_validation_tests]
barshadow_threshold_diff = config.get("additional_arguments",
"barshadow_threshold_diff")
save_barshadow_final_plot = config.getboolean("additional_arguments",
"save_barshadow_final_plot")
save_barshadow_intermediary_plots = config.getboolean(
"additional_arguments", "save_barshadow_intermediary_plots")
write_barshadow_files = config.getboolean("additional_arguments",
"write_barshadow_files")
barshadow_switches = [
barshadow_threshold_diff, save_barshadow_final_plot,
save_barshadow_intermediary_plots, write_barshadow_files
]
end_time = '0.0'
# Only run step if data is MOS
output_directory = config.get("calwebb_spec2_input_file",
"output_directory")
initial_input_file = config.get("calwebb_spec2_input_file", "input_file")
initial_input_file = os.path.join(output_directory, initial_input_file)
if os.path.isfile(initial_input_file):
inhdu = core_utils.read_hdrfits(initial_input_file,
info=False,
show_hdr=False)
detector = fits.getval(initial_input_file, "DETECTOR", 0)
else:
pytest.skip(
"Skipping " + step +
" because the initial input file given in PTT_config.cfg does not exist."
)
print("core_utils.check_MOS_true(inhdu)=",
core_utils.check_MOS_true(inhdu))
if core_utils.check_MOS_true(inhdu):
# if run_calwebb_spec2 is True calwebb_spec2 will be called, else individual steps will be ran
step_completed = False
# check if the filter is to be changed
change_filter_opaque = config.getboolean("calwebb_spec2_input_file",
"change_filter_opaque")
if change_filter_opaque:
is_filter_opaque, step_input_filename = change_filter_opaque2science.change_filter_opaque(
step_input_file, step=step)
if is_filter_opaque:
filter_opaque_msg = "With FILTER=OPAQUE, the calwebb_spec2 will run up to the extract_2d step. " \
"Barshadow pytest now set to Skip."
print(filter_opaque_msg)
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed, end_time)
pytest.skip("Skipping " + step + " because FILTER=OPAQUE.")
if run_calwebb_spec2:
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
return hdul, step_output_file, barshadow_switches, run_pytests
else:
if run_pipe_step:
# Create the logfile for PTT, but erase the previous one if it exists
PTTcalspec2_log = os.path.join(
output_directory,
'PTT_calspec2_' + detector + '_' + step + '.log')
if os.path.isfile(PTTcalspec2_log):
os.remove(PTTcalspec2_log)
print(
"Information outputed to screen from PTT will be logged in file: ",
PTTcalspec2_log)
for handler in logging.root.handlers[:]:
logging.root.removeHandler(handler)
logging.basicConfig(filename=PTTcalspec2_log,
level=logging.INFO)
# print pipeline version
import jwst
pipeline_version = "\n *** Using jwst pipeline version: " + jwst.__version__ + " *** \n"
print(pipeline_version)
logging.info(pipeline_version)
if change_filter_opaque:
logging.info(filter_opaque_msg)
if os.path.isfile(step_input_file):
msg = " *** Step " + step + " set to True"
print(msg)
logging.info(msg)
stp = BarShadowStep()
# check that previous pipeline steps were run up to this point
core_utils.check_completed_steps(step, step_input_file)
# get the right configuration files to run the step
#local_pipe_cfg_path = config.get("calwebb_spec2_input_file", "local_pipe_cfg_path")
# start the timer to compute the step running time
start_time = time.time()
#if local_pipe_cfg_path == "pipe_source_tree_code":
result = stp.call(step_input_file)
#else:
# result = stp.call(step_input_file, config_file=local_pipe_cfg_path+'/NOCONFIGFILE.cfg')
result.save(step_output_file)
# end the timer to compute calwebb_spec2 running time
end_time = repr(time.time() -
start_time) # this is in seconds
msg = " * calwebb_spec2 took " + end_time + " seconds to finish."
print(msg)
logging.info(msg)
step_completed = True
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
# rename and move the pipeline log file
pipelog = "pipeline_" + detector + ".log"
try:
calspec2_pilelog = "calspec2_pipeline_" + step + "_" + detector + ".log"
pytest_workdir = TESTSDIR
logfile = glob(pytest_workdir + "/" + pipelog)[0]
os.rename(
logfile,
os.path.join(output_directory, calspec2_pilelog))
except IndexError:
print(
"\n* WARNING: Something went wrong. Could not find a ",
pipelog, " file \n")
# add the running time for this step
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed, end_time)
return hdul, step_output_file, barshadow_switches, run_pytests
else:
msg = " The input file does not exist. Skipping step."
print(msg)
logging.info(msg)
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed, end_time)
pytest.skip("Skipping " + step +
" because the input file does not exist.")
else:
msg = "Skipping running pipeline step " + step
print(msg)
logging.info(msg)
end_time = core_utils.get_stp_run_time_from_screenfile(
step, detector, output_directory)
if os.path.isfile(step_output_file):
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
step_completed = True
# add the running time for this step
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed, end_time)
return hdul, step_output_file, barshadow_switches, run_pytests
else:
step_completed = False
# add the running time for this step
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed, end_time)
pytest.skip("Test skipped because input file " +
step_output_file + " does not exist.")
else:
pytest.skip("Skipping " + step + " because data is not MOS.")
def output_hdul(set_inandout_filenames, config):
# determine if the pipeline is to be run in full, per steps, or skipped
run_calwebb_spec2 = config.get("run_calwebb_spec2_in_full",
"run_calwebb_spec2")
if run_calwebb_spec2 == "skip":
print(
'\n * PTT finished processing run_calwebb_spec2 is set to skip. \n'
)
pytest.exit(
"Skipping pipeline run and tests for spec2, run_calwebb_spec2 is set to skip in PTT_config file."
)
elif "T" in run_calwebb_spec2:
run_calwebb_spec2 = True
else:
run_calwebb_spec2 = False
# get the general info
set_inandout_filenames_info = core_utils.read_info4outputhdul(
config, set_inandout_filenames)
step, txt_name, step_input_file, step_output_file, outstep_file_suffix = set_inandout_filenames_info
run_pipe_step = config.getboolean("run_pipe_steps", step)
# determine which tests are to be run
extract_1d_completion_tests = config.getboolean(
"run_pytest", "_".join((step, "completion", "tests")))
extract_1d_reffile_tests = config.getboolean(
"run_pytest", "_".join((step, "reffile", "tests")))
# extract_1d_validation_tests = config.getboolean("run_pytest", "_".join((step, "validation", "tests")))
run_pytests = [extract_1d_completion_tests,
extract_1d_reffile_tests] #, extract_1d_validation_tests]
# Make sure at this point the MSA shutter configuration file is removed from the calwebb_spec2_pytests directory
msa_shutter_conf = config.get("benchmark_intermediary_products",
"msa_conf_name")
msametfl = os.path.basename(msa_shutter_conf)
# remove the copy of the MSA shutter configuration file
inhdu = core_utils.read_hdrfits(step_input_file,
info=False,
show_hdr=False)
if core_utils.check_MOS_true(inhdu):
if TESTSDIR == os.path.dirname(msametfl):
print("Removing MSA config file from: ", TESTSDIR)
os.remove(msametfl)
# check if processing an image, then set proper variables
imaging_mode = False
mode_used = config.get("calwebb_spec2_input_file", "mode_used").lower()
if mode_used in ('image', 'confirm', 'taconfirm', 'wata', 'msata', 'bota',
'focus', 'mimf'):
run_calwebb_spec2 = True
imaging_mode = True
# if run_calwebb_spec2 is True calwebb_spec2 will be called, else individual steps will be ran
step_completed = False
end_time = '0.0'
# Get the detector used
output_directory = config.get("calwebb_spec2_input_file",
"output_directory")
initial_input_file = config.get("calwebb_spec2_input_file", "input_file")
initial_input_file = os.path.join(output_directory, initial_input_file)
if os.path.isfile(initial_input_file):
detector = fits.getval(initial_input_file, "DETECTOR", 0)
else:
pytest.skip(
"Skipping " + step +
" because the initial input file given in PTT_config.cfg does not exist."
)
# set the name of the pipeline log to be read for times and renamed at the end for book-keeping
pipelog = "pipeline_" + detector + ".log"
# check if the filter is to be changed
change_filter_opaque = config.getboolean("calwebb_spec2_input_file",
"change_filter_opaque")
if change_filter_opaque:
is_filter_opaque, step_input_filename = change_filter_opaque2science.change_filter_opaque(
step_input_file, step=step)
if is_filter_opaque:
filter_opaque_msg = "With FILTER=OPAQUE, the calwebb_spec2 will run up to the extract_2d step. " \
"Extract_1d pytest now set to Skip."
print(filter_opaque_msg)
core_utils.add_completed_steps(txt_name, step, outstep_file_suffix,
step_completed, end_time)
#core_utils.convert_html2pdf() # convert the html report into a pdf file
# move the final reporting files to the working directory
core_utils.move_txt_files_2workdir(config, detector)
pytest.skip("Skipping " + step + " because FILTER=OPAQUE.")
if run_calwebb_spec2:
# read the output header
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
# end the timer to compute the step running time of PTT
PTT_end_time = time.time()
core_utils.start_end_PTT_time(txt_name,
start_time=None,
end_time=PTT_end_time)
# move the final reporting files to the working directory
core_utils.move_txt_files_2workdir(config, detector)
return hdul, step_output_file, run_pytests
else:
if run_pipe_step:
# Create the logfile for PTT, but erase the previous one if it exists
PTTcalspec2_log = os.path.join(
output_directory,
'PTT_calspec2_' + detector + '_' + step + '.log')
if imaging_mode:
PTTcalspec2_log = PTTcalspec2_log.replace(
'calspec2', 'calimage2')
if os.path.isfile(PTTcalspec2_log):
os.remove(PTTcalspec2_log)
print(
"Information outputed to screen from PTT will be logged in file: ",
PTTcalspec2_log)
for handler in logging.root.handlers[:]:
logging.root.removeHandler(handler)
logging.basicConfig(filename=PTTcalspec2_log, level=logging.INFO)
# print pipeline version
import jwst
pipeline_version = "\n *** Using jwst pipeline version: " + jwst.__version__ + " *** \n"
print(pipeline_version)
logging.info(pipeline_version)
if change_filter_opaque:
logging.info(filter_opaque_msg)
if os.path.isfile(step_input_file):
msg = " *** Step " + step + " set to True"
print(msg)
logging.info(msg)
stp = Extract1dStep()
# check that previous pipeline steps were run up to this point
core_utils.check_completed_steps(step, step_input_file)
# get the right configuration files to run the step
local_pipe_cfg_path = config.get("calwebb_spec2_input_file",
"local_pipe_cfg_path")
# start the timer to compute the step running time
start_time = time.time()
if local_pipe_cfg_path == "pipe_source_tree_code":
result = stp.call(step_input_file)
else:
result = stp.call(step_input_file,
config_file=local_pipe_cfg_path +
'/extract_1d.cfg')
result.save(step_output_file)
# end the timer to compute the step running time
end_time = repr(time.time() - start_time) # this is in seconds
msg = "Step " + step + " took " + end_time + " seconds to finish"
print(msg)
logging.info(msg)
# rename and move the pipeline log file
calspec2_pilelog = "calspec2_pipeline_" + step + "_" + detector + ".log"
if imaging_mode:
calspec2_pilelog = calspec2_pilelog.replace(
'calspec2', 'calimage2')
os.rename(pipelog,
os.path.join(output_directory, calspec2_pilelog))
else:
msg = " The input file does not exist. Skipping step."
print(msg)
logging.info(msg)
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed, end_time)
#core_utils.convert_html2pdf() # convert the html report into a pdf file
# end the timer to compute the step running time of PTT
PTT_end_time = time.time()
core_utils.start_end_PTT_time(txt_name,
start_time=None,
end_time=PTT_end_time)
# move the final reporting files to the working directory
core_utils.move_txt_files_2workdir(config, detector)
# skip the test if input file does not exist
pytest.skip("Skipping " + step +
" because the input file does not exist.")
else:
msg = "Skipping running pipeline step " + step
print(msg)
logging.info(msg)
# get the running time for this step
end_time = core_utils.get_stp_run_time_from_screenfile(
step, detector, output_directory)
# add the running time for this step
if os.path.isfile(step_output_file):
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
step_completed = True
core_utils.add_completed_steps(txt_name, step, outstep_file_suffix,
step_completed, end_time)
else:
step_completed = False
core_utils.add_completed_steps(txt_name, step, outstep_file_suffix,
step_completed, end_time)
# move the final reporting files to the working directory
core_utils.move_txt_files_2workdir(config, detector)
pytest.skip("Test skipped because input file " + step_output_file +
" does not exist.")
# get the total running time and print it in the file
total_time = repr(core_utils.get_time_to_run_pipeline(txt_name))
total_time_min = repr(round(float(total_time) / 60.0, 2))
msg = "\n\n **** The total time for the pipeline to run was " + total_time + " seconds."
print(msg)
logging.info(msg)
line2write = "{:<20} {:<20} {:<20} {:<20}".format(
'', '', 'total_time ',
total_time + ' =' + total_time_min + 'min')
print(line2write)
logging.info(line2write)
with open(txt_name, "a") as tf:
tf.write(line2write + "\n")
# convert the html report into a pdf file
#core_utils.convert_html2pdf()
# end the timer to compute the step running time of PTT
PTT_end_time = time.time()
core_utils.start_end_PTT_time(txt_name,
start_time=None,
end_time=PTT_end_time)
# move the final reporting text files to the working directory
core_utils.move_txt_files_2workdir(config, detector)
# rename and move the PTT log file
try:
calspec2_pipelog = "calspec2_pipeline_" + step + "_" + detector + ".log"
if imaging_mode:
calspec2_pipelog = calspec2_pipelog.replace(
'calspec2', 'calimage2')
pytest_workdir = TESTSDIR
logfile = glob(pytest_workdir + "/" + pipelog)[0]
os.rename(logfile, os.path.join(output_directory,
calspec2_pipelog))
except IndexError:
print("\n* WARNING: Something went wrong. Could not find a ",
pipelog, " file \n")
return hdul, step_output_file, run_pytests
def output_hdul(set_inandout_filenames, config):
# determine if the pipeline is to be run in full, per steps, or skipped
run_calwebb_spec2 = config.get("run_calwebb_spec2_in_full",
"run_calwebb_spec2")
if run_calwebb_spec2 == "skip":
print(
'\n * PTT finished processing run_calwebb_spec2 is set to skip. \n'
)
pytest.exit(
"Skipping pipeline run and tests for spec2, run_calwebb_spec2 is set to skip in PTT_config file."
)
elif "T" in run_calwebb_spec2:
run_calwebb_spec2 = True
else:
run_calwebb_spec2 = False
# get the general info
set_inandout_filenames_info = core_utils.read_info4outputhdul(
config, set_inandout_filenames)
step, txt_name, step_input_file, step_output_file, outstep_file_suffix = set_inandout_filenames_info
# determine which steps are to be run, if not run in full
run_pipe_step = config.getboolean("run_pipe_steps", step)
# determine which tests are to be run
imprint_subtract_completion_tests = config.getboolean(
"run_pytest", "_".join((step, "completion", "tests")))
imprint_subtract_numerical_tests = config.getboolean(
"run_pytest", "_".join((step, "numerical", "tests")))
#imprint_subtract_validation_tests = config.getboolean("run_pytest", "_".join((step, "validation", "tests")))
run_pytests = [
imprint_subtract_completion_tests, imprint_subtract_numerical_tests
] #, imprint_subtract_validation_tests]
end_time = '0.0'
# Only run step if data is IFU or MSA
output_directory = config.get("calwebb_spec2_input_file",
"output_directory")
initial_input_file = config.get("calwebb_spec2_input_file", "input_file")
initial_input_file = os.path.join(output_directory, initial_input_file)
detector = fits.getval(initial_input_file, "DETECTOR", 0)
calspec2_pilelog = "calspec2_pipeline_" + step + "_" + detector + ".log"
pytest_workdir = TESTSDIR
if os.path.isfile(initial_input_file):
inhdu = core_utils.read_hdrfits(initial_input_file,
info=False,
show_hdr=False)
else:
pytest.skip(
"Skipping " + step +
" because the initial input file given in PTT_config.cfg does not exist."
)
if core_utils.check_IFU_true(inhdu) or core_utils.check_MOS_true(inhdu):
# if run_calwebb_spec2 is True calwebb_spec2 will be called, else individual steps will be ran
step_completed = False
if run_calwebb_spec2:
if os.path.isfile(step_output_file):
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
else:
pytest.skip("Skipping " + step +
" because the output file does not exist.")
return hdul, step_output_file, step_input_file, run_pytests
else:
if run_pipe_step:
# Create the logfile for PTT, but erase the previous one if it exists
PTTcalspec2_log = os.path.join(
output_directory,
'PTT_calspec2_' + detector + '_' + step + '.log')
if os.path.isfile(PTTcalspec2_log):
os.remove(PTTcalspec2_log)
print(
"Information outputed to screen from PTT will be logged in file: ",
PTTcalspec2_log)
for handler in logging.root.handlers[:]:
logging.root.removeHandler(handler)
logging.basicConfig(filename=PTTcalspec2_log,
level=logging.INFO)
# print pipeline version
import jwst
pipeline_version = "\n *** Using jwst pipeline version: " + jwst.__version__ + " *** \n"
print(pipeline_version)
logging.info(pipeline_version)
if os.path.isfile(step_input_file):
msg = " The input file " + step_input_file + " exists... will run step " + step
print(msg)
logging.info(msg)
msa_imprint_structure = config.get(
"additional_arguments", "msa_imprint_structure")
msg = "msa_imprint_structure file: " + msa_imprint_structure
print(msg)
logging.info(msg)
if not os.path.isfile(msa_imprint_structure):
print(
" Need msa_imprint_structure file to continue. Step will be skipped."
)
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed,
end_time)
pytest.skip(
"Skipping " + step +
" because msa_imprint_structure file in the configuration "
"file does not exist.")
else:
msg = "*** Step " + step + " set to True"
print(msg)
logging.info(msg)
stp = ImprintStep()
# check that previous pipeline steps were run up to this point
core_utils.check_completed_steps(step, step_input_file)
# get the right configuration files to run the step
local_pipe_cfg_path = config.get(
"calwebb_spec2_input_file", "local_pipe_cfg_path")
# start the timer to compute the step running time
start_time = time.time()
if local_pipe_cfg_path == "pipe_source_tree_code":
result = stp.call(step_input_file,
msa_imprint_structure)
else:
result = stp.call(step_input_file,
msa_imprint_structure,
config_file=local_pipe_cfg_path +
'/imprint.cfg')
if result is not None:
result.save(step_output_file)
# end the timer to compute the step running time
end_time = repr(time.time() -
start_time) # this is in seconds
msg = "Step " + step + " took " + end_time + " seconds to finish"
print(msg)
logging.info(msg)
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
step_completed = True
else:
hdul = core_utils.read_hdrfits(step_input_file,
info=False,
show_hdr=False)
# rename and move the pipeline log file
pipelog = "pipeline_" + detector + ".log"
try:
calspec2_pilelog = "calspec2_pipeline_" + step + "_" + detector + ".log"
pytest_workdir = TESTSDIR
logfile = glob(pytest_workdir + "/" + pipelog)[0]
os.rename(
logfile,
os.path.join(output_directory,
calspec2_pilelog))
except IndexError:
print(
"\n* WARNING: Something went wrong. Could not find a ",
pipelog, " file \n")
# add the running time for this step
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed,
end_time)
return hdul, step_output_file, step_input_file, run_pytests
else:
msg = " The input file does not exist. Skipping step."
print(msg)
logging.info(msg)
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed, end_time)
pytest.skip("Skipping " + step +
" because the input file does not exist.")
else:
msg = "Skipping running pipeline step " + step
print(msg)
logging.info(msg)
end_time = core_utils.get_stp_run_time_from_screenfile(
step, detector, output_directory)
if os.path.isfile(step_output_file):
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
step_completed = True
# add the running time for this step
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed, end_time)
return hdul, step_output_file, step_input_file, run_pytests
else:
step_completed = False
# add the running time for this step
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed, end_time)
pytest.skip("Test skipped because input file " +
step_output_file + " does not exist.")
else:
pytest.skip("Skipping " + step +
" because data is neither IFU or MOS.")
def output_hdul(set_inandout_filenames, config):
# determine if the pipeline is to be run in full, per steps, or skipped
run_calwebb_spec2 = config.get("run_calwebb_spec2_in_full",
"run_calwebb_spec2")
if run_calwebb_spec2 == "skip":
print(
'\n * PTT finished processing run_calwebb_spec2 is set to skip. \n'
)
pytest.exit(
"Skipping pipeline run and tests for spec2, run_calwebb_spec2 is set to skip in PTT_config file."
)
elif "T" in run_calwebb_spec2:
run_calwebb_spec2 = True
else:
run_calwebb_spec2 = False
# get the general info
set_inandout_filenames_info = core_utils.read_info4outputhdul(
config, set_inandout_filenames)
step, txt_name, step_input_file, step_output_file, outstep_file_suffix = set_inandout_filenames_info
msa_shutter_conf = config.get("benchmark_intermediary_products",
"msa_conf_name")
msametfl = os.path.basename(msa_shutter_conf)
dflat_path = config.get("benchmark_intermediary_products", "dflat_path")
sflat_path = config.get("benchmark_intermediary_products", "sflat_path")
fflat_path = config.get("benchmark_intermediary_products", "fflat_path")
flattest_threshold_diff = config.get("additional_arguments",
"flattest_threshold_diff")
save_flattest_plot = config.getboolean("additional_arguments",
"save_flattest_plot")
write_flattest_files = config.getboolean("additional_arguments",
"write_flattest_files")
flattest_paths = [
step_output_file, msa_shutter_conf, dflat_path, sflat_path, fflat_path
]
flattest_switches = [
flattest_threshold_diff, save_flattest_plot, write_flattest_files
]
run_pipe_step = config.getboolean("run_pipe_steps", step)
# determine which tests are to be run
flat_field_completion_tests = config.getboolean(
"run_pytest", "_".join((step, "completion", "tests")))
flat_field_reffile_tests = config.getboolean(
"run_pytest", "_".join((step, "reffile", "tests")))
flat_field_validation_tests = config.getboolean(
"run_pytest", "_".join((step, "validation", "tests")))
run_pytests = [
flat_field_completion_tests, flat_field_reffile_tests,
flat_field_validation_tests
]
# if run_calwebb_spec2 is True calwebb_spec2 will be called, else individual steps will be ran
step_completed = False
end_time = '0.0'
# check if the filter is to be changed
change_filter_opaque = config.getboolean("calwebb_spec2_input_file",
"change_filter_opaque")
if change_filter_opaque:
is_filter_opaque, step_input_filename = change_filter_opaque2science.change_filter_opaque(
step_input_file, step=step)
if is_filter_opaque:
filter_opaque_msg = "With FILTER=OPAQUE, the calwebb_spec2 will run up to the extract_2d step. Flat " \
"Field pytest now set to Skip."
print(filter_opaque_msg)
core_utils.add_completed_steps(txt_name, step, outstep_file_suffix,
step_completed, end_time)
pytest.skip("Skipping " + step + " because FILTER=OPAQUE.")
# get the MSA shutter configuration file full path only for MOS data
output_directory = config.get("calwebb_spec2_input_file",
"output_directory")
initial_input_file = config.get("calwebb_spec2_input_file", "input_file")
initial_input_file = os.path.join(output_directory, initial_input_file)
if os.path.isfile(initial_input_file):
inhdu = core_utils.read_hdrfits(initial_input_file,
info=False,
show_hdr=False)
detector = fits.getval(initial_input_file, "DETECTOR", 0)
else:
pytest.skip(
"Skipping " + step +
" because the initial input file given in PTT_config.cfg does not exist."
)
if run_calwebb_spec2:
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
flattest_paths = [
step_output_file, msa_shutter_conf, dflat_path, sflat_path,
fflat_path
]
return hdul, step_output_file, flattest_paths, flattest_switches, run_pytests
else:
if run_pipe_step:
# Create the logfile for PTT, but erase the previous one if it exists
PTTcalspec2_log = os.path.join(
output_directory,
'PTT_calspec2_' + detector + '_' + step + '.log')
if os.path.isfile(PTTcalspec2_log):
os.remove(PTTcalspec2_log)
print(
"Information outputed to screen from PTT will be logged in file: ",
PTTcalspec2_log)
for handler in logging.root.handlers[:]:
logging.root.removeHandler(handler)
logging.basicConfig(filename=PTTcalspec2_log, level=logging.INFO)
# print pipeline version
import jwst
pipeline_version = "\n *** Using jwst pipeline version: " + jwst.__version__ + " *** \n"
print(pipeline_version)
logging.info(pipeline_version)
if change_filter_opaque:
logging.info(filter_opaque_msg)
if os.path.isfile(step_input_file):
msg = " *** Step " + step + " set to True"
print(msg)
logging.info(msg)
stp = FlatFieldStep()
# check that previous pipeline steps were run up to this point
core_utils.check_completed_steps(step, step_input_file)
if core_utils.check_MOS_true(inhdu):
# copy the MSA shutter configuration file into the pytest directory
subprocess.run(["cp", msa_shutter_conf, "."])
# start the timer to compute the step running time
ontheflyflat = step_output_file.replace(
"flat_field.fits", "interpolatedflat.fits")
msg1 = "Step product will be saved as: " + step_output_file
msg2 = "on-the-fly flat will be saved as: " + ontheflyflat
# get the right configuration files to run the step
print(msg1)
print(msg2)
logging.info(msg1)
logging.info(msg2)
local_pipe_cfg_path = config.get("calwebb_spec2_input_file",
"local_pipe_cfg_path")
# start the timer to compute the step running time
start_time = time.time()
if local_pipe_cfg_path == "pipe_source_tree_code":
stp.call(step_input_file,
output_file=step_output_file,
save_interpolated_flat=True)
else:
stp.call(step_input_file,
output_file=step_output_file,
save_interpolated_flat=True,
config_file=local_pipe_cfg_path +
'/flat_field.cfg')
# end the timer to compute the step running time
end_time = repr(time.time() - start_time) # this is in seconds
msg = "Step " + step + " took " + end_time + " seconds to finish"
print(msg)
logging.info(msg)
# move the on-the-fly flat to the working directory
subprocess.run(
["mv", os.path.basename(ontheflyflat), ontheflyflat])
# raname and move the flat_field output
subprocess.run([
"mv",
os.path.basename(step_output_file).replace(
"_flat_field.fits", "_flatfieldstep.fits"),
step_output_file
])
if core_utils.check_MOS_true(inhdu):
if TESTSDIR == os.path.dirname(msametfl):
# remove the copy of the MSA shutter configuration file
print("Removing MSA config file from: ", TESTSDIR)
subprocess.run(["rm", msametfl])
step_completed = True
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
# rename and move the pipeline log file
pipelog = "pipeline_" + detector + ".log"
try:
calspec2_pilelog = "calspec2_pipeline_" + step + "_" + detector + ".log"
pytest_workdir = TESTSDIR
logfile = glob(pytest_workdir + "/" + pipelog)[0]
os.rename(logfile,
os.path.join(output_directory, calspec2_pilelog))
except IndexError:
print(
"\n* WARNING: Something went wrong. Could not find a ",
pipelog, " file \n")
# add the running time for this step
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed, end_time)
return hdul, step_output_file, flattest_paths, flattest_switches, run_pytests
else:
msg = " The input file does not exist. Skipping step."
print(msg)
logging.info(msg)
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed, end_time)
pytest.skip("Skipping " + step +
" because the input file does not exist.")
else:
msg = "Skipping running pipeline step " + step
print(msg)
logging.info(msg)
end_time = core_utils.get_stp_run_time_from_screenfile(
step, detector, output_directory)
if os.path.isfile(step_output_file):
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
step_completed = True
# add the running time for this step
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed, end_time)
return hdul, step_output_file, flattest_paths, flattest_switches, run_pytests
else:
step_completed = False
# add the running time for this step
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed, end_time)
pytest.skip("Test skipped because input file " +
step_output_file + " does not exist.")
def validate_flat_field(output_hdul):
hdu = output_hdul[0]
step_output_file, msa_shutter_conf, dflat_path, sflat_path, fflat_path = output_hdul[
2]
flattest_threshold_diff, save_flattest_plot, write_flattest_files = output_hdul[
3]
# show the figures
show_figs = False
log_msgs = None
# determine the testing mode
do_fs_flattest = False
if core_utils.check_FS_true(hdu) or core_utils.check_BOTS_true(hdu):
do_fs_flattest = True
elif 'bots' in step_output_file.lower() or 'fs' in step_output_file.lower(
):
do_fs_flattest = True
do_mos_flattest = False
if core_utils.check_MOS_true(hdu) or 'mos' in step_output_file.lower():
do_mos_flattest = True
do_ifu_flattest = False
if core_utils.check_IFU_true(hdu) or 'ifu' in step_output_file.lower():
do_ifu_flattest = True
# run the test
if do_fs_flattest:
median_diff, result_msg, log_msgs = flattest_fs.flattest(
step_output_file,
dflat_path=dflat_path,
sflat_path=sflat_path,
fflat_path=fflat_path,
writefile=write_flattest_files,
show_figs=show_figs,
save_figs=save_flattest_plot,
interpolated_flat=None,
threshold_diff=flattest_threshold_diff,
output_directory=None,
debug=False)
elif do_mos_flattest:
median_diff, result_msg, log_msgs = flattest_mos.flattest(
step_output_file,
dflat_path=dflat_path,
sflat_path=sflat_path,
fflat_path=fflat_path,
msa_shutter_conf=msa_shutter_conf,
writefile=write_flattest_files,
show_figs=show_figs,
save_figs=save_flattest_plot,
interpolated_flat=None,
threshold_diff=flattest_threshold_diff,
debug=False)
elif do_ifu_flattest:
median_diff, result_msg, log_msgs = flattest_ifu.flattest(
step_output_file,
dflat_path=dflat_path,
sflat_path=sflat_path,
fflat_path=fflat_path,
writefile=write_flattest_files,
mk_all_slices_plt=False,
show_figs=show_figs,
save_figs=save_flattest_plot,
interpolated_flat=None,
threshold_diff=flattest_threshold_diff,
debug=False)
else:
pytest.skip(
"Skipping pytest: The input fits file is not FS, MOS, or IFU. This tool does not yet include the "
"routine to verify this kind of file.")
if log_msgs is not None:
for msg in log_msgs:
logging.info(msg)
if median_diff == "skip":
logging.info(result_msg)
pytest.skip(result_msg)
else:
print(result_msg)
logging.info(result_msg)
return median_diff
def output_hdul(set_inandout_filenames, config):
# determine if the pipeline is to be run in full, per steps, or skipped
run_calwebb_spec3 = config.get("calwebb_spec3", "run_calwebb_spec3")
print("run_calwebb_spec3 = ", run_calwebb_spec3)
if run_calwebb_spec3 == "skip":
print(
'\n * PTT finished processing run_calwebb_spec3 is set to skip. \n'
)
pytest.exit(
"Finished processing file, run_calwebb_spec3 is set to skip in configuration file."
)
else:
run_calwebb_spec3 = bool(run_calwebb_spec3)
# get the general info
step, step_input_filename, output_file, in_file_suffix, outstep_file_suffix, True_steps_suffix_map = set_inandout_filenames
output_directory = config.get("calwebb_spec2_input_file",
"output_directory")
txt_name = os.path.join(output_directory, True_steps_suffix_map)
step_input_file = os.path.join(output_directory, step_input_filename)
step_output_file = os.path.join(output_directory, output_file)
mode_used = config.get("calwebb_spec2_input_file", "mode_used").lower()
# start the timer to compute the step running time of NPTT
nptt_start_time = time.time()
# determine which steps are to be run, if not run in full
run_pipe_step = config.getboolean("run_pipe_steps", step)
# determine which tests are to be run
master_background_completion_tests = config.getboolean(
"run_pytest", "_".join((step, "completion", "tests")))
master_background_reffile_tests = config.getboolean(
"run_pytest", "_".join((step, "reffile", "tests")))
master_background_validation_tests = config.getboolean(
"run_pytest", "_".join((step, "validation", "tests")))
run_pytests = [
master_background_completion_tests, master_background_reffile_tests,
master_background_validation_tests
]
# Get the detector used
detector = fits.getval(step_input_file, "DETECTOR", 0)
# get main header from input file
inhdu = core_utils.read_hdrfits(step_input_file,
info=False,
show_hdr=False)
# if run_calwebb_spec3 is True, calwebb_spec3 will be called, else individual steps will be ran
step_completed = False
end_time = '0.0'
# get the shutter configuration file for MOS data only
msa_shutter_conf = "No shutter configuration file will be used."
if core_utils.check_MOS_true(inhdu):
msa_shutter_conf = config.get("esa_intermediary_products",
"msa_conf_name")
# check if the configuration shutter file name is in the header of the fits file and if not add it
msametfl = fits.getval(step_input_file, "MSAMETFL", 0)
if os.path.basename(msa_shutter_conf) != msametfl:
msametfl = os.path.basename(msa_shutter_conf)
fits.setval(step_input_file, "MSAMETFL", 0, value=msametfl)
# copy the MSA shutter configuration file to this directory if not the working directory
if os.getcwd() != os.path.dirname(msa_shutter_conf):
# copy the MSA shutter configuration file into the pytest directory
print("Removing MSA config file from: ", os.getcwd())
subprocess.run(["cp", msa_shutter_conf, "."])
# check if processing an image, then set proper variables
if mode_used in ('image', 'confirm', 'taconfirm', 'wata', 'msata', 'bota',
'focus', 'mimf'):
run_calwebb_spec3 = True
imaging_mode = True
print('\n * Image processing stops after calwebb_spec2. ')
# end script for imaging case
if imaging_mode:
print('\n * NPTT finished processing imaging mode. \n')
pytest.exit(
"Imaging does not get processed through calwebb_spec3.")
# get the name of the configuration file and run the pipeline
calwebb_spec3_cfg = config.get("calwebb_spec3", "calwebb_spec3_cfg")
# copy the configuration file to create the pipeline log
if not os.path.isfile(os.path.join(output_directory, "stpipe-log.cfg")):
stpipelogcfg = calwebb_spec3_cfg.replace("calwebb_spec3.cfg",
"stpipe-log.cfg")
subprocess.run(["cp", stpipelogcfg, os.getcwd()])
# run the pipeline
if run_calwebb_spec3:
# Create the logfile for NPTT, but remove the previous log file
npttcalspec3_log = os.path.join(output_directory,
'NPTT_calspec3_' + detector + '.log')
if os.path.isfile(npttcalspec3_log):
os.remove(npttcalspec3_log)
print(
"Spec3 screen information output from NPTT will be logged in file: ",
npttcalspec3_log)
for handler in logging.root.handlers[:]:
logging.root.removeHandler(handler)
logging.basicConfig(filename=npttcalspec3_log, level=logging.INFO)
logging.info(pipeline_version)
run_calwebb_spec3_msg = " *** Will run pipeline in full ... "
print(run_calwebb_spec3_msg)
logging.info(run_calwebb_spec3_msg)
# create the map
txt_name = "spec3_full_run_map_" + detector + ".txt"
if os.path.isfile(txt_name):
os.remove(txt_name)
master_background_utils.create_completed_steps_txtfile(
txt_name, step_input_file)
# start the timer to compute the step running time of NPTT
core_utils.start_end_PTT_time(txt_name,
start_time=nptt_start_time,
end_time=None)
if mode_used == "bots":
calwebb_spec3_cfg = calwebb_spec3_cfg.replace(
"calwebb_spec3.cfg", "calwebb_tso-spec3.cfg")
print(
'\nUsing the following configuration file to run TSO pipeline:'
)
else:
print(
'\nUsing the following configuration file to run spectroscopic pipeline:'
)
print(calwebb_spec3_cfg, '\n')
# start the timer to compute the step running time
start_time = time.time()
# run the pipeline
print('Running pipeline... \n')
Spec3Pipeline.call(step_input_file, config_file=calwebb_spec3_cfg)
# end the timer to compute calwebb_spec3 running time
end_time = repr(time.time() - start_time) # this is in seconds
calspec3_time = " * Pipeline took " + end_time + " seconds to finish.\n"
print(calspec3_time)
logging.info(calspec3_time)
# add the detector string to the name of the files and move them to the working directory
core_utils.add_detector2filename(output_directory, step_input_file)
# state name of the final spec3 _cal file
if "spec2" in step_input_file:
final_output_name = step_input_file.replace("spec2", "spec3")
else:
final_output_name = "final_output_spec3_" + detector + "_cal.fits"
final_output_name_msg = "\nThe final pipeline product was saved in: " + final_output_name
print(final_output_name_msg)
logging.info(final_output_name_msg)
# read the assign wcs fits file
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
# scihdul = core_utils.read_hdrfits(step_output_file, info=False, show_hdr=False, ext=1)
if core_utils.check_MOS_true(inhdu):
if os.getcwd() != os.path.dirname(msa_shutter_conf):
# remove the copy of the MSA shutter configuration file
print("Removing MSA config file from: ", os.getcwd())
subprocess.run(["rm", msametfl])
# rename and move the pipeline log file
calspec3_pipelog = "calspec3_pipeline_" + detector + ".log"
try:
path_where_pipeline_was_run = os.getcwd()
logfile = glob(path_where_pipeline_was_run + "/pipeline.log")[0]
print(logfile)
os.rename(logfile, os.path.join(output_directory,
calspec3_pipelog))
except IndexError:
print(
"\nWARNING: Something went wrong. Could not find a pipeline.log file \n"
)
# make sure we are able to find calspec3_pipelog either in the calwebb_spec3 directory or in the working dir
if not os.path.isfile(calspec3_pipelog):
calspec3_pipelog = os.path.join(output_directory, calspec3_pipelog)
# add the running time for all steps
step_running_times = core_utils.calculate_step_run_time(
calspec3_pipelog)
end_time_list = []
for stp in core_utils.step_string_dict:
if stp in step_running_times:
step_completed = True
step_time = step_running_times[stp]["run_time"]
out_suffix = core_utils.step_string_dict[stp]["suffix"]
core_utils.add_completed_steps(txt_name, stp, out_suffix,
step_completed, step_time)
end_time_list.append(step_time)
# print total running time in the text file and move it to the indicated directory
string2print = "pipeline_total_time"
if float(end_time) <= sum(end_time_list):
tot_time = repr(sum(end_time_list))
else:
tot_time = end_time
master_background_utils.print_time2file(txt_name, tot_time,
string2print)
nptt_runtimes_msg = "Pipeline and NPTT run times written in file: " + os.path.basename(
txt_name) + " in working directory. \n"
print(nptt_runtimes_msg)
logging.info(nptt_runtimes_msg)
# move the final reporting text files to the working directory
if os.getcwd() != output_directory:
core_utils.move_txt_files_2workdir(config, detector)
return hdul, step_output_file, msa_shutter_conf, run_pytests, mode_used
else:
# create the map but remove a previous one if it exists
if os.path.isfile(txt_name):
os.remove(txt_name)
master_background_utils.create_completed_steps_txtfile(
txt_name, step_input_file)
# start the timer to compute the step running time of NPTT
core_utils.start_end_PTT_time(txt_name,
start_time=nptt_start_time,
end_time=None)
msg = "\n Pipeline and NPTT run times will be written in file: " + os.path.basename(
txt_name) + " in working directory. \n"
print(msg)
logging.info(msg)
if run_pipe_step:
# Create the logfile for NPTT, but erase the previous one if it exists
npttcalspec3_log = os.path.join(
output_directory,
'NPTT_calspec3_' + detector + '_' + step + '.log')
if os.path.isfile(npttcalspec3_log):
os.remove(npttcalspec3_log)
print(
"Output information on screen from NPTT will be logged in file: ",
npttcalspec3_log)
for handler in logging.root.handlers[:]:
logging.root.removeHandler(handler)
logging.basicConfig(filename=npttcalspec3_log, level=logging.INFO)
logging.info(pipeline_version)
# check that previous pipeline steps were run up to this point
core_utils.check_completed_steps(step, step_input_file)
if os.path.isfile(step_input_file):
msg = " *** Step " + step + " set to True"
print(msg)
logging.info(msg)
stp = MasterBackgroundStep()
# get the right configuration files to run the step
local_pipe_cfg_path = config.get("calwebb_spec2_input_file",
"local_pipe_cfg_path")
# start the timer to compute the step running time
print("running pipeline...")
start_time = time.time()
if local_pipe_cfg_path == "pipe_source_tree_code":
result = stp.call(step_input_file)
else:
result = stp.call(step_input_file,
config_file=local_pipe_cfg_path +
'/master_background.cfg')
result.save(step_output_file)
# end the timer to compute the step running time
end_time = repr(time.time() - start_time) # this is in seconds
msg = "Step " + step + " took " + end_time + " seconds to finish"
print(msg)
logging.info(msg)
if core_utils.check_MOS_true(inhdu):
# remove the copy of the MSA shutter configuration file
if os.getcwd() != os.path.dirname(msa_shutter_conf):
print("Removing MSA config file from: ", os.getcwd())
subprocess.run(["rm", msametfl])
# rename and move the pipeline log file
pipelog = "pipeline_" + detector + ".log"
try:
calspec3_pipelog = "calspec3_pipeline_" + step + "_" + detector + ".log"
pytest_workdir = TESTSDIR
logfile = glob(pytest_workdir + "/" + pipelog)[0]
os.rename(logfile,
os.path.join(output_directory, calspec3_pipelog))
except IndexError:
print(
"\n* WARNING: Something went wrong. Could not find a ",
pipelog, " file \n")
else:
msg = "Skipping step. Input file " + step_input_file + " does not exit."
print(msg)
logging.info(msg)
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed, end_time)
pytest.skip("Skipping " + step +
" because the input file does not exist.")
else:
print("Skipping running pipeline step ", step)
# add the running time for this step
end_time = core_utils.get_stp_run_time_from_screenfile(
step, detector, output_directory)
if os.path.isfile(step_output_file):
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
step_completed = True
# add the running time for this step
core_utils.add_completed_steps(txt_name, step, outstep_file_suffix,
step_completed, end_time)
return hdul, step_output_file, msa_shutter_conf, run_pytests, mode_used
else:
step_completed = False
# add the running time for this step
core_utils.add_completed_steps(txt_name, step, outstep_file_suffix,
step_completed, end_time)
pytest.skip("Test skipped because input file " + step_output_file +
" does not exist.")
def validate_pathloss(output_hdul):
hdu = output_hdul[0]
step_input_filename = output_hdul[2]
comparison_filename = output_hdul[1]
threshold_diff, save_figs, writefile = output_hdul[4]
# other default variables
show_figs = False
log_msgs = None
debug = False
# determine the type of source
srouce_type = fits.getval(comparison_filename, "SRCTYPE", "SCI", 1)
msg = "Source type is: " + srouce_type
print(msg)
logging.info(msg)
# get the corresponding reference file
reffile = hdu["R_PTHLOS"].replace("crds://", "")
# download the file if necessary
if not os.path.isfile(reffile):
reffile_url = "https://jwst-crds.stsci.edu/unchecked_get/references/jwst/" + reffile
urllib.request.urlretrieve(reffile_url, reffile)
if core_utils.check_FS_true(hdu) or core_utils.check_BOTS_true(hdu):
if "point" in srouce_type.lower():
median_diff, result_msg, log_msgs = pathloss_fs_ps.pathtest(
step_input_filename,
reffile,
comparison_filename,
writefile=writefile,
show_figs=show_figs,
save_figs=save_figs,
threshold_diff=threshold_diff,
debug=debug)
elif "extend" in srouce_type.lower():
median_diff, result_msg, log_msgs = pathloss_fs_uni.pathtest(
step_input_filename,
reffile,
comparison_filename,
writefile=writefile,
show_figs=show_figs,
save_figs=save_figs,
threshold_diff=threshold_diff,
debug=debug)
elif core_utils.check_MOS_true(hdu):
if "point" in srouce_type.lower():
median_diff, result_msg, log_msgs = pathloss_mos_ps.pathtest(
step_input_filename,
reffile,
comparison_filename,
writefile=writefile,
show_figs=show_figs,
save_figs=save_figs,
threshold_diff=threshold_diff,
debug=debug)
elif "extend" in srouce_type.lower():
median_diff, result_msg, log_msgs = pathloss_mos_uni.pathtest(
step_input_filename,
reffile,
comparison_filename,
writefile=writefile,
show_figs=show_figs,
save_figs=save_figs,
threshold_diff=threshold_diff,
debug=debug)
elif core_utils.check_IFU_true(hdu):
if "point" in srouce_type.lower():
median_diff, result_msg, log_msgs = pathloss_ifu_ps.pathtest(
step_input_filename,
reffile,
comparison_filename,
writefile=writefile,
show_figs=show_figs,
save_figs=save_figs,
threshold_diff=threshold_diff,
debug=debug)
elif "extend" in srouce_type.lower():
median_diff, result_msg, log_msgs = pathloss_ifu_uni.pathtest(
step_input_filename,
reffile,
comparison_filename,
writefile=writefile,
show_figs=show_figs,
save_figs=save_figs,
threshold_diff=threshold_diff,
debug=debug)
else:
pytest.skip(
"Skipping pytest: The input fits file is not FS, MOS, or IFU. This tool does not yet include the "
"routine to verify this kind of file.")
if log_msgs is not None:
for msg in log_msgs:
logging.info(msg)
if median_diff == "skip":
logging.info(result_msg)
pytest.skip(result_msg)
else:
print(result_msg)
logging.info(result_msg)
return median_diff
def output_hdul(set_inandout_filenames, config):
# determine if the pipeline is to be run in full, per steps, or skipped
run_calwebb_spec2 = config.get("run_calwebb_spec2_in_full",
"run_calwebb_spec2")
if run_calwebb_spec2 == "skip":
print(
'\n * PTT finished processing run_calwebb_spec2 is set to skip. \n'
)
pytest.exit(
"Skipping pipeline run and tests for spec2, run_calwebb_spec2 is set to skip in PTT_config file."
)
elif "T" in run_calwebb_spec2:
run_calwebb_spec2 = True
else:
run_calwebb_spec2 = False
# get the general info
set_inandout_filenames_info = core_utils.read_info4outputhdul(
config, set_inandout_filenames)
step, txt_name, step_input_file, step_output_file, outstep_file_suffix = set_inandout_filenames_info
# determine if the spec2 pipeline should be run or not - obtained from CRDS repo:
# https://github.com/spacetelescope/crds/blob/7fc216e73bd81d334b5d98d165be2caa3b6175e1/crds/jwst/jwst_system_crdscfg_b7.5.yaml
exp_type = fits.getval(step_input_file, 'EXP_TYPE')
allowed_exp_type_values = [
'NRS_AUTOFLAT', 'NRS_AUTOWAVE', 'NRS_BRIGHTOBJ', 'NRS_CONFIRM',
'NRS_FIXEDSLIT', 'NRS_FOCUS', 'NRS_IFU', 'NRS_IMAGE', 'NRS_LAMP',
'NRS_MIMF', 'NRS_MSASPEC', 'NRS_MSATA', 'NRS_TACONFIRM', 'NRS_TACQ',
'NRS_TASLIT', 'NRS_WATA'
]
if exp_type not in allowed_exp_type_values:
print(
"EXP_TYPE=", exp_type,
" is not in exposure types expected for running the stage 2 pipeline: ",
allowed_exp_type_values)
pytest.exit(
"The EXP_TYPE value of this data is not expected to run in the stage 2 pipeline."
)
# start the timer to compute the step running time of PTT
PTT_start_time = time.time()
# check if the filter is to be changed
change_filter_opaque = config.getboolean("calwebb_spec2_input_file",
"change_filter_opaque")
if change_filter_opaque:
_, step_input_file = change_filter_opaque2science.change_filter_opaque(
step_input_file)
change_filter_opaque_msg = " * With FILTER=OPAQUE, the calwebb_spec2 will run up to the extract_2d step. " \
"Further steps will be skipped. \n"
print(change_filter_opaque_msg)
# determine which steps are to be run, if not run in full
run_pipe_step = config.getboolean("run_pipe_steps", step)
# determine which tests are to be run
assign_wcs_completion_tests = config.getboolean(
"run_pytest", "_".join((step, "completion", "tests")))
assign_wcs_reffile_tests = config.getboolean(
"run_pytest", "_".join((step, "reffile", "tests")))
assign_wcs_validation_tests = config.getboolean(
"run_pytest", "_".join((step, "validation", "tests")))
run_pytests = [
assign_wcs_completion_tests, assign_wcs_reffile_tests,
assign_wcs_validation_tests
]
# get other relevant info from PTT config file
compare_assign_wcs_and_extract_2d_with_esa = config.getboolean(
"benchmark_intermediary_products",
"compare_assign_wcs_and_extract_2d_with_esa")
esa_files_path = config.get("benchmark_intermediary_products",
"esa_files_path")
data_directory = config.get("calwebb_spec2_input_file", "data_directory")
truth_file = os.path.join(
data_directory,
config.get("benchmark_intermediary_products", "truth_file_assign_wcs"))
if compare_assign_wcs_and_extract_2d_with_esa:
truth_file = esa_files_path
print("Will use this 'truth' file to compare result of assign_wcs: ")
print(truth_file)
wcs_threshold_diff = config.get("additional_arguments",
"wcs_threshold_diff")
save_wcs_plots = config.getboolean("additional_arguments",
"save_wcs_plots")
output_directory = config.get("calwebb_spec2_input_file",
"output_directory")
# Get the detector used
detector = fits.getval(step_input_file, "DETECTOR", 0)
# get main header from input file
inhdu = core_utils.read_hdrfits(step_input_file,
info=False,
show_hdr=False)
# if run_calwebb_spec2 is True calwebb_spec2 will be called, else individual steps will be ran
step_completed = False
end_time = '0.0'
# Check if data is IFU that the Image Model keyword is correct
mode_used = config.get("calwebb_spec2_input_file", "mode_used").lower()
if mode_used == "ifu":
DATAMODL = fits.getval(step_input_file, "DATAMODL", 0)
if DATAMODL != "IFUImageModel":
fits.setval(step_input_file, "DATAMODL", 0, value="IFUImageModel")
print("DATAMODL keyword changed to IFUImageModel.")
# get the shutter configuration file for MOS data only
msa_shutter_conf = "No shutter configuration file will be used."
if core_utils.check_MOS_true(inhdu):
msa_shutter_conf = config.get("benchmark_intermediary_products",
"msa_conf_name")
# check if the configuration shutter file name is in the header of the fits file and if not add it
msametfl = fits.getval(step_input_file, "MSAMETFL", 0)
if os.path.basename(msa_shutter_conf) != msametfl:
msametfl = os.path.basename(msa_shutter_conf)
fits.setval(step_input_file, "MSAMETFL", 0, value=msametfl)
# check if processing an image, then set proper variables
imaging_mode = False
if mode_used in ('image', 'confirm', 'taconfirm', 'wata', 'msata', 'bota',
'focus', 'mimf'):
run_calwebb_spec2 = True
imaging_mode = True
print(
'\n * Image processing will only be run in full with PTT. All intermediary products will be saved.'
)
print(
' -> For now, all pytests will be skipped since there are now image-specific routines yet. \n'
)
# TODO: add imaging tests
# get the name of the configuration file and run the pipeline
calwebb_spec2_cfg = config.get("run_calwebb_spec2_in_full",
"calwebb_spec2_cfg")
# copy the configuration file to create the pipeline log
pipelog = assign_wcs_utils.set_pipe_log(calwebb_spec2_cfg, detector)
# run the pipeline
if run_calwebb_spec2:
# Create the logfile for PTT, but remove the previous log file
PTTcalspec2_log = os.path.join(output_directory,
'PTT_calspec2_' + detector + '.log')
if imaging_mode:
PTTcalspec2_log = PTTcalspec2_log.replace('calspec2', 'calimage2')
if os.path.isfile(PTTcalspec2_log):
os.remove(PTTcalspec2_log)
print(
"Information outputed to screen from PTT will be logged in file: ",
PTTcalspec2_log)
for handler in logging.root.handlers[:]:
logging.root.removeHandler(handler)
logging.basicConfig(filename=PTTcalspec2_log, level=logging.INFO)
logging.info(pipeline_version)
if change_filter_opaque:
logging.info(change_filter_opaque_msg)
run_calwebb_spec2_msg = " *** Will run pipeline in full ... "
print(run_calwebb_spec2_msg)
logging.info(run_calwebb_spec2_msg)
# create the map
txt_name = "spec2_full_run_map_" + detector + ".txt"
if os.path.isfile(txt_name):
os.remove(txt_name)
assign_wcs_utils.create_completed_steps_txtfile(
txt_name, step_input_file)
# start the timer to compute the step running time of PTT
core_utils.start_end_PTT_time(txt_name,
start_time=PTT_start_time,
end_time=None)
if mode_used == "bots":
calwebb_spec2_cfg = calwebb_spec2_cfg.replace(
"calwebb_spec2.cfg", "calwebb_tso-spec2.cfg")
print(
'\nUsing the following configuration file to run TSO pipeline:'
)
print(calwebb_spec2_cfg, '\n')
if imaging_mode:
calwebb_image2_cfg = calwebb_spec2_cfg.replace(
"calwebb_spec2.cfg", "calwebb_image2.cfg")
print(
'\nUsing the following configuration file to run imaging pipeline:'
)
print(calwebb_image2_cfg, '\n')
else:
print(
'\nUsing the following configuration file to run spectroscopic pipeline:'
)
print(calwebb_spec2_cfg, '\n')
input_file = config.get("calwebb_spec2_input_file", "input_file")
if "_uncal_rate" in input_file:
input_file = input_file.replace("_uncal_rate", "")
if "_uncal" in input_file:
input_file = input_file.replace("_uncal", "")
# final_output_name = "final_output_spec2_"+detector+"_cal.fits"
if detector.lower() not in input_file.lower():
input_file = input_file.replace(".fits", "_" + detector + ".fits")
final_output_name = input_file.replace(".fits", "_cal.fits")
if '_rate' in final_output_name:
final_output_name = final_output_name.replace('_rate', '')
if core_utils.check_MOS_true(inhdu):
# copy the MSA shutter configuration file into the pytest directory
subprocess.run(["cp", msa_shutter_conf, "."])
# start the timer to compute the step running time
start_time = time.time()
# run the pipeline
print('Running pipeline... \n')
try:
if not imaging_mode:
Spec2Pipeline.call(
step_input_file,
config_file=calwebb_spec2_cfg) # , logcfg=stpipelogcfg)
else:
Image2Pipeline.call(step_input_file,
config_file=calwebb_image2_cfg)
except:
pytest.exit(
"\n * The pipeline crashed, probably due to no open slits fall on detector"
)
"""
# For the moment, the pipeline is using the wrong reference file for slit 400A1, so the
# file needs to be re-processed with the right reference file
if core_utils.check_FS_true(inhdu):
print("\n * WARNING: For the moment, the wrong reference file is being used for "
"processing slit 400A1. The file will be re-processed ")
# print(" $ jwst.pathloss.PathLossStep final_output_caldet1_NRS1_srctype.fits "
# "--override_pathloss=jwst-nirspec-a400.plrf.fits \n")
pathloss_400a1 = step_input_file.replace("srctype.fits", "pathloss_400A1.fits")
reffile_400a1 = "jwst-nirspec-a400.plrf.fits"
print("Re-processing slit with new reference file: ", reffile_400a1)
pl = PathLossStep()
pl.override_pathloss = reffile_400a1
pl.run(step_input_file)
subprocess.run(["mv", step_input_file.replace("srctype", "pathlossstep"), pathloss_400a1])
print("Saved pipeline re-processed file as: ", pathloss_400a1)
"""
# end the timer to compute calwebb_spec2 running time
end_time = repr(time.time() - start_time) # this is in seconds
calspec2_time = " * Pipeline took " + end_time + " seconds to finish.\n"
print(calspec2_time)
logging.info(calspec2_time)
# remove the copy of the MSA shutter configuration file
if core_utils.check_MOS_true(inhdu):
if TESTSDIR == os.path.dirname(msametfl):
print("Removing MSA config file from: ", TESTSDIR)
subprocess.run(["rm", msametfl])
# add the detector string to the name of the files and move them to the working directory
core_utils.add_detector2filename(output_directory, step_input_file)
final_output_name_msg = "\nThe final pipeline product was saved as: " + final_output_name
print(final_output_name_msg)
logging.info(final_output_name_msg)
# read the assign wcs fits file
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
# scihdul = core_utils.read_hdrfits(step_output_file, info=False, show_hdr=False, ext=1)
# rename and move the pipeline log file
try:
calspec2_pipelog = "calspec2_pipeline_" + detector + ".log"
if imaging_mode:
calspec2_pipelog = calspec2_pipelog.replace(
'calspec2', 'calimage2')
path_where_pipeline_was_run = os.getcwd()
logfile = glob(path_where_pipeline_was_run + "/" + pipelog)[0]
print(logfile)
os.rename(logfile, os.path.join(output_directory,
calspec2_pipelog))
except IndexError:
print("\nWARNING: Something went wrong. Could not find a ",
pipelog, " file \n")
# make sure we are able to find calspec2_pipelog either in the calwebb_spec2 directory or in the working dir
if not os.path.isfile(calspec2_pipelog):
calspec2_pipelog = os.path.join(output_directory, calspec2_pipelog)
# add the running time for all steps
step_running_times = core_utils.calculate_step_run_time(
calspec2_pipelog)
end_time_list = []
for stp in core_utils.step_string_dict:
if stp in step_running_times:
step_completed = True
step_time = step_running_times[stp]["run_time"]
out_suffix = core_utils.step_string_dict[stp]["suffix"]
core_utils.add_completed_steps(txt_name, stp, out_suffix,
step_completed, step_time)
end_time_list.append(step_time)
# print total running time in the text file and move it to the indicated directory
string2print = "pipeline_total_time"
if float(end_time) <= sum(end_time_list):
tot_time = repr(sum(end_time_list))
else:
tot_time = end_time
assign_wcs_utils.print_time2file(txt_name, tot_time, string2print)
PTT_runtimes_msg = "Pipeline and PTT run times written in file: " + os.path.basename(
txt_name) + " in working directory. \n"
print(PTT_runtimes_msg)
logging.info(PTT_runtimes_msg)
# move the final reporting text files to the working directory
core_utils.move_txt_files_2workdir(config, detector)
# rename the _cal file
last_suffix = os.path.basename(step_output_file).split(
".fits")[0].split("_")[-1]
cal_file = step_output_file.replace(last_suffix, "_cal")
if os.path.isfile(cal_file):
if "caldet1" in cal_file:
spec2_cal_file = cal_file.replace("caldet1", "spec2")
os.rename(cal_file, spec2_cal_file)
# end script for imaging case
if imaging_mode:
print('\nPTT finished processing imaging mode. \n')
pytest.exit(
"Skipping pytests for now because they need to be written for imaging mode."
)
return hdul, step_output_file, msa_shutter_conf, truth_file, wcs_threshold_diff, save_wcs_plots, \
run_pytests, mode_used, compare_assign_wcs_and_extract_2d_with_esa
else:
# create the map but remove a previous one if it exists
if os.path.isfile(txt_name):
os.remove(txt_name)
assign_wcs_utils.create_completed_steps_txtfile(
txt_name, step_input_file)
# start the timer to compute the step running time of PTT
core_utils.start_end_PTT_time(txt_name,
start_time=PTT_start_time,
end_time=None)
msg = "\n Pipeline and PTT run times will be written in file: " + os.path.basename(
txt_name) + " in working directory. \n"
print(msg)
logging.info(msg)
if run_pipe_step:
# Create the logfile for PTT, but erase the previous one if it exists
PTTcalspec2_log = os.path.join(
output_directory,
'PTT_calspec2_' + detector + '_' + step + '.log')
if imaging_mode:
PTTcalspec2_log = PTTcalspec2_log.replace(
'calspec2', 'calimage2')
if os.path.isfile(PTTcalspec2_log):
os.remove(PTTcalspec2_log)
print(
"Information outputed to screen from PTT will be logged in file: ",
PTTcalspec2_log)
for handler in logging.root.handlers[:]:
logging.root.removeHandler(handler)
logging.basicConfig(filename=PTTcalspec2_log, level=logging.INFO)
logging.info(pipeline_version)
if change_filter_opaque:
logging.info(change_filter_opaque_msg)
# check that previous pipeline steps were run up to this point
core_utils.check_completed_steps(step, step_input_file)
if os.path.isfile(step_input_file):
msg = " *** Step " + step + " set to True"
print(msg)
logging.info(msg)
stp = AssignWcsStep()
if core_utils.check_MOS_true(inhdu):
if os.getcwd() != output_directory:
# copy the MSA shutter configuration file into the pytest directory
subprocess.run(["cp", msa_shutter_conf, "."])
# get the right configuration files to run the step
local_pipe_cfg_path = config.get("calwebb_spec2_input_file",
"local_pipe_cfg_path")
# start the timer to compute the step running time
print("running pipeline...")
start_time = time.time()
if local_pipe_cfg_path == "pipe_source_tree_code":
result = stp.call(step_input_file)
else:
result = stp.call(step_input_file,
config_file=local_pipe_cfg_path +
'/assign_wcs.cfg')
result.save(step_output_file)
# end the timer to compute the step running time
end_time = repr(time.time() - start_time) # this is in seconds
msg = "Step " + step + " took " + end_time + " seconds to finish"
print(msg)
logging.info(msg)
if core_utils.check_MOS_true(inhdu):
if TESTSDIR == os.path.dirname(msametfl):
# remove the copy of the MSA shutter configuration file
print("Removing MSA config file from: ", TESTSDIR)
subprocess.run(["rm", msametfl])
# rename and move the pipeline log file
try:
calspec2_pipelog = "calspec2_pipeline_" + step + "_" + detector + ".log"
if imaging_mode:
calspec2_pipelog = calspec2_pipelog.replace(
'calspec2', 'calimage2')
pytest_workdir = TESTSDIR
logfile = glob(pytest_workdir + "/" + pipelog)[0]
os.rename(logfile,
os.path.join(output_directory, calspec2_pipelog))
except IndexError:
print(
"\n* WARNING: Something went wrong. Could not find a ",
pipelog, " file \n")
else:
msg = "Skipping step. Input file " + step_input_file + " does not exit."
print(msg)
logging.info(msg)
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed, end_time)
pytest.skip("Skipping " + step +
" because the input file does not exist.")
else:
print("Skipping running pipeline step ", step)
# add the running time for this step
end_time = core_utils.get_stp_run_time_from_screenfile(
step, detector, output_directory)
if os.path.isfile(step_output_file):
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
step_completed = True
# add the running time for this step
core_utils.add_completed_steps(txt_name, step, outstep_file_suffix,
step_completed, end_time)
return hdul, step_output_file, msa_shutter_conf, truth_file, wcs_threshold_diff, save_wcs_plots, \
run_pytests, mode_used, compare_assign_wcs_and_extract_2d_with_esa
else:
step_completed = False
# add the running time for this step
core_utils.add_completed_steps(txt_name, step, outstep_file_suffix,
step_completed, end_time)
pytest.skip("Test skipped because input file " + step_output_file +
" does not exist.")
def validate_wcs(output_hdul):
# get the input information for the wcs routine
hdu = output_hdul[0]
infile_name = output_hdul[1]
msa_conf_name = output_hdul[2]
truth_file = output_hdul[3]
mode_used = output_hdul[7]
compare_assign_wcs_and_extract_2d_with_esa = output_hdul[8]
esa_files_path = None
if compare_assign_wcs_and_extract_2d_with_esa:
esa_files_path = output_hdul[3]
truth_file = None
# define the threshold difference between the pipeline output and the truth files for the pytest to pass or fail
threshold_diff = float(output_hdul[4])
# save the output plots
save_wcs_plots = output_hdul[5]
# show the figures
show_figs = False
msg = "\n Performing WCS validation test... "
print(msg)
logging.info(msg)
log_msgs = None
if core_utils.check_FS_true(hdu):
result, log_msgs = compare_wcs_fs.compare_wcs(
infile_name,
truth_file=truth_file,
esa_files_path=esa_files_path,
show_figs=show_figs,
save_figs=save_wcs_plots,
threshold_diff=threshold_diff,
raw_data_root_file=None,
output_directory=None,
debug=False)
elif core_utils.check_MOS_true(hdu) and mode_used != "MOS_sim":
result, log_msgs = compare_wcs_mos.compare_wcs(
infile_name,
msa_conf_name=msa_conf_name,
truth_file=truth_file,
esa_files_path=esa_files_path,
show_figs=show_figs,
save_figs=save_wcs_plots,
threshold_diff=threshold_diff,
raw_data_root_file=None,
output_directory=None,
debug=False)
elif core_utils.check_IFU_true(hdu):
result, log_msgs = compare_wcs_ifu.compare_wcs(
infile_name,
truth_file=truth_file,
esa_files_path=esa_files_path,
show_figs=show_figs,
save_figs=save_wcs_plots,
threshold_diff=threshold_diff,
raw_data_root_file=None,
output_directory=None,
debug=False)
else:
# We do not have truth data to compare with for BOTS
pytest.skip("Skipping pytest: The fits file is not FS, MOS, or IFU.")
if log_msgs is not None:
for msg in log_msgs:
logging.info(msg)
if "skip" in result:
pytest.skip("Skipping assign_wcs pytest.")
elif "PASS" in result:
result = True
else:
result = False
return result
