def output_hdul(set_inandout_filenames, config):
# determine if the pipeline is to be run in full, per steps, or skipped
run_calwebb_spec2 = config.get("run_calwebb_spec2_in_full",
"run_calwebb_spec2")
if run_calwebb_spec2 == "skip":
print(
'\n * PTT finished processing run_calwebb_spec2 is set to skip. \n'
)
pytest.exit(
"Skipping pipeline run and tests for spec2, run_calwebb_spec2 is set to skip in PTT_config file."
)
elif "T" in run_calwebb_spec2:
run_calwebb_spec2 = True
else:
run_calwebb_spec2 = False
# get the general info
set_inandout_filenames_info = core_utils.read_info4outputhdul(
config, set_inandout_filenames)
step, txt_name, step_input_file, step_output_file, outstep_file_suffix = set_inandout_filenames_info
run_pipe_step = config.getboolean("run_pipe_steps", step)
# determine which tests are to be run
cube_build_completion_tests = config.getboolean(
"run_pytest", "_".join((step, "completion", "tests")))
#cube_build_reffile_tests = config.getboolean("run_pytest", "_".join((step, "reffile", "tests")))
#cube_build_validation_tests = config.getboolean("run_pytest", "_".join((step, "validation", "tests")))
run_pytests = [cube_build_completion_tests
] #, cube_build_reffile_tests, cube_build_validation_tests]
# Only run step if data is IFU
output_directory = config.get("calwebb_spec2_input_file",
"output_directory")
initial_input_file = config.get("calwebb_spec2_input_file", "input_file")
initial_input_file = os.path.join(output_directory, initial_input_file)
if os.path.isfile(initial_input_file):
inhdu = core_utils.read_hdrfits(initial_input_file,
info=False,
show_hdr=False)
detector = fits.getval(initial_input_file, "DETECTOR", 0)
filt = fits.getval(initial_input_file, 'filter')
grat = fits.getval(initial_input_file, 'grating')
gratfilt = grat + "-" + filt + "_s3d"
else:
pytest.skip(
"Skipping " + step +
" because the initial input file given in PTT_config.cfg does not exist."
)
end_time = '0.0'
if core_utils.check_IFU_true(inhdu):
# if run_calwebb_spec2 is True calwebb_spec2 will be called, else individual steps will be ran
step_completed = False
# check if the filter is to be changed
change_filter_opaque = config.getboolean("calwebb_spec2_input_file",
"change_filter_opaque")
if change_filter_opaque:
is_filter_opaque, step_input_filename = change_filter_opaque2science.change_filter_opaque(
step_input_file, step=step)
if is_filter_opaque:
filter_opaque_msg = "With FILTER=OPAQUE, the calwebb_spec2 will run up to the extract_2d step. " \
"Cube build pytest now set to Skip."
print(filter_opaque_msg)
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed, end_time)
pytest.skip("Skipping " + step + " because FILTER=OPAQUE.")
if run_calwebb_spec2:
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
return hdul, step_output_file, run_pytests
else:
if run_pipe_step:
# Create the logfile for PTT, but erase the previous one if it exists
PTTcalspec2_log = os.path.join(
output_directory,
'PTT_calspec2_' + detector + '_' + step + '.log')
if os.path.isfile(PTTcalspec2_log):
os.remove(PTTcalspec2_log)
print(
"Information outputed to screen from PTT will be logged in file: ",
PTTcalspec2_log)
for handler in logging.root.handlers[:]:
logging.root.removeHandler(handler)
logging.basicConfig(filename=PTTcalspec2_log,
level=logging.INFO)
# print pipeline version
import jwst
pipeline_version = "\n *** Using jwst pipeline version: " + jwst.__version__ + " *** \n"
print(pipeline_version)
logging.info(pipeline_version)
if change_filter_opaque:
logging.info(filter_opaque_msg)
if os.path.isfile(step_input_file):
msg = " *** Step " + step + " set to True"
print(msg)
logging.info(msg)
stp = CubeBuildStep()
# check that previous pipeline steps were run up to this point
core_utils.check_completed_steps(step, step_input_file)
# get the right configuration files to run the step
local_pipe_cfg_path = config.get(
"calwebb_spec2_input_file", "local_pipe_cfg_path")
# start the timer to compute the step running time
start_time = time.time()
if local_pipe_cfg_path == "pipe_source_tree_code":
result = stp.call(step_input_file)
else:
result = stp.call(step_input_file,
config_file=local_pipe_cfg_path +
'/cube_build.cfg')
result.save(step_output_file)
# end the timer to compute the step running time
end_time = repr(time.time() -
start_time) # this is in seconds
msg = "Step " + step + " took " + end_time + " seconds to finish"
print(msg)
logging.info(msg)
# determine the specific output of the cube step
specific_output_file = glob(
step_output_file.replace(
'cube.fits', (gratfilt + '*.fits').lower()))[0]
cube_suffix = specific_output_file.split(
'cube_build_')[-1].replace('.fits', '')
# record info
step_completed = True
hdul = core_utils.read_hdrfits(specific_output_file,
info=False,
show_hdr=False)
# rename and move the pipeline log file
pipelog = "pipeline_" + detector + ".log"
try:
calspec2_pilelog = "calspec2_pipeline_" + step + "_" + detector + ".log"
pytest_workdir = TESTSDIR
logfile = glob(pytest_workdir + "/" + pipelog)[0]
os.rename(
logfile,
os.path.join(output_directory, calspec2_pilelog))
except IndexError:
print(
"\n* WARNING: Something went wrong. Could not find a ",
pipelog, " file \n")
# add the running time for this step
core_utils.add_completed_steps(txt_name, step,
"_" + cube_suffix,
step_completed, end_time)
return hdul, step_output_file, run_pytests
else:
msg = " The input file does not exist. Skipping step."
print(msg)
logging.info(msg)
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed, end_time)
pytest.skip("Skipping " + step +
" because the input file does not exist.")
else:
msg = "Skipping running pipeline step " + step
print(msg)
logging.info(msg)
end_time = core_utils.get_stp_run_time_from_screenfile(
step, detector, output_directory)
# record info
# specific cube step suffix
cube_suffix = "_s3d"
if os.path.isfile(step_output_file):
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
step_completed = True
# add the running time for this step
core_utils.add_completed_steps(txt_name, step, cube_suffix,
step_completed, end_time)
return hdul, step_output_file, run_pytests
else:
step_completed = False
# add the running time for this step
core_utils.add_completed_steps(txt_name, step, cube_suffix,
step_completed, end_time)
pytest.skip("Test skipped because input file " +
step_output_file + " does not exist.")
else:
pytest.skip("Skipping " + step + " because data is not IFU.")
def output_hdul(set_inandout_filenames, config):
# determine if the pipeline is to be run in full, per steps, or skipped
run_calwebb_spec2 = config.get("run_calwebb_spec2_in_full",
"run_calwebb_spec2")
if run_calwebb_spec2 == "skip":
print(
'\n * PTT finished processing run_calwebb_spec2 is set to skip. \n'
)
pytest.exit(
"Skipping pipeline run and tests for spec2, run_calwebb_spec2 is set to skip in PTT_config file."
)
elif "T" in run_calwebb_spec2:
run_calwebb_spec2 = True
else:
run_calwebb_spec2 = False
# get the general info
set_inandout_filenames_info = core_utils.read_info4outputhdul(
config, set_inandout_filenames)
step, txt_name, step_input_file, step_output_file, outstep_file_suffix = set_inandout_filenames_info
run_pipe_step = config.getboolean("run_pipe_steps", step)
# determine which tests are to be run
extract_1d_completion_tests = config.getboolean(
"run_pytest", "_".join((step, "completion", "tests")))
extract_1d_reffile_tests = config.getboolean(
"run_pytest", "_".join((step, "reffile", "tests")))
# extract_1d_validation_tests = config.getboolean("run_pytest", "_".join((step, "validation", "tests")))
run_pytests = [extract_1d_completion_tests,
extract_1d_reffile_tests] #, extract_1d_validation_tests]
# Make sure at this point the MSA shutter configuration file is removed from the calwebb_spec2_pytests directory
msa_shutter_conf = config.get("benchmark_intermediary_products",
"msa_conf_name")
msametfl = os.path.basename(msa_shutter_conf)
# remove the copy of the MSA shutter configuration file
inhdu = core_utils.read_hdrfits(step_input_file,
info=False,
show_hdr=False)
if core_utils.check_MOS_true(inhdu):
if TESTSDIR == os.path.dirname(msametfl):
print("Removing MSA config file from: ", TESTSDIR)
os.remove(msametfl)
# check if processing an image, then set proper variables
imaging_mode = False
mode_used = config.get("calwebb_spec2_input_file", "mode_used").lower()
if mode_used in ('image', 'confirm', 'taconfirm', 'wata', 'msata', 'bota',
'focus', 'mimf'):
run_calwebb_spec2 = True
imaging_mode = True
# if run_calwebb_spec2 is True calwebb_spec2 will be called, else individual steps will be ran
step_completed = False
end_time = '0.0'
# Get the detector used
output_directory = config.get("calwebb_spec2_input_file",
"output_directory")
initial_input_file = config.get("calwebb_spec2_input_file", "input_file")
initial_input_file = os.path.join(output_directory, initial_input_file)
if os.path.isfile(initial_input_file):
detector = fits.getval(initial_input_file, "DETECTOR", 0)
else:
pytest.skip(
"Skipping " + step +
" because the initial input file given in PTT_config.cfg does not exist."
)
# set the name of the pipeline log to be read for times and renamed at the end for book-keeping
pipelog = "pipeline_" + detector + ".log"
# check if the filter is to be changed
change_filter_opaque = config.getboolean("calwebb_spec2_input_file",
"change_filter_opaque")
if change_filter_opaque:
is_filter_opaque, step_input_filename = change_filter_opaque2science.change_filter_opaque(
step_input_file, step=step)
if is_filter_opaque:
filter_opaque_msg = "With FILTER=OPAQUE, the calwebb_spec2 will run up to the extract_2d step. " \
"Extract_1d pytest now set to Skip."
print(filter_opaque_msg)
core_utils.add_completed_steps(txt_name, step, outstep_file_suffix,
step_completed, end_time)
#core_utils.convert_html2pdf() # convert the html report into a pdf file
# move the final reporting files to the working directory
core_utils.move_txt_files_2workdir(config, detector)
pytest.skip("Skipping " + step + " because FILTER=OPAQUE.")
if run_calwebb_spec2:
# read the output header
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
# end the timer to compute the step running time of PTT
PTT_end_time = time.time()
core_utils.start_end_PTT_time(txt_name,
start_time=None,
end_time=PTT_end_time)
# move the final reporting files to the working directory
core_utils.move_txt_files_2workdir(config, detector)
return hdul, step_output_file, run_pytests
else:
if run_pipe_step:
# Create the logfile for PTT, but erase the previous one if it exists
PTTcalspec2_log = os.path.join(
output_directory,
'PTT_calspec2_' + detector + '_' + step + '.log')
if imaging_mode:
PTTcalspec2_log = PTTcalspec2_log.replace(
'calspec2', 'calimage2')
if os.path.isfile(PTTcalspec2_log):
os.remove(PTTcalspec2_log)
print(
"Information outputed to screen from PTT will be logged in file: ",
PTTcalspec2_log)
for handler in logging.root.handlers[:]:
logging.root.removeHandler(handler)
logging.basicConfig(filename=PTTcalspec2_log, level=logging.INFO)
# print pipeline version
import jwst
pipeline_version = "\n *** Using jwst pipeline version: " + jwst.__version__ + " *** \n"
print(pipeline_version)
logging.info(pipeline_version)
if change_filter_opaque:
logging.info(filter_opaque_msg)
if os.path.isfile(step_input_file):
msg = " *** Step " + step + " set to True"
print(msg)
logging.info(msg)
stp = Extract1dStep()
# check that previous pipeline steps were run up to this point
core_utils.check_completed_steps(step, step_input_file)
# get the right configuration files to run the step
local_pipe_cfg_path = config.get("calwebb_spec2_input_file",
"local_pipe_cfg_path")
# start the timer to compute the step running time
start_time = time.time()
if local_pipe_cfg_path == "pipe_source_tree_code":
result = stp.call(step_input_file)
else:
result = stp.call(step_input_file,
config_file=local_pipe_cfg_path +
'/extract_1d.cfg')
result.save(step_output_file)
# end the timer to compute the step running time
end_time = repr(time.time() - start_time) # this is in seconds
msg = "Step " + step + " took " + end_time + " seconds to finish"
print(msg)
logging.info(msg)
# rename and move the pipeline log file
calspec2_pilelog = "calspec2_pipeline_" + step + "_" + detector + ".log"
if imaging_mode:
calspec2_pilelog = calspec2_pilelog.replace(
'calspec2', 'calimage2')
os.rename(pipelog,
os.path.join(output_directory, calspec2_pilelog))
else:
msg = " The input file does not exist. Skipping step."
print(msg)
logging.info(msg)
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed, end_time)
#core_utils.convert_html2pdf() # convert the html report into a pdf file
# end the timer to compute the step running time of PTT
PTT_end_time = time.time()
core_utils.start_end_PTT_time(txt_name,
start_time=None,
end_time=PTT_end_time)
# move the final reporting files to the working directory
core_utils.move_txt_files_2workdir(config, detector)
# skip the test if input file does not exist
pytest.skip("Skipping " + step +
" because the input file does not exist.")
else:
msg = "Skipping running pipeline step " + step
print(msg)
logging.info(msg)
# get the running time for this step
end_time = core_utils.get_stp_run_time_from_screenfile(
step, detector, output_directory)
# add the running time for this step
if os.path.isfile(step_output_file):
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
step_completed = True
core_utils.add_completed_steps(txt_name, step, outstep_file_suffix,
step_completed, end_time)
else:
step_completed = False
core_utils.add_completed_steps(txt_name, step, outstep_file_suffix,
step_completed, end_time)
# move the final reporting files to the working directory
core_utils.move_txt_files_2workdir(config, detector)
pytest.skip("Test skipped because input file " + step_output_file +
" does not exist.")
# get the total running time and print it in the file
total_time = repr(core_utils.get_time_to_run_pipeline(txt_name))
total_time_min = repr(round(float(total_time) / 60.0, 2))
msg = "\n\n **** The total time for the pipeline to run was " + total_time + " seconds."
print(msg)
logging.info(msg)
line2write = "{:<20} {:<20} {:<20} {:<20}".format(
'', '', 'total_time ',
total_time + ' =' + total_time_min + 'min')
print(line2write)
logging.info(line2write)
with open(txt_name, "a") as tf:
tf.write(line2write + "\n")
# convert the html report into a pdf file
#core_utils.convert_html2pdf()
# end the timer to compute the step running time of PTT
PTT_end_time = time.time()
core_utils.start_end_PTT_time(txt_name,
start_time=None,
end_time=PTT_end_time)
# move the final reporting text files to the working directory
core_utils.move_txt_files_2workdir(config, detector)
# rename and move the PTT log file
try:
calspec2_pipelog = "calspec2_pipeline_" + step + "_" + detector + ".log"
if imaging_mode:
calspec2_pipelog = calspec2_pipelog.replace(
'calspec2', 'calimage2')
pytest_workdir = TESTSDIR
logfile = glob(pytest_workdir + "/" + pipelog)[0]
os.rename(logfile, os.path.join(output_directory,
calspec2_pipelog))
except IndexError:
print("\n* WARNING: Something went wrong. Could not find a ",
pipelog, " file \n")
return hdul, step_output_file, run_pytests
def output_hdul(set_inandout_filenames, config):
set_inandout_filenames_info = core_utils.read_info4outputhdul(
config, set_inandout_filenames)
step, txt_name, step_input_file, step_output_file, run_calwebb_spec2, outstep_file_suffix = set_inandout_filenames_info
run_pipe_step = config.getboolean("run_pipe_steps", step)
# determine which tests are to be run
photom_completion_tests = config.getboolean(
"run_pytest", "_".join((step, "completion", "tests")))
#photom_reffile_tests = config.getboolean("run_pytest", "_".join((step, "reffile", "tests")))
#photom_validation_tests = config.getboolean("run_pytest", "_".join((step, "validation", "tests")))
run_pytests = [photom_completion_tests
] #, photom_reffile_tests, photom_validation_tests]
# if run_calwebb_spec2 is True calwebb_spec2 will be called, else individual steps will be ran
step_completed = False
end_time = '0.0'
output_directory = config.get("calwebb_spec2_input_file",
"output_directory")
initial_input_file = config.get("calwebb_spec2_input_file", "input_file")
initial_input_file = os.path.join(output_directory, initial_input_file)
detector = fits.getval(initial_input_file, "DETECTOR", 0)
if not os.path.isfile(initial_input_file):
pytest.skip(
"Skipping " + step +
" because the initial input file given in PTT_config.cfg does not exist."
)
# check if the filter is to be changed
change_filter_opaque = config.getboolean("calwebb_spec2_input_file",
"change_filter_opaque")
if change_filter_opaque:
is_filter_opaque, step_input_filename = change_filter_opaque2science.change_filter_opaque(
step_input_file, step=step)
if is_filter_opaque:
filter_opaque_msg = "With FILTER=OPAQUE, the calwebb_spec2 will run up to the extract_2d step. Photometry pytest now set to Skip."
print(filter_opaque_msg)
core_utils.add_completed_steps(txt_name, step, outstep_file_suffix,
step_completed, end_time)
pytest.skip("Skipping " + step + " because FILTER=OPAQUE.")
if run_calwebb_spec2:
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
return hdul, step_output_file, run_pytests
else:
if run_pipe_step:
# Create the logfile for PTT, but erase the previous one if it exists
PTTcalspec2_log = os.path.join(
output_directory,
'PTT_calspec2_' + detector + '_' + step + '.log')
if os.path.isfile(PTTcalspec2_log):
os.remove(PTTcalspec2_log)
print(
"Information outputed to screen from PTT will be logged in file: ",
PTTcalspec2_log)
for handler in logging.root.handlers[:]:
logging.root.removeHandler(handler)
logging.basicConfig(filename=PTTcalspec2_log, level=logging.INFO)
# print pipeline version
import jwst
pipeline_version = "\n *** Using jwst pipeline version: " + jwst.__version__ + " *** \n"
print(pipeline_version)
logging.info(pipeline_version)
if change_filter_opaque:
logging.info(filter_opaque_msg)
if os.path.isfile(step_input_file):
msg = " *** Step " + step + " set to True"
print(msg)
logging.info(msg)
stp = PhotomStep()
# check that previous pipeline steps were run up to this point
core_utils.check_completed_steps(step, step_input_file)
# get the right configuration files to run the step
local_pipe_cfg_path = config.get("calwebb_spec2_input_file",
"local_pipe_cfg_path")
# start the timer to compute the step running time
start_time = time.time()
if local_pipe_cfg_path == "pipe_source_tree_code":
result = stp.call(step_input_file)
else:
result = stp.call(step_input_file,
config_file=local_pipe_cfg_path +
'/photom.cfg')
result.save(step_output_file)
# end the timer to compute the step running time
end_time = repr(time.time() - start_time) # this is in seconds
msg = "Step " + step + " took " + end_time + " seconds to finish"
print(msg)
logging.info(msg)
step_completed = True
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
# rename and move the pipeline log file
try:
calspec2_pilelog = "calspec2_pipeline_" + step + "_" + detector + ".log"
pytest_workdir = TESTSDIR
logfile = glob(pytest_workdir + "/pipeline.log")[0]
os.rename(logfile,
os.path.join(output_directory, calspec2_pilelog))
except:
IndexError
# add the running time for this step
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed, end_time)
return hdul, step_output_file, run_pytests
else:
msg = "Skipping running pipeline step " + step
print(msg)
logging.info(msg)
end_time = core_utils.get_stp_run_time_from_screenfile(
step, detector, output_directory)
if os.path.isfile(step_output_file):
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
step_completed = True
# add the running time for this step
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed, end_time)
return hdul, step_output_file, run_pytests
else:
step_completed = False
# add the running time for this step
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed, end_time)
pytest.skip("Test skipped because input file " +
step_output_file + " does not exist.")
else:
msg = " The input file does not exist. Skipping step."
print(msg)
logging.info(msg)
core_utils.add_completed_steps(txt_name, step, outstep_file_suffix,
step_completed, end_time)
pytest.skip("Skipping " + step +
" because the input file does not exist.")
def output_hdul(set_inandout_filenames, config):
# determine if the pipeline is to be run in full, per steps, or skipped
run_calwebb_spec2 = config.get("run_calwebb_spec2_in_full",
"run_calwebb_spec2")
if run_calwebb_spec2 == "skip":
print(
'\n * PTT finished processing run_calwebb_spec2 is set to skip. \n'
)
pytest.exit(
"Skipping pipeline run and tests for spec2, run_calwebb_spec2 is set to skip in PTT_config file."
)
elif "T" in run_calwebb_spec2:
run_calwebb_spec2 = True
else:
run_calwebb_spec2 = False
# get the general info
set_inandout_filenames_info = core_utils.read_info4outputhdul(
config, set_inandout_filenames)
step, txt_name, step_input_file, step_output_file, outstep_file_suffix = set_inandout_filenames_info
msa_shutter_conf = config.get("benchmark_intermediary_products",
"msa_conf_name")
msametfl = os.path.basename(msa_shutter_conf)
dflat_path = config.get("benchmark_intermediary_products", "dflat_path")
sflat_path = config.get("benchmark_intermediary_products", "sflat_path")
fflat_path = config.get("benchmark_intermediary_products", "fflat_path")
flattest_threshold_diff = config.get("additional_arguments",
"flattest_threshold_diff")
save_flattest_plot = config.getboolean("additional_arguments",
"save_flattest_plot")
write_flattest_files = config.getboolean("additional_arguments",
"write_flattest_files")
flattest_paths = [
step_output_file, msa_shutter_conf, dflat_path, sflat_path, fflat_path
]
flattest_switches = [
flattest_threshold_diff, save_flattest_plot, write_flattest_files
]
run_pipe_step = config.getboolean("run_pipe_steps", step)
# determine which tests are to be run
flat_field_completion_tests = config.getboolean(
"run_pytest", "_".join((step, "completion", "tests")))
flat_field_reffile_tests = config.getboolean(
"run_pytest", "_".join((step, "reffile", "tests")))
flat_field_validation_tests = config.getboolean(
"run_pytest", "_".join((step, "validation", "tests")))
run_pytests = [
flat_field_completion_tests, flat_field_reffile_tests,
flat_field_validation_tests
]
# if run_calwebb_spec2 is True calwebb_spec2 will be called, else individual steps will be ran
step_completed = False
end_time = '0.0'
# check if the filter is to be changed
change_filter_opaque = config.getboolean("calwebb_spec2_input_file",
"change_filter_opaque")
if change_filter_opaque:
is_filter_opaque, step_input_filename = change_filter_opaque2science.change_filter_opaque(
step_input_file, step=step)
if is_filter_opaque:
filter_opaque_msg = "With FILTER=OPAQUE, the calwebb_spec2 will run up to the extract_2d step. Flat " \
"Field pytest now set to Skip."
print(filter_opaque_msg)
core_utils.add_completed_steps(txt_name, step, outstep_file_suffix,
step_completed, end_time)
pytest.skip("Skipping " + step + " because FILTER=OPAQUE.")
# get the MSA shutter configuration file full path only for MOS data
output_directory = config.get("calwebb_spec2_input_file",
"output_directory")
initial_input_file = config.get("calwebb_spec2_input_file", "input_file")
initial_input_file = os.path.join(output_directory, initial_input_file)
if os.path.isfile(initial_input_file):
inhdu = core_utils.read_hdrfits(initial_input_file,
info=False,
show_hdr=False)
detector = fits.getval(initial_input_file, "DETECTOR", 0)
else:
pytest.skip(
"Skipping " + step +
" because the initial input file given in PTT_config.cfg does not exist."
)
if run_calwebb_spec2:
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
flattest_paths = [
step_output_file, msa_shutter_conf, dflat_path, sflat_path,
fflat_path
]
return hdul, step_output_file, flattest_paths, flattest_switches, run_pytests
else:
if run_pipe_step:
# Create the logfile for PTT, but erase the previous one if it exists
PTTcalspec2_log = os.path.join(
output_directory,
'PTT_calspec2_' + detector + '_' + step + '.log')
if os.path.isfile(PTTcalspec2_log):
os.remove(PTTcalspec2_log)
print(
"Information outputed to screen from PTT will be logged in file: ",
PTTcalspec2_log)
for handler in logging.root.handlers[:]:
logging.root.removeHandler(handler)
logging.basicConfig(filename=PTTcalspec2_log, level=logging.INFO)
# print pipeline version
import jwst
pipeline_version = "\n *** Using jwst pipeline version: " + jwst.__version__ + " *** \n"
print(pipeline_version)
logging.info(pipeline_version)
if change_filter_opaque:
logging.info(filter_opaque_msg)
if os.path.isfile(step_input_file):
msg = " *** Step " + step + " set to True"
print(msg)
logging.info(msg)
stp = FlatFieldStep()
# check that previous pipeline steps were run up to this point
core_utils.check_completed_steps(step, step_input_file)
if core_utils.check_MOS_true(inhdu):
# copy the MSA shutter configuration file into the pytest directory
subprocess.run(["cp", msa_shutter_conf, "."])
# start the timer to compute the step running time
ontheflyflat = step_output_file.replace(
"flat_field.fits", "interpolatedflat.fits")
msg1 = "Step product will be saved as: " + step_output_file
msg2 = "on-the-fly flat will be saved as: " + ontheflyflat
# get the right configuration files to run the step
print(msg1)
print(msg2)
logging.info(msg1)
logging.info(msg2)
local_pipe_cfg_path = config.get("calwebb_spec2_input_file",
"local_pipe_cfg_path")
# start the timer to compute the step running time
start_time = time.time()
if local_pipe_cfg_path == "pipe_source_tree_code":
stp.call(step_input_file,
output_file=step_output_file,
save_interpolated_flat=True)
else:
stp.call(step_input_file,
output_file=step_output_file,
save_interpolated_flat=True,
config_file=local_pipe_cfg_path +
'/flat_field.cfg')
# end the timer to compute the step running time
end_time = repr(time.time() - start_time) # this is in seconds
msg = "Step " + step + " took " + end_time + " seconds to finish"
print(msg)
logging.info(msg)
# move the on-the-fly flat to the working directory
subprocess.run(
["mv", os.path.basename(ontheflyflat), ontheflyflat])
# raname and move the flat_field output
subprocess.run([
"mv",
os.path.basename(step_output_file).replace(
"_flat_field.fits", "_flatfieldstep.fits"),
step_output_file
])
if core_utils.check_MOS_true(inhdu):
if TESTSDIR == os.path.dirname(msametfl):
# remove the copy of the MSA shutter configuration file
print("Removing MSA config file from: ", TESTSDIR)
subprocess.run(["rm", msametfl])
step_completed = True
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
# rename and move the pipeline log file
pipelog = "pipeline_" + detector + ".log"
try:
calspec2_pilelog = "calspec2_pipeline_" + step + "_" + detector + ".log"
pytest_workdir = TESTSDIR
logfile = glob(pytest_workdir + "/" + pipelog)[0]
os.rename(logfile,
os.path.join(output_directory, calspec2_pilelog))
except IndexError:
print(
"\n* WARNING: Something went wrong. Could not find a ",
pipelog, " file \n")
# add the running time for this step
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed, end_time)
return hdul, step_output_file, flattest_paths, flattest_switches, run_pytests
else:
msg = " The input file does not exist. Skipping step."
print(msg)
logging.info(msg)
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed, end_time)
pytest.skip("Skipping " + step +
" because the input file does not exist.")
else:
msg = "Skipping running pipeline step " + step
print(msg)
logging.info(msg)
end_time = core_utils.get_stp_run_time_from_screenfile(
step, detector, output_directory)
if os.path.isfile(step_output_file):
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
step_completed = True
# add the running time for this step
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed, end_time)
return hdul, step_output_file, flattest_paths, flattest_switches, run_pytests
else:
step_completed = False
# add the running time for this step
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed, end_time)
pytest.skip("Test skipped because input file " +
step_output_file + " does not exist.")
def output_hdul(set_inandout_filenames, config):
set_inandout_filenames_info = core_utils.read_info4outputhdul(
config, set_inandout_filenames)
step, txt_name, step_input_file, step_output_file, run_calwebb_spec2, outstep_file_suffix = set_inandout_filenames_info
# start the timer to compute the step running time of PTT
PTT_start_time = time.time()
# check if the filter is to be changed
change_filter_opaque = config.getboolean("calwebb_spec2_input_file",
"change_filter_opaque")
if change_filter_opaque:
_, step_input_file = change_filter_opaque2science.change_filter_opaque(
step_input_file)
change_filter_opaque_msg = " * With FILTER=OPAQUE, the calwebb_spec2 will run up to the extract_2d step. Further steps will be skipped. \n"
print(change_filter_opaque_msg)
# determine if the pipeline is to be run in full
run_calwebb_spec2 = config.getboolean("run_calwebb_spec2_in_full",
"run_calwebb_spec2")
# determine which steps are to be run, if not run in full
run_pipe_step = config.getboolean("run_pipe_steps", step)
# determine which tests are to be run
assign_wcs_completion_tests = config.getboolean(
"run_pytest", "_".join((step, "completion", "tests")))
assign_wcs_reffile_tests = config.getboolean(
"run_pytest", "_".join((step, "reffile", "tests")))
assign_wcs_validation_tests = config.getboolean(
"run_pytest", "_".join((step, "validation", "tests")))
run_pytests = [
assign_wcs_completion_tests, assign_wcs_reffile_tests,
assign_wcs_validation_tests
]
# get other relevant info from PTT config file
esa_files_path = config.get("esa_intermediary_products", "esa_files_path")
wcs_threshold_diff = config.get("additional_arguments",
"wcs_threshold_diff")
save_wcs_plots = config.getboolean("additional_arguments",
"save_wcs_plots")
output_directory = config.get("calwebb_spec2_input_file",
"output_directory")
# Get the detector used
detector = fits.getval(step_input_file, "DETECTOR", 0)
# get main header from input file
inhdu = core_utils.read_hdrfits(step_input_file,
info=False,
show_hdr=False)
# if run_calwebb_spec2 is True calwebb_spec2 will be called, else individual steps will be ran
step_completed = False
end_time = '0.0'
# Check if data is IFU that the Image Model keyword is correct
mode_used = config.get("calwebb_spec2_input_file", "mode_used").lower()
if mode_used == "ifu":
DATAMODL = fits.getval(step_input_file, "DATAMODL", 0)
if DATAMODL != "IFUImageModel":
fits.setval(step_input_file, "DATAMODL", 0, value="IFUImageModel")
print("DATAMODL keyword changed to IFUImageModel.")
# get the shutter configuration file for MOS data only
msa_shutter_conf = "No shutter configuration file will be used."
if core_utils.check_MOS_true(inhdu):
msa_shutter_conf = config.get("esa_intermediary_products",
"msa_conf_name")
# check if the configuration shutter file name is in the header of the fits file and if not add it
msametfl = fits.getval(step_input_file, "MSAMETFL", 0)
if os.path.basename(msa_shutter_conf) != msametfl:
msametfl = os.path.basename(msa_shutter_conf)
fits.setval(step_input_file, "MSAMETFL", 0, value=msametfl)
# check if processing an image, then set proper variables
imaging_mode = False
if mode_used in ('image', 'confirm', 'taconfirm', 'wata', 'msata', 'bota',
'focus', 'mimf'):
run_calwebb_spec2 = True
imaging_mode = True
print(
'\n * Image processing will only be run in full with PTT. All intermediary products will be saved.'
)
print(
' -> For now, all pytests will be skipped since there are now image-specific routines yet. \n'
)
# TODO: add imaging tests
# get the name of the configuration file and run the pipeline
calwebb_spec2_cfg = config.get("run_calwebb_spec2_in_full",
"calwebb_spec2_cfg")
# copy the configuration file to create the pipeline log
stpipelogcfg = calwebb_spec2_cfg.replace("calwebb_spec2.cfg",
"stpipe-log.cfg")
subprocess.run(["cp", stpipelogcfg, os.getcwd()])
# run the pipeline
if run_calwebb_spec2:
# Create the logfile for PTT, but remove the previous log file
PTTcalspec2_log = os.path.join(output_directory,
'PTT_calspec2_' + detector + '.log')
if imaging_mode:
PTTcalspec2_log = PTTcalspec2_log.replace('calspec2', 'calimage2')
if os.path.isfile(PTTcalspec2_log):
os.remove(PTTcalspec2_log)
print(
"Information outputed to screen from PTT will be logged in file: ",
PTTcalspec2_log)
for handler in logging.root.handlers[:]:
logging.root.removeHandler(handler)
logging.basicConfig(filename=PTTcalspec2_log, level=logging.INFO)
logging.info(pipeline_version)
if change_filter_opaque:
logging.info(change_filter_opaque_msg)
run_calwebb_spec2_msg = " *** Will run pipeline in full ... "
print(run_calwebb_spec2_msg)
logging.info(run_calwebb_spec2_msg)
# create the map
txt_name = "full_run_map_" + detector + ".txt"
if os.path.isfile(txt_name):
os.remove(txt_name)
assign_wcs_utils.create_completed_steps_txtfile(
txt_name, step_input_file)
# start the timer to compute the step running time of PTT
core_utils.start_end_PTT_time(txt_name,
start_time=PTT_start_time,
end_time=None)
if mode_used == "bots":
calwebb_spec2_cfg = calwebb_spec2_cfg.replace(
"calwebb_spec2.cfg", "calwebb_tso-spec2.cfg")
print(
'\nUsing the following configuration file to run TSO pipeline:'
)
print(calwebb_spec2_cfg, '\n')
if imaging_mode:
calwebb_image2_cfg = calwebb_spec2_cfg.replace(
"calwebb_spec2.cfg", "calwebb_image2.cfg")
print(
'\nUsing the following configuration file to run imaging pipeline:'
)
print(calwebb_image2_cfg, '\n')
else:
print(
'\nUsing the following configuration file to run spectroscopic pipeline:'
)
print(calwebb_spec2_cfg, '\n')
input_file = config.get("calwebb_spec2_input_file", "input_file")
if "_uncal_rate" in input_file:
input_file = input_file.replace("_uncal_rate", "")
if "_uncal" in input_file:
input_file = input_file.replace("_uncal", "")
final_output_name = input_file.replace(".fits", "_cal.fits")
if core_utils.check_MOS_true(inhdu):
# copy the MSA shutter configuration file into the pytest directory
subprocess.run(["cp", msa_shutter_conf, "."])
# start the timer to compute the step running time
start_time = time.time()
# run the pipeline
print('Running pipeline... \n')
if not imaging_mode:
Spec2Pipeline.call(
step_input_file,
config_file=calwebb_spec2_cfg) #, logcfg=stpipelogcfg)
else:
Image2Pipeline.call(step_input_file,
config_file=calwebb_image2_cfg)
subprocess.run(["rm", "stpipe-log.cfg"])
# For the moment, the pipeline is using the wrong reference file for slit 400A1, so the
# file needs to be re-processed with the right reference file
if core_utils.check_FS_true(inhdu):
print(
"\n * WARNING: For the moment, the wrong reference file is being used for "
"processing slit 400A1. The file will be re-processed ")
# print(" $ jwst.pathloss.PathLossStep final_output_caldet1_NRS1_srctype.fits "
# "--override_pathloss=jwst-nirspec-a400.plrf.fits \n")
pathloss_400a1 = step_input_file.replace("srctype.fits",
"pathloss_400A1.fits")
reffile_400a1 = "jwst-nirspec-a400.plrf.fits"
print("Re-processing slit with new reference file: ",
reffile_400a1)
pl = PathLossStep()
pl.override_pathloss = reffile_400a1
pl.run(step_input_file)
subprocess.run([
"mv",
step_input_file.replace("srctype", "pathlossstep"),
pathloss_400a1
])
print("Saved pipeline re-processed file as: ", pathloss_400a1)
# end the timer to compute calwebb_spec2 running time
end_time = repr(time.time() - start_time) # this is in seconds
calspec2_time = " * Pipeline took " + end_time + " seconds to finish.\n"
print(calspec2_time)
logging.info(calspec2_time)
# remove the copy of the MSA shutter configuration file
if core_utils.check_MOS_true(inhdu):
subprocess.run(["rm", msametfl])
# add the detector string to the name of the files and move them to the working directory
core_utils.add_detector2filename(output_directory, step_input_file)
final_output_name_msg = "\nThe final pipeline product was saved in: " + final_output_name
print(final_output_name_msg)
logging.info(final_output_name_msg)
# read the assign wcs fits file
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
# scihdul = core_utils.read_hdrfits(step_output_file, info=False, show_hdr=False, ext=1)
# rename and move the pipeline log file
try:
calspec2_pipelog = "calspec2_pipeline_" + detector + ".log"
if imaging_mode:
calspec2_pipelog = calspec2_pipelog.replace(
'calspec2', 'calimage2')
path_where_pipeline_was_run = os.getcwd()
logfile = glob(path_where_pipeline_was_run + "/pipeline.log")[0]
print(logfile)
os.rename(logfile, os.path.join(output_directory,
calspec2_pipelog))
except IndexError:
print(
"\nWARNING: Something went wrong. Could not find a pipeline.log file \n"
)
# make sure we are able to find calspec2_pipelog either in the calwebb_spec2 directory or in the working dir
if not os.path.isfile(calspec2_pipelog):
calspec2_pipelog = os.path.join(output_directory, calspec2_pipelog)
# add the running time for all steps
step_running_times = core_utils.calculate_step_run_time(
calspec2_pipelog)
end_time_list = []
for stp in core_utils.step_string_dict:
if stp in step_running_times:
step_completed = True
step_time = step_running_times[stp]["run_time"]
out_suffix = core_utils.step_string_dict[stp]["suffix"]
core_utils.add_completed_steps(txt_name, stp, out_suffix,
step_completed, step_time)
end_time_list.append(step_time)
# print total running time in the text file and move it to the indicated directory
string2print = "pipeline_total_time"
if float(end_time) <= sum(end_time_list):
tot_time = repr(sum(end_time_list))
else:
tot_time = end_time
assign_wcs_utils.print_time2file(txt_name, tot_time, string2print)
PTT_runtimes_msg = "Pipeline and PTT run times written in file: " + os.path.basename(
txt_name) + " in working directory. \n"
print(PTT_runtimes_msg)
logging.info(PTT_runtimes_msg)
# move the final reporting text files to the working directory
core_utils.move_txt_files_2workdir(config, detector)
# end script for imaging case
if imaging_mode:
print('\nPTT finished processing imaging mode. \n')
pytest.exit(
"Skipping pytests for now because they need to be written for imaging mode."
)
return hdul, step_output_file, msa_shutter_conf, esa_files_path, wcs_threshold_diff, save_wcs_plots, run_pytests, mode_used
else:
# create the map but remove a previous one if it exists
if os.path.isfile(txt_name):
os.remove(txt_name)
assign_wcs_utils.create_completed_steps_txtfile(
txt_name, step_input_file)
# start the timer to compute the step running time of PTT
core_utils.start_end_PTT_time(txt_name,
start_time=PTT_start_time,
end_time=None)
msg = "\n Pipeline and PTT run times will be written in file: " + os.path.basename(
txt_name) + " in working directory. \n"
print(msg)
logging.info(msg)
if run_pipe_step:
# Create the logfile for PTT, but erase the previous one if it exists
PTTcalspec2_log = os.path.join(
output_directory,
'PTT_calspec2_' + detector + '_' + step + '.log')
if imaging_mode:
PTTcalspec2_log = PTTcalspec2_log.replace(
'calspec2', 'calimage2')
if os.path.isfile(PTTcalspec2_log):
os.remove(PTTcalspec2_log)
print(
"Information outputed to screen from PTT will be logged in file: ",
PTTcalspec2_log)
for handler in logging.root.handlers[:]:
logging.root.removeHandler(handler)
logging.basicConfig(filename=PTTcalspec2_log, level=logging.INFO)
logging.info(pipeline_version)
if change_filter_opaque:
logging.info(change_filter_opaque_msg)
# check that previous pipeline steps were run up to this point
core_utils.check_completed_steps(step, step_input_file)
if os.path.isfile(step_input_file):
msg = " *** Step " + step + " set to True"
print(msg)
logging.info(msg)
stp = AssignWcsStep()
if core_utils.check_MOS_true(inhdu):
# copy the MSA shutter configuration file into the pytest directory
subprocess.run(["cp", msa_shutter_conf, "."])
# get the right configuration files to run the step
local_pipe_cfg_path = config.get("calwebb_spec2_input_file",
"local_pipe_cfg_path")
# start the timer to compute the step running time
print("running pipeline...")
start_time = time.time()
if local_pipe_cfg_path == "pipe_source_tree_code":
result = stp.call(step_input_file)
else:
result = stp.call(step_input_file,
config_file=local_pipe_cfg_path +
'/assign_wcs.cfg')
result.save(step_output_file)
# end the timer to compute the step running time
end_time = repr(time.time() - start_time) # this is in seconds
msg = "Step " + step + " took " + end_time + " seconds to finish"
print(msg)
logging.info(msg)
if core_utils.check_MOS_true(inhdu):
# remove the copy of the MSA shutter configuration file
subprocess.run(["rm", msametfl])
# rename and move the pipeline log file
try:
calspec2_pipelog = "calspec2_pipeline_" + step + "_" + detector + ".log"
if imaging_mode:
calspec2_pipelog = calspec2_pipelog.replace(
'calspec2', 'calimage2')
pytest_workdir = TESTSDIR
logfile = glob(pytest_workdir + "/pipeline.log")[0]
os.rename(logfile,
os.path.join(output_directory, calspec2_pipelog))
except IndexError:
print(
"\n* WARNING: Something went wrong. Could not find a pipeline.log file \n"
)
else:
msg = "Skipping step. Input file " + step_input_file + " does not exit."
print(msg)
logging.info(msg)
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed, end_time)
pytest.skip("Skipping " + step +
" because the input file does not exist.")
else:
print("Skipping running pipeline step ", step)
# add the running time for this step
end_time = core_utils.get_stp_run_time_from_screenfile(
step, detector, output_directory)
if os.path.isfile(step_output_file):
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
step_completed = True
# add the running time for this step
core_utils.add_completed_steps(txt_name, step, outstep_file_suffix,
step_completed, end_time)
return hdul, step_output_file, msa_shutter_conf, esa_files_path, wcs_threshold_diff, save_wcs_plots, run_pytests, mode_used
else:
step_completed = False
# add the running time for this step
core_utils.add_completed_steps(txt_name, step, outstep_file_suffix,
step_completed, end_time)
pytest.skip("Test skipped because input file " + step_output_file +
" does not exist.")
def output_hdul(set_inandout_filenames, config):
# determine if the pipeline is to be run in full, per steps, or skipped
run_calwebb_spec2 = config.get("run_calwebb_spec2_in_full",
"run_calwebb_spec2")
if run_calwebb_spec2 == "skip":
print(
'\n * PTT finished processing run_calwebb_spec2 is set to skip. \n'
)
pytest.exit(
"Skipping pipeline run and tests for spec2, run_calwebb_spec2 is set to skip in PTT_config file."
)
elif "T" in run_calwebb_spec2:
run_calwebb_spec2 = True
else:
run_calwebb_spec2 = False
# get the general info
set_inandout_filenames_info = core_utils.read_info4outputhdul(
config, set_inandout_filenames)
step, txt_name, step_input_file, step_output_file, outstep_file_suffix = set_inandout_filenames_info
run_pipe_step = config.getboolean("run_pipe_steps", step)
# determine which tests are to be run
pathloss_completion_tests = config.getboolean(
"run_pytest", "_".join((step, "completion", "tests")))
pathloss_reffile_tests = config.getboolean(
"run_pytest", "_".join((step, "reffile", "tests")))
pathloss_validation_tests = config.getboolean(
"run_pytest", "_".join((step, "validation", "tests")))
run_pytests = [
pathloss_completion_tests, pathloss_reffile_tests,
pathloss_validation_tests
]
pathloss_threshold_diff = config.get("additional_arguments",
"pathloss_threshold_diff")
save_flattest_plot = config.getboolean("additional_arguments",
"save_pathloss_plot")
write_flattest_files = config.getboolean("additional_arguments",
"write_pathloss_files")
pathloss_switches = [
pathloss_threshold_diff, save_flattest_plot, write_flattest_files
]
# if run_calwebb_spec2 is True calwebb_spec2 will be called, else individual steps will be ran
step_completed = False
end_time = '0.0'
# check if the filter is to be changed
change_filter_opaque = config.getboolean("calwebb_spec2_input_file",
"change_filter_opaque")
if change_filter_opaque:
is_filter_opaque, step_input_filename = change_filter_opaque2science.change_filter_opaque(
step_input_file, step=step)
if is_filter_opaque:
filter_opaque_msg = "With FILTER=OPAQUE, the calwebb_spec2 will run up to the extract_2d step. " \
"Pathloss pytest now set to Skip."
print(filter_opaque_msg)
core_utils.add_completed_steps(txt_name, step, outstep_file_suffix,
step_completed, end_time)
pytest.skip("Skipping " + step + " because FILTER=OPAQUE.")
# only run this step if data is not BOTS
output_directory = config.get("calwebb_spec2_input_file",
"output_directory")
initial_input_file = config.get("calwebb_spec2_input_file", "input_file")
initial_input_file = os.path.join(output_directory, initial_input_file)
if os.path.isfile(initial_input_file):
inhdu = core_utils.read_hdrfits(initial_input_file,
info=False,
show_hdr=False)
detector = fits.getval(initial_input_file, "DETECTOR", 0)
else:
pytest.skip(
"Skipping " + step +
" because the initial input file given in PTT_config.cfg does not exist."
)
if not core_utils.check_BOTS_true(inhdu):
if run_calwebb_spec2:
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
return hdul, step_output_file, step_input_file, run_pytests, pathloss_switches
else:
if run_pipe_step:
# Create the logfile for PTT, but erase the previous one if it exists
PTTcalspec2_log = os.path.join(
output_directory,
'PTT_calspec2_' + detector + '_' + step + '.log')
if os.path.isfile(PTTcalspec2_log):
os.remove(PTTcalspec2_log)
print(
"Information outputed to screen from PTT will be logged in file: ",
PTTcalspec2_log)
for handler in logging.root.handlers[:]:
logging.root.removeHandler(handler)
logging.basicConfig(filename=PTTcalspec2_log,
level=logging.INFO)
# print pipeline version
import jwst
pipeline_version = "\n *** Using jwst pipeline version: " + jwst.__version__ + " *** \n"
print(pipeline_version)
logging.info(pipeline_version)
if change_filter_opaque:
logging.info(filter_opaque_msg)
if os.path.isfile(step_input_file):
msg = " *** Step " + step + " set to True"
print(msg)
logging.info(msg)
stp = PathLossStep()
# check that previous pipeline steps were run up to this point
core_utils.check_completed_steps(step, step_input_file)
# get the right configuration files to run the step
local_pipe_cfg_path = config.get(
"calwebb_spec2_input_file", "local_pipe_cfg_path")
# start the timer to compute the step running time
start_time = time.time()
if local_pipe_cfg_path == "pipe_source_tree_code":
result = stp.call(step_input_file)
else:
result = stp.call(step_input_file,
config_file=local_pipe_cfg_path +
'/pathloss.cfg')
result.save(step_output_file)
# For the moment, the pipeline is using the wrong reference file for slit 400A1, so the
# file needs to be re-processed with the right reference file BUT we are
# skipping this slit
"""
if core_utils.check_FS_true(inhdu):
print("\n * WARNING: For the moment, the wrong reference file is being used for "
"processing slit 400A1. The file will be re-processed ")
# The file can manually be re-run with
# $ strun jwst.pathloss.PathLossStep final_output_caldet1_NRS1_srctype.fits
# --override_pathloss=jwst-nirspec-a400.plrf.fits
pathloss_400a1 = step_input_file.replace("srctype.fits", "pathloss_400A1.fits")
reffile_400a1 = "jwst-nirspec-a400.plrf.fits"
print("Re-processing slit with new reference file: ", reffile_400a1)
pl = PathLossStep()
pl.override_pathloss = reffile_400a1
pl_result = pl.run(step_input_file)
pl_result.save(pathloss_400a1)
#import subprocess
#subprocess.run(["mv", step_input_file.replace("srctype", "pathlossstep"), pathloss_400a1])
print("Saved pipeline re-processed file as: ", pathloss_400a1)
"""
# end the timer to compute calwebb_spec2 running time
end_time = repr(time.time() -
start_time) # this is in seconds
msg = " * calwebb_spec2 took " + end_time + " seconds to finish."
print(msg)
logging.info(msg)
step_completed = True
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
# rename and move the pipeline log file
pipelog = "pipeline_" + detector + ".log"
try:
calspec2_pilelog = "calspec2_pipeline_" + step + "_" + detector + ".log"
pytest_workdir = TESTSDIR
logfile = glob(pytest_workdir + "/" + pipelog)[0]
os.rename(
logfile,
os.path.join(output_directory, calspec2_pilelog))
except IndexError:
print(
"\n* WARNING: Something went wrong. Could not find a ",
pipelog, " file \n")
# add the running time for this step
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed, end_time)
return hdul, step_output_file, step_input_file, run_pytests, pathloss_switches
else:
msg = " The input file does not exist. Skipping step."
print(msg)
logging.info(msg)
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed, end_time)
pytest.skip("Skipping " + step +
" because the input file does not exist.")
else:
msg = "Skipping running pipeline step " + step
print(msg)
logging.info(msg)
end_time = core_utils.get_stp_run_time_from_screenfile(
step, detector, output_directory)
if os.path.isfile(step_output_file):
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
step_completed = True
# add the running time for this step
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed, end_time)
return hdul, step_output_file, step_input_file, run_pytests, pathloss_switches
else:
step_completed = False
# add the running time for this step
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed, end_time)
pytest.skip("Test skipped because input file " +
step_output_file + " does not exist.")
else:
pytest.skip("Skipping " + step + " because data is BOTS.")
