def check_output_is_zero(output_hdul):
"""
This test is a simple subtraction of the input file minus a copy of the input file (instead of the actual
MSA imprint file. The output is expected to be zero.
:param output_hdul: list
:return: result: boolean
"""
# output_hdul = hdul, step_output_file, step_input_file, run_pytests
step_input_file = output_hdul[2]
step_output_file = output_hdul[1]
# Only run test if data is IFU or MSA
inhdu = core_utils.read_hdrfits(step_input_file,
info=False,
show_hdr=False)
if core_utils.check_IFU_true(inhdu) or core_utils.check_MOS_true(inhdu):
# set specifics for the test
msa_imprint_structure = copy.deepcopy(step_input_file)
result_to_check = step_output_file.replace(".fits", "_zerotest.fits")
# run the step with the specifics
stp = ImprintStep()
res = stp.call(step_input_file, msa_imprint_structure)
res.save(result_to_check)
# check that the end product of image - image is zero
c = fits.getdata(result_to_check)
subtraction = sum(c.flatten())
result = False
if subtraction == 0.0:
result = True
# erase test output file
subprocess.run(["rm", result_to_check])
return result
def output_hdul(set_inandout_filenames, config):
# determine if the pipeline is to be run in full, per steps, or skipped
run_calwebb_spec2 = config.get("run_calwebb_spec2_in_full",
"run_calwebb_spec2")
if run_calwebb_spec2 == "skip":
print(
'\n * PTT finished processing run_calwebb_spec2 is set to skip. \n'
)
pytest.exit(
"Skipping pipeline run and tests for spec2, run_calwebb_spec2 is set to skip in PTT_config file."
)
elif "T" in run_calwebb_spec2:
run_calwebb_spec2 = True
else:
run_calwebb_spec2 = False
# get the general info
set_inandout_filenames_info = core_utils.read_info4outputhdul(
config, set_inandout_filenames)
step, txt_name, step_input_file, step_output_file, outstep_file_suffix = set_inandout_filenames_info
run_pipe_step = config.getboolean("run_pipe_steps", step)
# determine which tests are to be run
extract_2d_completion_tests = config.getboolean(
"run_pytest", "_".join((step, "completion", "tests")))
extract_2d_validation_tests = config.getboolean(
"run_pytest", "_".join((step, "validation", "tests")))
assign_wcs_validation_tests = config.getboolean(
"run_pytest", "_".join((step, "validation", "tests")))
run_pytests = [
extract_2d_completion_tests, extract_2d_validation_tests,
assign_wcs_validation_tests
]
# get other relevant info from PTT config file
compare_assign_wcs_and_extract_2d_with_esa = config.getboolean(
"benchmark_intermediary_products",
"compare_assign_wcs_and_extract_2d_with_esa")
esa_files_path = config.get("benchmark_intermediary_products",
"esa_files_path")
data_directory = config.get("calwebb_spec2_input_file", "data_directory")
truth_file = os.path.join(
data_directory,
config.get("benchmark_intermediary_products", "truth_file_assign_wcs"))
if compare_assign_wcs_and_extract_2d_with_esa:
truth_file = esa_files_path
print("Will use this 'truth' file to compare result of extract_2d: ")
print(truth_file)
msa_conf_name = config.get("benchmark_intermediary_products",
"msa_conf_name")
extract_2d_threshold_diff = int(
config.get("additional_arguments", "extract_2d_threshold_diff"))
# Check if the mode used is MOS_sim and get the threshold for the assign_wcs test
mode_used = config.get("calwebb_spec2_input_file", "mode_used").lower()
wcs_threshold_diff = config.get("additional_arguments",
"wcs_threshold_diff")
save_wcs_plots = config.getboolean("additional_arguments",
"save_wcs_plots")
# if run_calwebb_spec2 is True calwebb_spec2 will be called, else individual steps will be ran
step_completed = False
end_time = '0.0'
# only do this step if data is NOT IFU
output_directory = config.get("calwebb_spec2_input_file",
"output_directory")
initial_input_file = config.get("calwebb_spec2_input_file", "input_file")
initial_input_file = os.path.join(output_directory, initial_input_file)
if os.path.isfile(initial_input_file):
inhdu = core_utils.read_hdrfits(initial_input_file,
info=False,
show_hdr=False)
detector = fits.getval(initial_input_file, "DETECTOR", 0)
else:
pytest.skip(
"Skipping " + step +
" because the initial input file given in PTT_config.cfg does not exist."
)
if not core_utils.check_IFU_true(inhdu):
if run_calwebb_spec2:
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
return hdul, step_output_file, msa_conf_name, truth_file, run_pytests, mode_used, wcs_threshold_diff, \
save_wcs_plots, extract_2d_threshold_diff, compare_assign_wcs_and_extract_2d_with_esa
else:
if run_pipe_step:
# Create the logfile for PTT, but erase the previous one if it exists
PTTcalspec2_log = os.path.join(
output_directory,
'PTT_calspec2_' + detector + '_' + step + '.log')
if os.path.isfile(PTTcalspec2_log):
os.remove(PTTcalspec2_log)
print(
"Information outputed to screen from PTT will be logged in file: ",
PTTcalspec2_log)
for handler in logging.root.handlers[:]:
logging.root.removeHandler(handler)
logging.basicConfig(filename=PTTcalspec2_log,
level=logging.INFO)
# print pipeline version
import jwst
pipeline_version = "\n *** Using jwst pipeline version: " + jwst.__version__ + " *** \n"
print(pipeline_version)
logging.info(pipeline_version)
if os.path.isfile(step_input_file):
msg = " The input file " + step_input_file + " exists... will run step " + step
print(msg)
logging.info(msg)
stp = Extract2dStep()
# check that previous pipeline steps were run up to this point
core_utils.check_completed_steps(step, step_input_file)
# get the right configuration files to run the step
local_pipe_cfg_path = config.get(
"calwebb_spec2_input_file", "local_pipe_cfg_path")
# start the timer to compute the step running time
start_time = time.time()
if local_pipe_cfg_path == "pipe_source_tree_code":
result = stp.call(step_input_file)
else:
result = stp.call(step_input_file,
config_file=local_pipe_cfg_path +
'/extract_2d.cfg')
result.save(step_output_file)
step_completed = True
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
# end the timer to compute the step running time
end_time = repr(time.time() -
start_time) # this is in seconds
msg = "Step " + step + " took " + end_time + " seconds to finish"
print(msg)
logging.info(msg)
# rename and move the pipeline log file
pipelog = "pipeline_" + detector + ".log"
try:
calspec2_pilelog = "calspec2_pipeline_" + step + "_" + detector + ".log"
pytest_workdir = TESTSDIR
logfile = glob(pytest_workdir + "/" + pipelog)[0]
os.rename(
logfile,
os.path.join(output_directory, calspec2_pilelog))
except IndexError:
print(
"\n* WARNING: Something went wrong. Could not find a ",
pipelog, " file \n")
# add the running time for this step
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed, end_time)
return hdul, step_output_file, msa_conf_name, truth_file, run_pytests, mode_used, \
wcs_threshold_diff, save_wcs_plots, extract_2d_threshold_diff, compare_assign_wcs_and_extract_2d_with_esa
else:
msg = " The input file does not exist. Skipping step."
print(msg)
logging.info(msg)
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed, end_time)
pytest.skip("Skiping " + step +
" because the input file does not exist.")
else:
msg = "Skipping running pipeline step " + step
print(msg)
logging.info(msg)
end_time = core_utils.get_stp_run_time_from_screenfile(
step, detector, output_directory)
if os.path.isfile(step_output_file):
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
step_completed = True
# add the running time for this step
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed, end_time)
return hdul, step_output_file, msa_conf_name, truth_file, run_pytests, mode_used, \
wcs_threshold_diff, save_wcs_plots, extract_2d_threshold_diff, compare_assign_wcs_and_extract_2d_with_esa
else:
step_completed = False
# add the running time for this step
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed, end_time)
pytest.skip("Test skipped because input file " +
step_output_file + " does not exist.")
else:
core_utils.add_completed_steps(txt_name, step, outstep_file_suffix,
step_completed, end_time)
pytest.skip("Skipping " + step + " because data is IFU.")
def output_hdul(set_inandout_filenames, config):
# determine if the pipeline is to be run in full, per steps, or skipped
run_calwebb_spec2 = config.get("run_calwebb_spec2_in_full", "run_calwebb_spec2")
if run_calwebb_spec2 == "skip":
print('\n * PTT finished processing run_calwebb_spec2 is set to skip. \n')
pytest.exit("Skipping pipeline run and tests for spec2, run_calwebb_spec2 is set to skip in PTT_config file.")
elif "T" in run_calwebb_spec2:
run_calwebb_spec2 = True
else:
run_calwebb_spec2 = False
# get the general info
set_inandout_filenames_info = core_utils.read_info4outputhdul(config, set_inandout_filenames)
step, txt_name, step_input_file, step_output_file, outstep_file_suffix = set_inandout_filenames_info
# determine which steps are to be run, if not run in full
run_pipe_step = config.getboolean("run_pipe_steps", step)
# determine which tests are to be run
bkg_subtract_completion_tests = config.getboolean("run_pytest", "_".join((step, "completion", "tests")))
bkg_subtract_numerical_tests = config.getboolean("run_pytest", "_".join((step, "numerical", "tests")))
#bkg_subtract_validation_tests = config.getboolean("run_pytest", "_".join((step, "validation", "tests")))
run_pytests = [bkg_subtract_completion_tests, bkg_subtract_numerical_tests]#, bkg_subtract_validation_tests]
# if run_calwebb_spec2 is True calwebb_spec2 will be called, else individual steps will be ran
step_completed = False
end_time = '0.0'
# skip if BOTS data, else perform step
output_directory = config.get("calwebb_spec2_input_file", "output_directory")
initial_input_file = config.get("calwebb_spec2_input_file", "input_file")
initial_input_file = os.path.join(output_directory, initial_input_file)
detector = fits.getval(initial_input_file, "DETECTOR", 0)
calspec2_pilelog = "calspec2_pipeline_" + step + "_" + detector + ".log"
pytest_workdir = TESTSDIR
if os.path.isfile(initial_input_file):
inhdu = core_utils.read_hdrfits(initial_input_file, info=False, show_hdr=False)
else:
pytest.skip("Skipping "+step+" because the initial input file given in PTT_config.cfg does not exist.")
if not core_utils.check_BOTS_true(inhdu):
if run_calwebb_spec2:
if os.path.isfile(step_output_file):
hdul = core_utils.read_hdrfits(step_output_file, info=False, show_hdr=False)
else:
pytest.skip("Skipping "+step+" because the output file does not exist.")
return hdul, step_output_file, step_input_file, run_pytests
else:
if run_pipe_step:
# Create the logfile for PTT, but erase the previous one if it exists
PTTcalspec2_log = os.path.join(output_directory, 'PTT_calspec2_'+detector+'_'+step+'.log')
if os.path.isfile(PTTcalspec2_log):
os.remove(PTTcalspec2_log)
print("Information outputed to screen from PTT will be logged in file: ", PTTcalspec2_log)
for handler in logging.root.handlers[:]:
logging.root.removeHandler(handler)
logging.basicConfig(filename=PTTcalspec2_log, level=logging.INFO)
# print pipeline version
import jwst
pipeline_version = "\n *** Using jwst pipeline version: "+jwst.__version__+" *** \n"
print(pipeline_version)
logging.info(pipeline_version)
if os.path.isfile(step_input_file):
msg = " The input file "+step_input_file+" exists... will run step "+step
print(msg)
logging.info(msg)
bkg_list = core_utils.getlist("additional_arguments", "bkg_list")
existing_bgfiles = 0
for bg_file in bkg_list:
if os.path.isfile(bg_file):
existing_bgfiles += 1
if existing_bgfiles == 0:
msg = " Need at least one background file to continue. Step will be skipped."
print(msg)
logging.info(msg)
core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time)
pytest.skip("Skipping "+step+" because files listed on bkg_list in the configuration "
"file do not exist.")
else:
# continue since the file(s) exist
msg = "*** Step "+step+" set to True"
print(msg)
logging.info(msg)
stp = BackgroundStep()
# check that previous pipeline steps were run up to this point
core_utils.check_completed_steps(step, step_input_file)
# get the right configuration files to run the step
local_pipe_cfg_path = config.get("calwebb_spec2_input_file", "local_pipe_cfg_path")
# start the timer to compute the step running time
start_time = time.time()
print("running pipeline...")
if local_pipe_cfg_path == "pipe_source_tree_code":
result = stp.call(step_input_file, bkg_list)
else:
result = stp.call(step_input_file, bkg_list, config_file=local_pipe_cfg_path+'/background.cfg')
if result is not None:
result.save(step_output_file)
# end the timer to compute the step running time
end_time = repr(time.time() - start_time) # this is in seconds
msg = "Step "+step+" took "+end_time+" seconds to finish"
print(msg)
logging.info(msg)
hdul = core_utils.read_hdrfits(step_output_file, info=False, show_hdr=False)
step_completed = True
# rename and move the pipeline log file
pipelog = "pipeline_" + detector + ".log"
try:
calspec2_pilelog = "calspec2_pipeline_" + step + "_" + detector + ".log"
pytest_workdir = TESTSDIR
logfile = glob(pytest_workdir + "/" + pipelog)[0]
os.rename(logfile, os.path.join(output_directory, calspec2_pilelog))
except IndexError:
print("\n* WARNING: Something went wrong. Could not find a ", pipelog, " file \n")
# add the running time for this step
core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time)
return hdul, step_output_file, step_input_file, run_pytests
else:
print(" The input file does not exist. Skipping step.")
core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time)
pytest.skip("Skipping "+step+" because the input file does not exist.")
else:
msg = "Skipping running pipeline step "+step
print(msg)
logging.info(msg)
end_time = core_utils.get_stp_run_time_from_screenfile(step, detector, output_directory)
if os.path.isfile(step_output_file):
hdul = core_utils.read_hdrfits(step_output_file, info=False, show_hdr=False)
step_completed = True
# add the running time for this step
core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time)
return hdul, step_output_file, step_input_file, run_pytests
else:
# add the running time for this step
core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time)
pytest.skip("Test skipped because input file "+step_output_file+" does not exist.")
else:
msg = "Skipping "+step+" because data is BOTS."
print(msg)
logging.info(msg)
core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time)
pytest.skip(msg)
def output_hdul(set_inandout_filenames, config):
# determine if the pipeline is to be run in full, per steps, or skipped
run_calwebb_spec2 = config.get("run_calwebb_spec2_in_full",
"run_calwebb_spec2")
if run_calwebb_spec2 == "skip":
print(
'\n * PTT finished processing run_calwebb_spec2 is set to skip. \n'
)
pytest.exit(
"Skipping pipeline run and tests for spec2, run_calwebb_spec2 is set to skip in PTT_config file."
)
elif "T" in run_calwebb_spec2:
run_calwebb_spec2 = True
else:
run_calwebb_spec2 = False
# get the general info
set_inandout_filenames_info = core_utils.read_info4outputhdul(
config, set_inandout_filenames)
step, txt_name, step_input_file, step_output_file, outstep_file_suffix = set_inandout_filenames_info
run_pipe_step = config.getboolean("run_pipe_steps", step)
# determine which tests are to be run
cube_build_completion_tests = config.getboolean(
"run_pytest", "_".join((step, "completion", "tests")))
#cube_build_reffile_tests = config.getboolean("run_pytest", "_".join((step, "reffile", "tests")))
#cube_build_validation_tests = config.getboolean("run_pytest", "_".join((step, "validation", "tests")))
run_pytests = [cube_build_completion_tests
] #, cube_build_reffile_tests, cube_build_validation_tests]
# Only run step if data is IFU
output_directory = config.get("calwebb_spec2_input_file",
"output_directory")
initial_input_file = config.get("calwebb_spec2_input_file", "input_file")
initial_input_file = os.path.join(output_directory, initial_input_file)
if os.path.isfile(initial_input_file):
inhdu = core_utils.read_hdrfits(initial_input_file,
info=False,
show_hdr=False)
detector = fits.getval(initial_input_file, "DETECTOR", 0)
filt = fits.getval(initial_input_file, 'filter')
grat = fits.getval(initial_input_file, 'grating')
gratfilt = grat + "-" + filt + "_s3d"
else:
pytest.skip(
"Skipping " + step +
" because the initial input file given in PTT_config.cfg does not exist."
)
end_time = '0.0'
if core_utils.check_IFU_true(inhdu):
# if run_calwebb_spec2 is True calwebb_spec2 will be called, else individual steps will be ran
step_completed = False
# check if the filter is to be changed
change_filter_opaque = config.getboolean("calwebb_spec2_input_file",
"change_filter_opaque")
if change_filter_opaque:
is_filter_opaque, step_input_filename = change_filter_opaque2science.change_filter_opaque(
step_input_file, step=step)
if is_filter_opaque:
filter_opaque_msg = "With FILTER=OPAQUE, the calwebb_spec2 will run up to the extract_2d step. " \
"Cube build pytest now set to Skip."
print(filter_opaque_msg)
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed, end_time)
pytest.skip("Skipping " + step + " because FILTER=OPAQUE.")
if run_calwebb_spec2:
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
return hdul, step_output_file, run_pytests
else:
if run_pipe_step:
# Create the logfile for PTT, but erase the previous one if it exists
PTTcalspec2_log = os.path.join(
output_directory,
'PTT_calspec2_' + detector + '_' + step + '.log')
if os.path.isfile(PTTcalspec2_log):
os.remove(PTTcalspec2_log)
print(
"Information outputed to screen from PTT will be logged in file: ",
PTTcalspec2_log)
for handler in logging.root.handlers[:]:
logging.root.removeHandler(handler)
logging.basicConfig(filename=PTTcalspec2_log,
level=logging.INFO)
# print pipeline version
import jwst
pipeline_version = "\n *** Using jwst pipeline version: " + jwst.__version__ + " *** \n"
print(pipeline_version)
logging.info(pipeline_version)
if change_filter_opaque:
logging.info(filter_opaque_msg)
if os.path.isfile(step_input_file):
msg = " *** Step " + step + " set to True"
print(msg)
logging.info(msg)
stp = CubeBuildStep()
# check that previous pipeline steps were run up to this point
core_utils.check_completed_steps(step, step_input_file)
# get the right configuration files to run the step
local_pipe_cfg_path = config.get(
"calwebb_spec2_input_file", "local_pipe_cfg_path")
# start the timer to compute the step running time
start_time = time.time()
if local_pipe_cfg_path == "pipe_source_tree_code":
result = stp.call(step_input_file)
else:
result = stp.call(step_input_file,
config_file=local_pipe_cfg_path +
'/cube_build.cfg')
result.save(step_output_file)
# end the timer to compute the step running time
end_time = repr(time.time() -
start_time) # this is in seconds
msg = "Step " + step + " took " + end_time + " seconds to finish"
print(msg)
logging.info(msg)
# determine the specific output of the cube step
specific_output_file = glob(
step_output_file.replace(
'cube.fits', (gratfilt + '*.fits').lower()))[0]
cube_suffix = specific_output_file.split(
'cube_build_')[-1].replace('.fits', '')
# record info
step_completed = True
hdul = core_utils.read_hdrfits(specific_output_file,
info=False,
show_hdr=False)
# rename and move the pipeline log file
pipelog = "pipeline_" + detector + ".log"
try:
calspec2_pilelog = "calspec2_pipeline_" + step + "_" + detector + ".log"
pytest_workdir = TESTSDIR
logfile = glob(pytest_workdir + "/" + pipelog)[0]
os.rename(
logfile,
os.path.join(output_directory, calspec2_pilelog))
except IndexError:
print(
"\n* WARNING: Something went wrong. Could not find a ",
pipelog, " file \n")
# add the running time for this step
core_utils.add_completed_steps(txt_name, step,
"_" + cube_suffix,
step_completed, end_time)
return hdul, step_output_file, run_pytests
else:
msg = " The input file does not exist. Skipping step."
print(msg)
logging.info(msg)
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed, end_time)
pytest.skip("Skipping " + step +
" because the input file does not exist.")
else:
msg = "Skipping running pipeline step " + step
print(msg)
logging.info(msg)
end_time = core_utils.get_stp_run_time_from_screenfile(
step, detector, output_directory)
# record info
# specific cube step suffix
cube_suffix = "_s3d"
if os.path.isfile(step_output_file):
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
step_completed = True
# add the running time for this step
core_utils.add_completed_steps(txt_name, step, cube_suffix,
step_completed, end_time)
return hdul, step_output_file, run_pytests
else:
step_completed = False
# add the running time for this step
core_utils.add_completed_steps(txt_name, step, cube_suffix,
step_completed, end_time)
pytest.skip("Test skipped because input file " +
step_output_file + " does not exist.")
else:
pytest.skip("Skipping " + step + " because data is not IFU.")
def output_hdul(set_inandout_filenames, config):
# determine if the pipeline is to be run in full, per steps, or skipped
run_calwebb_spec2 = config.get("run_calwebb_spec2_in_full",
"run_calwebb_spec2")
if run_calwebb_spec2 == "skip":
print(
'\n * PTT finished processing run_calwebb_spec2 is set to skip. \n'
)
pytest.exit(
"Skipping pipeline run and tests for spec2, run_calwebb_spec2 is set to skip in PTT_config file."
)
elif "T" in run_calwebb_spec2:
run_calwebb_spec2 = True
else:
run_calwebb_spec2 = False
# get the general info
set_inandout_filenames_info = core_utils.read_info4outputhdul(
config, set_inandout_filenames)
step, txt_name, step_input_file, step_output_file, outstep_file_suffix = set_inandout_filenames_info
run_pipe_step = config.getboolean("run_pipe_steps", step)
# determine which tests are to be run
extract_1d_completion_tests = config.getboolean(
"run_pytest", "_".join((step, "completion", "tests")))
extract_1d_reffile_tests = config.getboolean(
"run_pytest", "_".join((step, "reffile", "tests")))
# extract_1d_validation_tests = config.getboolean("run_pytest", "_".join((step, "validation", "tests")))
run_pytests = [extract_1d_completion_tests,
extract_1d_reffile_tests] #, extract_1d_validation_tests]
# Make sure at this point the MSA shutter configuration file is removed from the calwebb_spec2_pytests directory
msa_shutter_conf = config.get("benchmark_intermediary_products",
"msa_conf_name")
msametfl = os.path.basename(msa_shutter_conf)
# remove the copy of the MSA shutter configuration file
inhdu = core_utils.read_hdrfits(step_input_file,
info=False,
show_hdr=False)
if core_utils.check_MOS_true(inhdu):
if TESTSDIR == os.path.dirname(msametfl):
print("Removing MSA config file from: ", TESTSDIR)
os.remove(msametfl)
# check if processing an image, then set proper variables
imaging_mode = False
mode_used = config.get("calwebb_spec2_input_file", "mode_used").lower()
if mode_used in ('image', 'confirm', 'taconfirm', 'wata', 'msata', 'bota',
'focus', 'mimf'):
run_calwebb_spec2 = True
imaging_mode = True
# if run_calwebb_spec2 is True calwebb_spec2 will be called, else individual steps will be ran
step_completed = False
end_time = '0.0'
# Get the detector used
output_directory = config.get("calwebb_spec2_input_file",
"output_directory")
initial_input_file = config.get("calwebb_spec2_input_file", "input_file")
initial_input_file = os.path.join(output_directory, initial_input_file)
if os.path.isfile(initial_input_file):
detector = fits.getval(initial_input_file, "DETECTOR", 0)
else:
pytest.skip(
"Skipping " + step +
" because the initial input file given in PTT_config.cfg does not exist."
)
# set the name of the pipeline log to be read for times and renamed at the end for book-keeping
pipelog = "pipeline_" + detector + ".log"
# check if the filter is to be changed
change_filter_opaque = config.getboolean("calwebb_spec2_input_file",
"change_filter_opaque")
if change_filter_opaque:
is_filter_opaque, step_input_filename = change_filter_opaque2science.change_filter_opaque(
step_input_file, step=step)
if is_filter_opaque:
filter_opaque_msg = "With FILTER=OPAQUE, the calwebb_spec2 will run up to the extract_2d step. " \
"Extract_1d pytest now set to Skip."
print(filter_opaque_msg)
core_utils.add_completed_steps(txt_name, step, outstep_file_suffix,
step_completed, end_time)
#core_utils.convert_html2pdf() # convert the html report into a pdf file
# move the final reporting files to the working directory
core_utils.move_txt_files_2workdir(config, detector)
pytest.skip("Skipping " + step + " because FILTER=OPAQUE.")
if run_calwebb_spec2:
# read the output header
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
# end the timer to compute the step running time of PTT
PTT_end_time = time.time()
core_utils.start_end_PTT_time(txt_name,
start_time=None,
end_time=PTT_end_time)
# move the final reporting files to the working directory
core_utils.move_txt_files_2workdir(config, detector)
return hdul, step_output_file, run_pytests
else:
if run_pipe_step:
# Create the logfile for PTT, but erase the previous one if it exists
PTTcalspec2_log = os.path.join(
output_directory,
'PTT_calspec2_' + detector + '_' + step + '.log')
if imaging_mode:
PTTcalspec2_log = PTTcalspec2_log.replace(
'calspec2', 'calimage2')
if os.path.isfile(PTTcalspec2_log):
os.remove(PTTcalspec2_log)
print(
"Information outputed to screen from PTT will be logged in file: ",
PTTcalspec2_log)
for handler in logging.root.handlers[:]:
logging.root.removeHandler(handler)
logging.basicConfig(filename=PTTcalspec2_log, level=logging.INFO)
# print pipeline version
import jwst
pipeline_version = "\n *** Using jwst pipeline version: " + jwst.__version__ + " *** \n"
print(pipeline_version)
logging.info(pipeline_version)
if change_filter_opaque:
logging.info(filter_opaque_msg)
if os.path.isfile(step_input_file):
msg = " *** Step " + step + " set to True"
print(msg)
logging.info(msg)
stp = Extract1dStep()
# check that previous pipeline steps were run up to this point
core_utils.check_completed_steps(step, step_input_file)
# get the right configuration files to run the step
local_pipe_cfg_path = config.get("calwebb_spec2_input_file",
"local_pipe_cfg_path")
# start the timer to compute the step running time
start_time = time.time()
if local_pipe_cfg_path == "pipe_source_tree_code":
result = stp.call(step_input_file)
else:
result = stp.call(step_input_file,
config_file=local_pipe_cfg_path +
'/extract_1d.cfg')
result.save(step_output_file)
# end the timer to compute the step running time
end_time = repr(time.time() - start_time) # this is in seconds
msg = "Step " + step + " took " + end_time + " seconds to finish"
print(msg)
logging.info(msg)
# rename and move the pipeline log file
calspec2_pilelog = "calspec2_pipeline_" + step + "_" + detector + ".log"
if imaging_mode:
calspec2_pilelog = calspec2_pilelog.replace(
'calspec2', 'calimage2')
os.rename(pipelog,
os.path.join(output_directory, calspec2_pilelog))
else:
msg = " The input file does not exist. Skipping step."
print(msg)
logging.info(msg)
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed, end_time)
#core_utils.convert_html2pdf() # convert the html report into a pdf file
# end the timer to compute the step running time of PTT
PTT_end_time = time.time()
core_utils.start_end_PTT_time(txt_name,
start_time=None,
end_time=PTT_end_time)
# move the final reporting files to the working directory
core_utils.move_txt_files_2workdir(config, detector)
# skip the test if input file does not exist
pytest.skip("Skipping " + step +
" because the input file does not exist.")
else:
msg = "Skipping running pipeline step " + step
print(msg)
logging.info(msg)
# get the running time for this step
end_time = core_utils.get_stp_run_time_from_screenfile(
step, detector, output_directory)
# add the running time for this step
if os.path.isfile(step_output_file):
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
step_completed = True
core_utils.add_completed_steps(txt_name, step, outstep_file_suffix,
step_completed, end_time)
else:
step_completed = False
core_utils.add_completed_steps(txt_name, step, outstep_file_suffix,
step_completed, end_time)
# move the final reporting files to the working directory
core_utils.move_txt_files_2workdir(config, detector)
pytest.skip("Test skipped because input file " + step_output_file +
" does not exist.")
# get the total running time and print it in the file
total_time = repr(core_utils.get_time_to_run_pipeline(txt_name))
total_time_min = repr(round(float(total_time) / 60.0, 2))
msg = "\n\n **** The total time for the pipeline to run was " + total_time + " seconds."
print(msg)
logging.info(msg)
line2write = "{:<20} {:<20} {:<20} {:<20}".format(
'', '', 'total_time ',
total_time + ' =' + total_time_min + 'min')
print(line2write)
logging.info(line2write)
with open(txt_name, "a") as tf:
tf.write(line2write + "\n")
# convert the html report into a pdf file
#core_utils.convert_html2pdf()
# end the timer to compute the step running time of PTT
PTT_end_time = time.time()
core_utils.start_end_PTT_time(txt_name,
start_time=None,
end_time=PTT_end_time)
# move the final reporting text files to the working directory
core_utils.move_txt_files_2workdir(config, detector)
# rename and move the PTT log file
try:
calspec2_pipelog = "calspec2_pipeline_" + step + "_" + detector + ".log"
if imaging_mode:
calspec2_pipelog = calspec2_pipelog.replace(
'calspec2', 'calimage2')
pytest_workdir = TESTSDIR
logfile = glob(pytest_workdir + "/" + pipelog)[0]
os.rename(logfile, os.path.join(output_directory,
calspec2_pipelog))
except IndexError:
print("\n* WARNING: Something went wrong. Could not find a ",
pipelog, " file \n")
return hdul, step_output_file, run_pytests
def output_hdul(set_inandout_filenames, config):
# determine if the pipeline is to be run in full, per steps, or skipped
run_calwebb_spec2 = config.get("run_calwebb_spec2_in_full",
"run_calwebb_spec2")
if run_calwebb_spec2 == "skip":
print(
'\n * PTT finished processing run_calwebb_spec2 is set to skip. \n'
)
pytest.exit(
"Skipping pipeline run and tests for spec2, run_calwebb_spec2 is set to skip in PTT_config file."
)
elif "T" in run_calwebb_spec2:
run_calwebb_spec2 = True
else:
run_calwebb_spec2 = False
# get the general info
set_inandout_filenames_info = core_utils.read_info4outputhdul(
config, set_inandout_filenames)
step, txt_name, step_input_file, step_output_file, outstep_file_suffix = set_inandout_filenames_info
msa_shutter_conf = config.get("benchmark_intermediary_products",
"msa_conf_name")
msametfl = os.path.basename(msa_shutter_conf)
dflat_path = config.get("benchmark_intermediary_products", "dflat_path")
sflat_path = config.get("benchmark_intermediary_products", "sflat_path")
fflat_path = config.get("benchmark_intermediary_products", "fflat_path")
flattest_threshold_diff = config.get("additional_arguments",
"flattest_threshold_diff")
save_flattest_plot = config.getboolean("additional_arguments",
"save_flattest_plot")
write_flattest_files = config.getboolean("additional_arguments",
"write_flattest_files")
flattest_paths = [
step_output_file, msa_shutter_conf, dflat_path, sflat_path, fflat_path
]
flattest_switches = [
flattest_threshold_diff, save_flattest_plot, write_flattest_files
]
run_pipe_step = config.getboolean("run_pipe_steps", step)
# determine which tests are to be run
flat_field_completion_tests = config.getboolean(
"run_pytest", "_".join((step, "completion", "tests")))
flat_field_reffile_tests = config.getboolean(
"run_pytest", "_".join((step, "reffile", "tests")))
flat_field_validation_tests = config.getboolean(
"run_pytest", "_".join((step, "validation", "tests")))
run_pytests = [
flat_field_completion_tests, flat_field_reffile_tests,
flat_field_validation_tests
]
# if run_calwebb_spec2 is True calwebb_spec2 will be called, else individual steps will be ran
step_completed = False
end_time = '0.0'
# check if the filter is to be changed
change_filter_opaque = config.getboolean("calwebb_spec2_input_file",
"change_filter_opaque")
if change_filter_opaque:
is_filter_opaque, step_input_filename = change_filter_opaque2science.change_filter_opaque(
step_input_file, step=step)
if is_filter_opaque:
filter_opaque_msg = "With FILTER=OPAQUE, the calwebb_spec2 will run up to the extract_2d step. Flat " \
"Field pytest now set to Skip."
print(filter_opaque_msg)
core_utils.add_completed_steps(txt_name, step, outstep_file_suffix,
step_completed, end_time)
pytest.skip("Skipping " + step + " because FILTER=OPAQUE.")
# get the MSA shutter configuration file full path only for MOS data
output_directory = config.get("calwebb_spec2_input_file",
"output_directory")
initial_input_file = config.get("calwebb_spec2_input_file", "input_file")
initial_input_file = os.path.join(output_directory, initial_input_file)
if os.path.isfile(initial_input_file):
inhdu = core_utils.read_hdrfits(initial_input_file,
info=False,
show_hdr=False)
detector = fits.getval(initial_input_file, "DETECTOR", 0)
else:
pytest.skip(
"Skipping " + step +
" because the initial input file given in PTT_config.cfg does not exist."
)
if run_calwebb_spec2:
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
flattest_paths = [
step_output_file, msa_shutter_conf, dflat_path, sflat_path,
fflat_path
]
return hdul, step_output_file, flattest_paths, flattest_switches, run_pytests
else:
if run_pipe_step:
# Create the logfile for PTT, but erase the previous one if it exists
PTTcalspec2_log = os.path.join(
output_directory,
'PTT_calspec2_' + detector + '_' + step + '.log')
if os.path.isfile(PTTcalspec2_log):
os.remove(PTTcalspec2_log)
print(
"Information outputed to screen from PTT will be logged in file: ",
PTTcalspec2_log)
for handler in logging.root.handlers[:]:
logging.root.removeHandler(handler)
logging.basicConfig(filename=PTTcalspec2_log, level=logging.INFO)
# print pipeline version
import jwst
pipeline_version = "\n *** Using jwst pipeline version: " + jwst.__version__ + " *** \n"
print(pipeline_version)
logging.info(pipeline_version)
if change_filter_opaque:
logging.info(filter_opaque_msg)
if os.path.isfile(step_input_file):
msg = " *** Step " + step + " set to True"
print(msg)
logging.info(msg)
stp = FlatFieldStep()
# check that previous pipeline steps were run up to this point
core_utils.check_completed_steps(step, step_input_file)
if core_utils.check_MOS_true(inhdu):
# copy the MSA shutter configuration file into the pytest directory
subprocess.run(["cp", msa_shutter_conf, "."])
# start the timer to compute the step running time
ontheflyflat = step_output_file.replace(
"flat_field.fits", "interpolatedflat.fits")
msg1 = "Step product will be saved as: " + step_output_file
msg2 = "on-the-fly flat will be saved as: " + ontheflyflat
# get the right configuration files to run the step
print(msg1)
print(msg2)
logging.info(msg1)
logging.info(msg2)
local_pipe_cfg_path = config.get("calwebb_spec2_input_file",
"local_pipe_cfg_path")
# start the timer to compute the step running time
start_time = time.time()
if local_pipe_cfg_path == "pipe_source_tree_code":
stp.call(step_input_file,
output_file=step_output_file,
save_interpolated_flat=True)
else:
stp.call(step_input_file,
output_file=step_output_file,
save_interpolated_flat=True,
config_file=local_pipe_cfg_path +
'/flat_field.cfg')
# end the timer to compute the step running time
end_time = repr(time.time() - start_time) # this is in seconds
msg = "Step " + step + " took " + end_time + " seconds to finish"
print(msg)
logging.info(msg)
# move the on-the-fly flat to the working directory
subprocess.run(
["mv", os.path.basename(ontheflyflat), ontheflyflat])
# raname and move the flat_field output
subprocess.run([
"mv",
os.path.basename(step_output_file).replace(
"_flat_field.fits", "_flatfieldstep.fits"),
step_output_file
])
if core_utils.check_MOS_true(inhdu):
if TESTSDIR == os.path.dirname(msametfl):
# remove the copy of the MSA shutter configuration file
print("Removing MSA config file from: ", TESTSDIR)
subprocess.run(["rm", msametfl])
step_completed = True
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
# rename and move the pipeline log file
pipelog = "pipeline_" + detector + ".log"
try:
calspec2_pilelog = "calspec2_pipeline_" + step + "_" + detector + ".log"
pytest_workdir = TESTSDIR
logfile = glob(pytest_workdir + "/" + pipelog)[0]
os.rename(logfile,
os.path.join(output_directory, calspec2_pilelog))
except IndexError:
print(
"\n* WARNING: Something went wrong. Could not find a ",
pipelog, " file \n")
# add the running time for this step
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed, end_time)
return hdul, step_output_file, flattest_paths, flattest_switches, run_pytests
else:
msg = " The input file does not exist. Skipping step."
print(msg)
logging.info(msg)
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed, end_time)
pytest.skip("Skipping " + step +
" because the input file does not exist.")
else:
msg = "Skipping running pipeline step " + step
print(msg)
logging.info(msg)
end_time = core_utils.get_stp_run_time_from_screenfile(
step, detector, output_directory)
if os.path.isfile(step_output_file):
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
step_completed = True
# add the running time for this step
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed, end_time)
return hdul, step_output_file, flattest_paths, flattest_switches, run_pytests
else:
step_completed = False
# add the running time for this step
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed, end_time)
pytest.skip("Test skipped because input file " +
step_output_file + " does not exist.")
def output_hdul(set_inandout_filenames, config):
# determine if the pipeline is to be run in full, per steps, or skipped
run_calwebb_spec3 = config.get("calwebb_spec3", "run_calwebb_spec3")
print("run_calwebb_spec3 = ", run_calwebb_spec3)
if run_calwebb_spec3 == "skip":
print(
'\n * PTT finished processing run_calwebb_spec3 is set to skip. \n'
)
pytest.exit(
"Finished processing file, run_calwebb_spec3 is set to skip in configuration file."
)
else:
run_calwebb_spec3 = bool(run_calwebb_spec3)
# get the general info
step, step_input_filename, output_file, in_file_suffix, outstep_file_suffix, True_steps_suffix_map = set_inandout_filenames
output_directory = config.get("calwebb_spec2_input_file",
"output_directory")
txt_name = os.path.join(output_directory, True_steps_suffix_map)
step_input_file = os.path.join(output_directory, step_input_filename)
step_output_file = os.path.join(output_directory, output_file)
mode_used = config.get("calwebb_spec2_input_file", "mode_used").lower()
# start the timer to compute the step running time of NPTT
nptt_start_time = time.time()
# determine which steps are to be run, if not run in full
run_pipe_step = config.getboolean("run_pipe_steps", step)
# determine which tests are to be run
master_background_completion_tests = config.getboolean(
"run_pytest", "_".join((step, "completion", "tests")))
master_background_reffile_tests = config.getboolean(
"run_pytest", "_".join((step, "reffile", "tests")))
master_background_validation_tests = config.getboolean(
"run_pytest", "_".join((step, "validation", "tests")))
run_pytests = [
master_background_completion_tests, master_background_reffile_tests,
master_background_validation_tests
]
# Get the detector used
detector = fits.getval(step_input_file, "DETECTOR", 0)
# get main header from input file
inhdu = core_utils.read_hdrfits(step_input_file,
info=False,
show_hdr=False)
# if run_calwebb_spec3 is True, calwebb_spec3 will be called, else individual steps will be ran
step_completed = False
end_time = '0.0'
# get the shutter configuration file for MOS data only
msa_shutter_conf = "No shutter configuration file will be used."
if core_utils.check_MOS_true(inhdu):
msa_shutter_conf = config.get("esa_intermediary_products",
"msa_conf_name")
# check if the configuration shutter file name is in the header of the fits file and if not add it
msametfl = fits.getval(step_input_file, "MSAMETFL", 0)
if os.path.basename(msa_shutter_conf) != msametfl:
msametfl = os.path.basename(msa_shutter_conf)
fits.setval(step_input_file, "MSAMETFL", 0, value=msametfl)
# copy the MSA shutter configuration file to this directory if not the working directory
if os.getcwd() != os.path.dirname(msa_shutter_conf):
# copy the MSA shutter configuration file into the pytest directory
print("Removing MSA config file from: ", os.getcwd())
subprocess.run(["cp", msa_shutter_conf, "."])
# check if processing an image, then set proper variables
if mode_used in ('image', 'confirm', 'taconfirm', 'wata', 'msata', 'bota',
'focus', 'mimf'):
run_calwebb_spec3 = True
imaging_mode = True
print('\n * Image processing stops after calwebb_spec2. ')
# end script for imaging case
if imaging_mode:
print('\n * NPTT finished processing imaging mode. \n')
pytest.exit(
"Imaging does not get processed through calwebb_spec3.")
# get the name of the configuration file and run the pipeline
calwebb_spec3_cfg = config.get("calwebb_spec3", "calwebb_spec3_cfg")
# copy the configuration file to create the pipeline log
if not os.path.isfile(os.path.join(output_directory, "stpipe-log.cfg")):
stpipelogcfg = calwebb_spec3_cfg.replace("calwebb_spec3.cfg",
"stpipe-log.cfg")
subprocess.run(["cp", stpipelogcfg, os.getcwd()])
# run the pipeline
if run_calwebb_spec3:
# Create the logfile for NPTT, but remove the previous log file
npttcalspec3_log = os.path.join(output_directory,
'NPTT_calspec3_' + detector + '.log')
if os.path.isfile(npttcalspec3_log):
os.remove(npttcalspec3_log)
print(
"Spec3 screen information output from NPTT will be logged in file: ",
npttcalspec3_log)
for handler in logging.root.handlers[:]:
logging.root.removeHandler(handler)
logging.basicConfig(filename=npttcalspec3_log, level=logging.INFO)
logging.info(pipeline_version)
run_calwebb_spec3_msg = " *** Will run pipeline in full ... "
print(run_calwebb_spec3_msg)
logging.info(run_calwebb_spec3_msg)
# create the map
txt_name = "spec3_full_run_map_" + detector + ".txt"
if os.path.isfile(txt_name):
os.remove(txt_name)
master_background_utils.create_completed_steps_txtfile(
txt_name, step_input_file)
# start the timer to compute the step running time of NPTT
core_utils.start_end_PTT_time(txt_name,
start_time=nptt_start_time,
end_time=None)
if mode_used == "bots":
calwebb_spec3_cfg = calwebb_spec3_cfg.replace(
"calwebb_spec3.cfg", "calwebb_tso-spec3.cfg")
print(
'\nUsing the following configuration file to run TSO pipeline:'
)
else:
print(
'\nUsing the following configuration file to run spectroscopic pipeline:'
)
print(calwebb_spec3_cfg, '\n')
# start the timer to compute the step running time
start_time = time.time()
# run the pipeline
print('Running pipeline... \n')
Spec3Pipeline.call(step_input_file, config_file=calwebb_spec3_cfg)
# end the timer to compute calwebb_spec3 running time
end_time = repr(time.time() - start_time) # this is in seconds
calspec3_time = " * Pipeline took " + end_time + " seconds to finish.\n"
print(calspec3_time)
logging.info(calspec3_time)
# add the detector string to the name of the files and move them to the working directory
core_utils.add_detector2filename(output_directory, step_input_file)
# state name of the final spec3 _cal file
if "spec2" in step_input_file:
final_output_name = step_input_file.replace("spec2", "spec3")
else:
final_output_name = "final_output_spec3_" + detector + "_cal.fits"
final_output_name_msg = "\nThe final pipeline product was saved in: " + final_output_name
print(final_output_name_msg)
logging.info(final_output_name_msg)
# read the assign wcs fits file
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
# scihdul = core_utils.read_hdrfits(step_output_file, info=False, show_hdr=False, ext=1)
if core_utils.check_MOS_true(inhdu):
if os.getcwd() != os.path.dirname(msa_shutter_conf):
# remove the copy of the MSA shutter configuration file
print("Removing MSA config file from: ", os.getcwd())
subprocess.run(["rm", msametfl])
# rename and move the pipeline log file
calspec3_pipelog = "calspec3_pipeline_" + detector + ".log"
try:
path_where_pipeline_was_run = os.getcwd()
logfile = glob(path_where_pipeline_was_run + "/pipeline.log")[0]
print(logfile)
os.rename(logfile, os.path.join(output_directory,
calspec3_pipelog))
except IndexError:
print(
"\nWARNING: Something went wrong. Could not find a pipeline.log file \n"
)
# make sure we are able to find calspec3_pipelog either in the calwebb_spec3 directory or in the working dir
if not os.path.isfile(calspec3_pipelog):
calspec3_pipelog = os.path.join(output_directory, calspec3_pipelog)
# add the running time for all steps
step_running_times = core_utils.calculate_step_run_time(
calspec3_pipelog)
end_time_list = []
for stp in core_utils.step_string_dict:
if stp in step_running_times:
step_completed = True
step_time = step_running_times[stp]["run_time"]
out_suffix = core_utils.step_string_dict[stp]["suffix"]
core_utils.add_completed_steps(txt_name, stp, out_suffix,
step_completed, step_time)
end_time_list.append(step_time)
# print total running time in the text file and move it to the indicated directory
string2print = "pipeline_total_time"
if float(end_time) <= sum(end_time_list):
tot_time = repr(sum(end_time_list))
else:
tot_time = end_time
master_background_utils.print_time2file(txt_name, tot_time,
string2print)
nptt_runtimes_msg = "Pipeline and NPTT run times written in file: " + os.path.basename(
txt_name) + " in working directory. \n"
print(nptt_runtimes_msg)
logging.info(nptt_runtimes_msg)
# move the final reporting text files to the working directory
if os.getcwd() != output_directory:
core_utils.move_txt_files_2workdir(config, detector)
return hdul, step_output_file, msa_shutter_conf, run_pytests, mode_used
else:
# create the map but remove a previous one if it exists
if os.path.isfile(txt_name):
os.remove(txt_name)
master_background_utils.create_completed_steps_txtfile(
txt_name, step_input_file)
# start the timer to compute the step running time of NPTT
core_utils.start_end_PTT_time(txt_name,
start_time=nptt_start_time,
end_time=None)
msg = "\n Pipeline and NPTT run times will be written in file: " + os.path.basename(
txt_name) + " in working directory. \n"
print(msg)
logging.info(msg)
if run_pipe_step:
# Create the logfile for NPTT, but erase the previous one if it exists
npttcalspec3_log = os.path.join(
output_directory,
'NPTT_calspec3_' + detector + '_' + step + '.log')
if os.path.isfile(npttcalspec3_log):
os.remove(npttcalspec3_log)
print(
"Output information on screen from NPTT will be logged in file: ",
npttcalspec3_log)
for handler in logging.root.handlers[:]:
logging.root.removeHandler(handler)
logging.basicConfig(filename=npttcalspec3_log, level=logging.INFO)
logging.info(pipeline_version)
# check that previous pipeline steps were run up to this point
core_utils.check_completed_steps(step, step_input_file)
if os.path.isfile(step_input_file):
msg = " *** Step " + step + " set to True"
print(msg)
logging.info(msg)
stp = MasterBackgroundStep()
# get the right configuration files to run the step
local_pipe_cfg_path = config.get("calwebb_spec2_input_file",
"local_pipe_cfg_path")
# start the timer to compute the step running time
print("running pipeline...")
start_time = time.time()
if local_pipe_cfg_path == "pipe_source_tree_code":
result = stp.call(step_input_file)
else:
result = stp.call(step_input_file,
config_file=local_pipe_cfg_path +
'/master_background.cfg')
result.save(step_output_file)
# end the timer to compute the step running time
end_time = repr(time.time() - start_time) # this is in seconds
msg = "Step " + step + " took " + end_time + " seconds to finish"
print(msg)
logging.info(msg)
if core_utils.check_MOS_true(inhdu):
# remove the copy of the MSA shutter configuration file
if os.getcwd() != os.path.dirname(msa_shutter_conf):
print("Removing MSA config file from: ", os.getcwd())
subprocess.run(["rm", msametfl])
# rename and move the pipeline log file
pipelog = "pipeline_" + detector + ".log"
try:
calspec3_pipelog = "calspec3_pipeline_" + step + "_" + detector + ".log"
pytest_workdir = TESTSDIR
logfile = glob(pytest_workdir + "/" + pipelog)[0]
os.rename(logfile,
os.path.join(output_directory, calspec3_pipelog))
except IndexError:
print(
"\n* WARNING: Something went wrong. Could not find a ",
pipelog, " file \n")
else:
msg = "Skipping step. Input file " + step_input_file + " does not exit."
print(msg)
logging.info(msg)
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed, end_time)
pytest.skip("Skipping " + step +
" because the input file does not exist.")
else:
print("Skipping running pipeline step ", step)
# add the running time for this step
end_time = core_utils.get_stp_run_time_from_screenfile(
step, detector, output_directory)
if os.path.isfile(step_output_file):
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
step_completed = True
# add the running time for this step
core_utils.add_completed_steps(txt_name, step, outstep_file_suffix,
step_completed, end_time)
return hdul, step_output_file, msa_shutter_conf, run_pytests, mode_used
else:
step_completed = False
# add the running time for this step
core_utils.add_completed_steps(txt_name, step, outstep_file_suffix,
step_completed, end_time)
pytest.skip("Test skipped because input file " + step_output_file +
" does not exist.")
def output_hdul(set_inandout_filenames, config):
# determine if the pipeline is to be run in full, per steps, or skipped
run_calwebb_spec3 = config.get("calwebb_spec3", "run_calwebb_spec3")
print("run_calwebb_spec3 = ", run_calwebb_spec3)
if run_calwebb_spec3 == "skip":
print('\n * PTT finished processing run_calwebb_spec3 is set to skip. \n')
pytest.exit("Finished processing file, run_calwebb_spec3 is set to skip in configuration file.")
else:
run_calwebb_spec3 = bool(run_calwebb_spec3)
# get the general info
step, step_input_filename, output_file, in_file_suffix, outstep_file_suffix, True_steps_suffix_map = set_inandout_filenames
output_directory = config.get("calwebb_spec2_input_file", "output_directory")
txt_name = os.path.join(output_directory, True_steps_suffix_map)
step_input_file = os.path.join(output_directory, step_input_filename)
step_output_file = os.path.join(output_directory, output_file)
mode_used = config.get("calwebb_spec2_input_file", "mode_used").lower()
# start the timer to compute the step running time of NPTT
nptt_start_time = time.time()
# determine if steps is to be run
run_pipe_step = config.getboolean("run_pipe_steps", step)
# determine which tests are to be run
#completion_tests = config.getboolean("run_pytest", "_".join((step, "completion", "tests")))
#reffile_tests = config.getboolean("run_pytest", "_".join((step, "reffile", "tests")))
#validation_tests = config.getboolean("run_pytest", "_".join((step, "validation", "tests")))
#run_pytests = [completion_tests, reffile_tests, validation_tests]
# TODO - booleans for different tests have not been implemented in the configuration file yet
run_pytests = [False]
# Get the detector used
detector = fits.getval(step_input_file, "DETECTOR", 0)
# get main header from input file
inhdu = core_utils.read_hdrfits(step_input_file, info=False, show_hdr=False)
# if run_calwebb_spec3 is True, calwebb_spec3 will be called, else individual steps will be ran
step_completed = False
end_time = '0.0'
# only do this step if data is NOT IFU
if mode_used == 'ifu':
pytest.skip("Skipping test for step "+step+" because data is IFU.")
# run the pipeline
if run_calwebb_spec3:
hdul = core_utils.read_hdrfits(step_output_file, info=False, show_hdr=False)
return hdul, step_output_file, run_pytests, mode_used
else:
if run_pipe_step:
# Create the logfile for NPTT, but erase the previous one if it exists
npttcalspec3_log = os.path.join(output_directory, 'NPTT_calspec3_' + detector + '_' + step + '.log')
if os.path.isfile(npttcalspec3_log):
os.remove(npttcalspec3_log)
print("Output information on screen from NPTT will be logged in file: ", npttcalspec3_log)
for handler in logging.root.handlers[:]:
logging.root.removeHandler(handler)
logging.basicConfig(filename=npttcalspec3_log, level=logging.INFO)
logging.info(pipeline_version)
# check that previous pipeline steps were run up to this point
core_utils.check_completed_steps(step, step_input_file)
# TODO - need a function to gather all the same source files into a list to feed into this pipeline step
if os.path.isfile(step_input_file):
msg = " *** Step " + step + " set to True"
print(msg)
logging.info(msg)
print("Not running this individual step at the moment.")
"""
*** NEED THE COMMANDS TO RUN THIS STEP - requires a list of _cal.fits files
stp = MasterBackgroundStep()
# get the right configuration files to run the step
local_pipe_cfg_path = config.get("calwebb_spec2_input_file", "local_pipe_cfg_path")
# start the timer to compute the step running time
print("running pipeline...")
start_time = time.time()
if local_pipe_cfg_path == "pipe_source_tree_code":
result = stp.call(step_input_file)
else:
result = stp.call(step_input_file, config_file=local_pipe_cfg_path + '/master_background.cfg')
result.save(step_output_file)
# end the timer to compute the step running time
end_time = repr(time.time() - start_time) # this is in seconds
msg = "Step " + step + " took " + end_time + " seconds to finish"
print(msg)
logging.info(msg)
# rename and move the pipeline log file
pipelog = "pipeline_" + detector + ".log"
try:
calspec3_pipelog = "calspec3_pipeline_" + step + "_" + detector + ".log"
pytest_workdir = TESTSDIR
logfile = glob(pytest_workdir + "/" + pipelog)[0]
os.rename(logfile, os.path.join(output_directory, calspec3_pipelog))
except IndexError:
print("\n* WARNING: Something went wrong. Could not find a ", pipelog, " file \n")
"""
else:
msg = "Skipping step. Input file " + step_input_file + " does not exit."
print(msg)
logging.info(msg)
core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time)
pytest.skip("Skipping " + step + " because the input file does not exist.")
else:
print("Skipping running pipeline step ", step)
# add the running time for this step
end_time = core_utils.get_stp_run_time_from_screenfile(step, detector, output_directory)
if os.path.isfile(step_output_file):
hdul = core_utils.read_hdrfits(step_output_file, info=False, show_hdr=False)
step_completed = True
# add the running time for this step
core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time)
return hdul, step_output_file, run_pytests, mode_used
else:
step_completed = False
# add the running time for this step
core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time)
pytest.skip("Test skipped because input file "+step_output_file+" does not exist.")
def output_hdul(set_inandout_filenames, config):
# determine if the pipeline is to be run in full, per steps, or skipped
run_calwebb_spec2 = config.get("run_calwebb_spec2_in_full",
"run_calwebb_spec2")
if run_calwebb_spec2 == "skip":
print(
'\n * PTT finished processing run_calwebb_spec2 is set to skip. \n'
)
pytest.exit(
"Skipping pipeline run and tests for spec2, run_calwebb_spec2 is set to skip in PTT_config file."
)
elif "T" in run_calwebb_spec2:
run_calwebb_spec2 = True
else:
run_calwebb_spec2 = False
# get the general info
set_inandout_filenames_info = core_utils.read_info4outputhdul(
config, set_inandout_filenames)
step, txt_name, step_input_file, step_output_file, outstep_file_suffix = set_inandout_filenames_info
# determine which steps are to be run, if not run in full
run_pipe_step = config.getboolean("run_pipe_steps", step)
end_time = '0.0'
# Only run step if data is MOS or IFU
output_directory = config.get("calwebb_spec2_input_file",
"output_directory")
initial_input_file = config.get("calwebb_spec2_input_file", "input_file")
initial_input_file = os.path.join(output_directory, initial_input_file)
if os.path.isfile(initial_input_file):
inhdu = core_utils.read_hdrfits(initial_input_file,
info=False,
show_hdr=False)
detector = fits.getval(initial_input_file, "DETECTOR", 0)
else:
pytest.skip(
"Skipping " + step +
" because the initial input file given in PTT_config.cfg does not exist."
)
print("Is data MOS or IFU?", core_utils.check_MOS_true(inhdu),
core_utils.check_IFU_true(inhdu))
if core_utils.check_MOS_true(inhdu) or core_utils.check_IFU_true(inhdu):
# determine which tests are to be run
msa_flagging_completion_tests = config.getboolean(
"run_pytest", "_".join((step, "completion", "tests")))
# msa_flagging_reffile_tests = config.getboolean("run_pytest", "_".join((step, "reffile", "tests")))
msa_flagging_validation_tests = config.getboolean(
"run_pytest", "_".join((step, "validation", "tests")))
run_pytests = [
msa_flagging_completion_tests, msa_flagging_validation_tests
]
# get other info from the configuration file relevant only for this step
msa_flagging_operability_ref = config.get(
"benchmark_intermediary_products", "msa_flagging_operability_ref")
msa_flagging_threshold = config.get("additional_arguments",
"msa_flagging_threshold")
stellarity = config.get("additional_arguments", "stellarity")
save_msa_flagging_figs = config.getboolean("additional_arguments",
"save_msa_flagging_plots")
if stellarity == "source_type":
stellarity = None
testing_arguments = [
msa_flagging_operability_ref, msa_flagging_threshold, stellarity,
save_msa_flagging_figs
]
# if run_calwebb_spec2 is True calwebb_spec2 will be called, else individual steps will be ran
step_completed = False
if run_calwebb_spec2:
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
return hdul, step_output_file, step_input_file, run_pytests, testing_arguments
else:
if run_pipe_step:
if os.path.isfile(step_input_file):
msg = " The input file " + step_input_file + " exists... will run step " + step
print(msg)
logging.info(msg)
stp = MSAFlagOpenStep()
# check that previous pipeline steps were run up to this point
core_utils.check_completed_steps(step, step_input_file)
# Create the logfile for PTT, but erase the previous one if it exists
PTTcalspec2_log = os.path.join(
output_directory,
'PTT_calspec2_' + detector + '_' + step + '.log')
if os.path.isfile(PTTcalspec2_log):
os.remove(PTTcalspec2_log)
print(
"Information outputed to screen from PTT will be logged in file: ",
PTTcalspec2_log)
for handler in logging.root.handlers[:]:
logging.root.removeHandler(handler)
logging.basicConfig(filename=PTTcalspec2_log,
level=logging.INFO)
# print pipeline version
import jwst
pipeline_version = "\n *** Using jwst pipeline version: " + jwst.__version__ + " *** \n"
print(pipeline_version)
logging.info(pipeline_version)
# get the right configuration files to run the step
# local_pipe_cfg_path = config.get("calwebb_spec2_input_file", "local_pipe_cfg_path")
# start the timer to compute the step running time
start_time = time.time()
# if local_pipe_cfg_path == "pipe_source_tree_code":
result = stp.call(step_input_file)
# else:
# result = stp.call(step_input_file, config_file=local_pipe_cfg_path+'/NOCONFIGFI.cfg')
result.save(step_output_file)
# end the timer to compute the step running time
end_time = repr(time.time() -
start_time) # this is in seconds
msg = "Step " + step + " took " + end_time + " seconds to finish"
print(msg)
logging.info(msg)
# rename and move the pipeline log file
pipelog = "pipeline_" + detector + ".log"
try:
calspec2_pilelog = "calspec2_pipeline_" + step + "_" + detector + ".log"
pytest_workdir = TESTSDIR
logfile = glob(pytest_workdir + "/" + pipelog)[0]
os.rename(
logfile,
os.path.join(output_directory, calspec2_pilelog))
except IndexError:
print(
"\n* WARNING: Something went wrong. Could not find a ",
pipelog, " file \n")
# add the running time for this step
step_completed = True
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed, end_time)
return hdul, step_output_file, step_input_file, run_pytests, testing_arguments
else:
msg = " The input file does not exist. Skipping step."
print(msg)
logging.info(msg)
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed, end_time)
pytest.skip("Skipping " + step +
" because the input file does not exist.")
else:
msg = "Skipping running pipeline step " + step
print(msg)
logging.info(msg)
end_time = core_utils.get_stp_run_time_from_screenfile(
step, detector, output_directory)
if os.path.isfile(step_output_file):
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
step_completed = True
# add the running time for this step
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed, end_time)
return hdul, step_output_file, step_input_file, run_pytests, testing_arguments
else:
step_completed = False
# add the running time for this step
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed, end_time)
pytest.skip("Test skipped because input file " +
step_output_file + " does not exist.")
else:
pytest.skip("Skipping " + step +
" because data is neither MOS or IFU.")
def output_hdul(set_inandout_filenames, config):
# determine if the pipeline is to be run in full, per steps, or skipped
run_calwebb_spec2 = config.get("run_calwebb_spec2_in_full",
"run_calwebb_spec2")
if run_calwebb_spec2 == "skip":
print(
'\n * PTT finished processing run_calwebb_spec2 is set to skip. \n'
)
pytest.exit(
"Skipping pipeline run and tests for spec2, run_calwebb_spec2 is set to skip in PTT_config file."
)
elif "T" in run_calwebb_spec2:
run_calwebb_spec2 = True
else:
run_calwebb_spec2 = False
# get the general info
set_inandout_filenames_info = core_utils.read_info4outputhdul(
config, set_inandout_filenames)
step, txt_name, step_input_file, step_output_file, outstep_file_suffix = set_inandout_filenames_info
run_pipe_step = config.getboolean("run_pipe_steps", step)
# determine which tests are to be run
pathloss_completion_tests = config.getboolean(
"run_pytest", "_".join((step, "completion", "tests")))
pathloss_reffile_tests = config.getboolean(
"run_pytest", "_".join((step, "reffile", "tests")))
pathloss_validation_tests = config.getboolean(
"run_pytest", "_".join((step, "validation", "tests")))
run_pytests = [
pathloss_completion_tests, pathloss_reffile_tests,
pathloss_validation_tests
]
pathloss_threshold_diff = config.get("additional_arguments",
"pathloss_threshold_diff")
save_flattest_plot = config.getboolean("additional_arguments",
"save_pathloss_plot")
write_flattest_files = config.getboolean("additional_arguments",
"write_pathloss_files")
pathloss_switches = [
pathloss_threshold_diff, save_flattest_plot, write_flattest_files
]
# if run_calwebb_spec2 is True calwebb_spec2 will be called, else individual steps will be ran
step_completed = False
end_time = '0.0'
# check if the filter is to be changed
change_filter_opaque = config.getboolean("calwebb_spec2_input_file",
"change_filter_opaque")
if change_filter_opaque:
is_filter_opaque, step_input_filename = change_filter_opaque2science.change_filter_opaque(
step_input_file, step=step)
if is_filter_opaque:
filter_opaque_msg = "With FILTER=OPAQUE, the calwebb_spec2 will run up to the extract_2d step. " \
"Pathloss pytest now set to Skip."
print(filter_opaque_msg)
core_utils.add_completed_steps(txt_name, step, outstep_file_suffix,
step_completed, end_time)
pytest.skip("Skipping " + step + " because FILTER=OPAQUE.")
# only run this step if data is not BOTS
output_directory = config.get("calwebb_spec2_input_file",
"output_directory")
initial_input_file = config.get("calwebb_spec2_input_file", "input_file")
initial_input_file = os.path.join(output_directory, initial_input_file)
if os.path.isfile(initial_input_file):
inhdu = core_utils.read_hdrfits(initial_input_file,
info=False,
show_hdr=False)
detector = fits.getval(initial_input_file, "DETECTOR", 0)
else:
pytest.skip(
"Skipping " + step +
" because the initial input file given in PTT_config.cfg does not exist."
)
if not core_utils.check_BOTS_true(inhdu):
if run_calwebb_spec2:
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
return hdul, step_output_file, step_input_file, run_pytests, pathloss_switches
else:
if run_pipe_step:
# Create the logfile for PTT, but erase the previous one if it exists
PTTcalspec2_log = os.path.join(
output_directory,
'PTT_calspec2_' + detector + '_' + step + '.log')
if os.path.isfile(PTTcalspec2_log):
os.remove(PTTcalspec2_log)
print(
"Information outputed to screen from PTT will be logged in file: ",
PTTcalspec2_log)
for handler in logging.root.handlers[:]:
logging.root.removeHandler(handler)
logging.basicConfig(filename=PTTcalspec2_log,
level=logging.INFO)
# print pipeline version
import jwst
pipeline_version = "\n *** Using jwst pipeline version: " + jwst.__version__ + " *** \n"
print(pipeline_version)
logging.info(pipeline_version)
if change_filter_opaque:
logging.info(filter_opaque_msg)
if os.path.isfile(step_input_file):
msg = " *** Step " + step + " set to True"
print(msg)
logging.info(msg)
stp = PathLossStep()
# check that previous pipeline steps were run up to this point
core_utils.check_completed_steps(step, step_input_file)
# get the right configuration files to run the step
local_pipe_cfg_path = config.get(
"calwebb_spec2_input_file", "local_pipe_cfg_path")
# start the timer to compute the step running time
start_time = time.time()
if local_pipe_cfg_path == "pipe_source_tree_code":
result = stp.call(step_input_file)
else:
result = stp.call(step_input_file,
config_file=local_pipe_cfg_path +
'/pathloss.cfg')
result.save(step_output_file)
# For the moment, the pipeline is using the wrong reference file for slit 400A1, so the
# file needs to be re-processed with the right reference file BUT we are
# skipping this slit
"""
if core_utils.check_FS_true(inhdu):
print("\n * WARNING: For the moment, the wrong reference file is being used for "
"processing slit 400A1. The file will be re-processed ")
# The file can manually be re-run with
# $ strun jwst.pathloss.PathLossStep final_output_caldet1_NRS1_srctype.fits
# --override_pathloss=jwst-nirspec-a400.plrf.fits
pathloss_400a1 = step_input_file.replace("srctype.fits", "pathloss_400A1.fits")
reffile_400a1 = "jwst-nirspec-a400.plrf.fits"
print("Re-processing slit with new reference file: ", reffile_400a1)
pl = PathLossStep()
pl.override_pathloss = reffile_400a1
pl_result = pl.run(step_input_file)
pl_result.save(pathloss_400a1)
#import subprocess
#subprocess.run(["mv", step_input_file.replace("srctype", "pathlossstep"), pathloss_400a1])
print("Saved pipeline re-processed file as: ", pathloss_400a1)
"""
# end the timer to compute calwebb_spec2 running time
end_time = repr(time.time() -
start_time) # this is in seconds
msg = " * calwebb_spec2 took " + end_time + " seconds to finish."
print(msg)
logging.info(msg)
step_completed = True
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
# rename and move the pipeline log file
pipelog = "pipeline_" + detector + ".log"
try:
calspec2_pilelog = "calspec2_pipeline_" + step + "_" + detector + ".log"
pytest_workdir = TESTSDIR
logfile = glob(pytest_workdir + "/" + pipelog)[0]
os.rename(
logfile,
os.path.join(output_directory, calspec2_pilelog))
except IndexError:
print(
"\n* WARNING: Something went wrong. Could not find a ",
pipelog, " file \n")
# add the running time for this step
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed, end_time)
return hdul, step_output_file, step_input_file, run_pytests, pathloss_switches
else:
msg = " The input file does not exist. Skipping step."
print(msg)
logging.info(msg)
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed, end_time)
pytest.skip("Skipping " + step +
" because the input file does not exist.")
else:
msg = "Skipping running pipeline step " + step
print(msg)
logging.info(msg)
end_time = core_utils.get_stp_run_time_from_screenfile(
step, detector, output_directory)
if os.path.isfile(step_output_file):
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
step_completed = True
# add the running time for this step
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed, end_time)
return hdul, step_output_file, step_input_file, run_pytests, pathloss_switches
else:
step_completed = False
# add the running time for this step
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed, end_time)
pytest.skip("Test skipped because input file " +
step_output_file + " does not exist.")
else:
pytest.skip("Skipping " + step + " because data is BOTS.")
def output_hdul(set_inandout_filenames, config):
# determine if the pipeline is to be run in full, per steps, or skipped
run_calwebb_spec2 = config.get("run_calwebb_spec2_in_full",
"run_calwebb_spec2")
if run_calwebb_spec2 == "skip":
print(
'\n * PTT finished processing run_calwebb_spec2 is set to skip. \n'
)
pytest.exit(
"Skipping pipeline run and tests for spec2, run_calwebb_spec2 is set to skip in PTT_config file."
)
elif "T" in run_calwebb_spec2:
run_calwebb_spec2 = True
else:
run_calwebb_spec2 = False
# get the general info
set_inandout_filenames_info = core_utils.read_info4outputhdul(
config, set_inandout_filenames)
step, txt_name, step_input_file, step_output_file, outstep_file_suffix = set_inandout_filenames_info
# determine if the spec2 pipeline should be run or not - obtained from CRDS repo:
# https://github.com/spacetelescope/crds/blob/7fc216e73bd81d334b5d98d165be2caa3b6175e1/crds/jwst/jwst_system_crdscfg_b7.5.yaml
exp_type = fits.getval(step_input_file, 'EXP_TYPE')
allowed_exp_type_values = [
'NRS_AUTOFLAT', 'NRS_AUTOWAVE', 'NRS_BRIGHTOBJ', 'NRS_CONFIRM',
'NRS_FIXEDSLIT', 'NRS_FOCUS', 'NRS_IFU', 'NRS_IMAGE', 'NRS_LAMP',
'NRS_MIMF', 'NRS_MSASPEC', 'NRS_MSATA', 'NRS_TACONFIRM', 'NRS_TACQ',
'NRS_TASLIT', 'NRS_WATA'
]
if exp_type not in allowed_exp_type_values:
print(
"EXP_TYPE=", exp_type,
" is not in exposure types expected for running the stage 2 pipeline: ",
allowed_exp_type_values)
pytest.exit(
"The EXP_TYPE value of this data is not expected to run in the stage 2 pipeline."
)
# start the timer to compute the step running time of PTT
PTT_start_time = time.time()
# check if the filter is to be changed
change_filter_opaque = config.getboolean("calwebb_spec2_input_file",
"change_filter_opaque")
if change_filter_opaque:
_, step_input_file = change_filter_opaque2science.change_filter_opaque(
step_input_file)
change_filter_opaque_msg = " * With FILTER=OPAQUE, the calwebb_spec2 will run up to the extract_2d step. " \
"Further steps will be skipped. \n"
print(change_filter_opaque_msg)
# determine which steps are to be run, if not run in full
run_pipe_step = config.getboolean("run_pipe_steps", step)
# determine which tests are to be run
assign_wcs_completion_tests = config.getboolean(
"run_pytest", "_".join((step, "completion", "tests")))
assign_wcs_reffile_tests = config.getboolean(
"run_pytest", "_".join((step, "reffile", "tests")))
assign_wcs_validation_tests = config.getboolean(
"run_pytest", "_".join((step, "validation", "tests")))
run_pytests = [
assign_wcs_completion_tests, assign_wcs_reffile_tests,
assign_wcs_validation_tests
]
# get other relevant info from PTT config file
compare_assign_wcs_and_extract_2d_with_esa = config.getboolean(
"benchmark_intermediary_products",
"compare_assign_wcs_and_extract_2d_with_esa")
esa_files_path = config.get("benchmark_intermediary_products",
"esa_files_path")
data_directory = config.get("calwebb_spec2_input_file", "data_directory")
truth_file = os.path.join(
data_directory,
config.get("benchmark_intermediary_products", "truth_file_assign_wcs"))
if compare_assign_wcs_and_extract_2d_with_esa:
truth_file = esa_files_path
print("Will use this 'truth' file to compare result of assign_wcs: ")
print(truth_file)
wcs_threshold_diff = config.get("additional_arguments",
"wcs_threshold_diff")
save_wcs_plots = config.getboolean("additional_arguments",
"save_wcs_plots")
output_directory = config.get("calwebb_spec2_input_file",
"output_directory")
# Get the detector used
detector = fits.getval(step_input_file, "DETECTOR", 0)
# get main header from input file
inhdu = core_utils.read_hdrfits(step_input_file,
info=False,
show_hdr=False)
# if run_calwebb_spec2 is True calwebb_spec2 will be called, else individual steps will be ran
step_completed = False
end_time = '0.0'
# Check if data is IFU that the Image Model keyword is correct
mode_used = config.get("calwebb_spec2_input_file", "mode_used").lower()
if mode_used == "ifu":
DATAMODL = fits.getval(step_input_file, "DATAMODL", 0)
if DATAMODL != "IFUImageModel":
fits.setval(step_input_file, "DATAMODL", 0, value="IFUImageModel")
print("DATAMODL keyword changed to IFUImageModel.")
# get the shutter configuration file for MOS data only
msa_shutter_conf = "No shutter configuration file will be used."
if core_utils.check_MOS_true(inhdu):
msa_shutter_conf = config.get("benchmark_intermediary_products",
"msa_conf_name")
# check if the configuration shutter file name is in the header of the fits file and if not add it
msametfl = fits.getval(step_input_file, "MSAMETFL", 0)
if os.path.basename(msa_shutter_conf) != msametfl:
msametfl = os.path.basename(msa_shutter_conf)
fits.setval(step_input_file, "MSAMETFL", 0, value=msametfl)
# check if processing an image, then set proper variables
imaging_mode = False
if mode_used in ('image', 'confirm', 'taconfirm', 'wata', 'msata', 'bota',
'focus', 'mimf'):
run_calwebb_spec2 = True
imaging_mode = True
print(
'\n * Image processing will only be run in full with PTT. All intermediary products will be saved.'
)
print(
' -> For now, all pytests will be skipped since there are now image-specific routines yet. \n'
)
# TODO: add imaging tests
# get the name of the configuration file and run the pipeline
calwebb_spec2_cfg = config.get("run_calwebb_spec2_in_full",
"calwebb_spec2_cfg")
# copy the configuration file to create the pipeline log
pipelog = assign_wcs_utils.set_pipe_log(calwebb_spec2_cfg, detector)
# run the pipeline
if run_calwebb_spec2:
# Create the logfile for PTT, but remove the previous log file
PTTcalspec2_log = os.path.join(output_directory,
'PTT_calspec2_' + detector + '.log')
if imaging_mode:
PTTcalspec2_log = PTTcalspec2_log.replace('calspec2', 'calimage2')
if os.path.isfile(PTTcalspec2_log):
os.remove(PTTcalspec2_log)
print(
"Information outputed to screen from PTT will be logged in file: ",
PTTcalspec2_log)
for handler in logging.root.handlers[:]:
logging.root.removeHandler(handler)
logging.basicConfig(filename=PTTcalspec2_log, level=logging.INFO)
logging.info(pipeline_version)
if change_filter_opaque:
logging.info(change_filter_opaque_msg)
run_calwebb_spec2_msg = " *** Will run pipeline in full ... "
print(run_calwebb_spec2_msg)
logging.info(run_calwebb_spec2_msg)
# create the map
txt_name = "spec2_full_run_map_" + detector + ".txt"
if os.path.isfile(txt_name):
os.remove(txt_name)
assign_wcs_utils.create_completed_steps_txtfile(
txt_name, step_input_file)
# start the timer to compute the step running time of PTT
core_utils.start_end_PTT_time(txt_name,
start_time=PTT_start_time,
end_time=None)
if mode_used == "bots":
calwebb_spec2_cfg = calwebb_spec2_cfg.replace(
"calwebb_spec2.cfg", "calwebb_tso-spec2.cfg")
print(
'\nUsing the following configuration file to run TSO pipeline:'
)
print(calwebb_spec2_cfg, '\n')
if imaging_mode:
calwebb_image2_cfg = calwebb_spec2_cfg.replace(
"calwebb_spec2.cfg", "calwebb_image2.cfg")
print(
'\nUsing the following configuration file to run imaging pipeline:'
)
print(calwebb_image2_cfg, '\n')
else:
print(
'\nUsing the following configuration file to run spectroscopic pipeline:'
)
print(calwebb_spec2_cfg, '\n')
input_file = config.get("calwebb_spec2_input_file", "input_file")
if "_uncal_rate" in input_file:
input_file = input_file.replace("_uncal_rate", "")
if "_uncal" in input_file:
input_file = input_file.replace("_uncal", "")
# final_output_name = "final_output_spec2_"+detector+"_cal.fits"
if detector.lower() not in input_file.lower():
input_file = input_file.replace(".fits", "_" + detector + ".fits")
final_output_name = input_file.replace(".fits", "_cal.fits")
if '_rate' in final_output_name:
final_output_name = final_output_name.replace('_rate', '')
if core_utils.check_MOS_true(inhdu):
# copy the MSA shutter configuration file into the pytest directory
subprocess.run(["cp", msa_shutter_conf, "."])
# start the timer to compute the step running time
start_time = time.time()
# run the pipeline
print('Running pipeline... \n')
try:
if not imaging_mode:
Spec2Pipeline.call(
step_input_file,
config_file=calwebb_spec2_cfg) # , logcfg=stpipelogcfg)
else:
Image2Pipeline.call(step_input_file,
config_file=calwebb_image2_cfg)
except:
pytest.exit(
"\n * The pipeline crashed, probably due to no open slits fall on detector"
)
"""
# For the moment, the pipeline is using the wrong reference file for slit 400A1, so the
# file needs to be re-processed with the right reference file
if core_utils.check_FS_true(inhdu):
print("\n * WARNING: For the moment, the wrong reference file is being used for "
"processing slit 400A1. The file will be re-processed ")
# print(" $ jwst.pathloss.PathLossStep final_output_caldet1_NRS1_srctype.fits "
# "--override_pathloss=jwst-nirspec-a400.plrf.fits \n")
pathloss_400a1 = step_input_file.replace("srctype.fits", "pathloss_400A1.fits")
reffile_400a1 = "jwst-nirspec-a400.plrf.fits"
print("Re-processing slit with new reference file: ", reffile_400a1)
pl = PathLossStep()
pl.override_pathloss = reffile_400a1
pl.run(step_input_file)
subprocess.run(["mv", step_input_file.replace("srctype", "pathlossstep"), pathloss_400a1])
print("Saved pipeline re-processed file as: ", pathloss_400a1)
"""
# end the timer to compute calwebb_spec2 running time
end_time = repr(time.time() - start_time) # this is in seconds
calspec2_time = " * Pipeline took " + end_time + " seconds to finish.\n"
print(calspec2_time)
logging.info(calspec2_time)
# remove the copy of the MSA shutter configuration file
if core_utils.check_MOS_true(inhdu):
if TESTSDIR == os.path.dirname(msametfl):
print("Removing MSA config file from: ", TESTSDIR)
subprocess.run(["rm", msametfl])
# add the detector string to the name of the files and move them to the working directory
core_utils.add_detector2filename(output_directory, step_input_file)
final_output_name_msg = "\nThe final pipeline product was saved as: " + final_output_name
print(final_output_name_msg)
logging.info(final_output_name_msg)
# read the assign wcs fits file
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
# scihdul = core_utils.read_hdrfits(step_output_file, info=False, show_hdr=False, ext=1)
# rename and move the pipeline log file
try:
calspec2_pipelog = "calspec2_pipeline_" + detector + ".log"
if imaging_mode:
calspec2_pipelog = calspec2_pipelog.replace(
'calspec2', 'calimage2')
path_where_pipeline_was_run = os.getcwd()
logfile = glob(path_where_pipeline_was_run + "/" + pipelog)[0]
print(logfile)
os.rename(logfile, os.path.join(output_directory,
calspec2_pipelog))
except IndexError:
print("\nWARNING: Something went wrong. Could not find a ",
pipelog, " file \n")
# make sure we are able to find calspec2_pipelog either in the calwebb_spec2 directory or in the working dir
if not os.path.isfile(calspec2_pipelog):
calspec2_pipelog = os.path.join(output_directory, calspec2_pipelog)
# add the running time for all steps
step_running_times = core_utils.calculate_step_run_time(
calspec2_pipelog)
end_time_list = []
for stp in core_utils.step_string_dict:
if stp in step_running_times:
step_completed = True
step_time = step_running_times[stp]["run_time"]
out_suffix = core_utils.step_string_dict[stp]["suffix"]
core_utils.add_completed_steps(txt_name, stp, out_suffix,
step_completed, step_time)
end_time_list.append(step_time)
# print total running time in the text file and move it to the indicated directory
string2print = "pipeline_total_time"
if float(end_time) <= sum(end_time_list):
tot_time = repr(sum(end_time_list))
else:
tot_time = end_time
assign_wcs_utils.print_time2file(txt_name, tot_time, string2print)
PTT_runtimes_msg = "Pipeline and PTT run times written in file: " + os.path.basename(
txt_name) + " in working directory. \n"
print(PTT_runtimes_msg)
logging.info(PTT_runtimes_msg)
# move the final reporting text files to the working directory
core_utils.move_txt_files_2workdir(config, detector)
# rename the _cal file
last_suffix = os.path.basename(step_output_file).split(
".fits")[0].split("_")[-1]
cal_file = step_output_file.replace(last_suffix, "_cal")
if os.path.isfile(cal_file):
if "caldet1" in cal_file:
spec2_cal_file = cal_file.replace("caldet1", "spec2")
os.rename(cal_file, spec2_cal_file)
# end script for imaging case
if imaging_mode:
print('\nPTT finished processing imaging mode. \n')
pytest.exit(
"Skipping pytests for now because they need to be written for imaging mode."
)
return hdul, step_output_file, msa_shutter_conf, truth_file, wcs_threshold_diff, save_wcs_plots, \
run_pytests, mode_used, compare_assign_wcs_and_extract_2d_with_esa
else:
# create the map but remove a previous one if it exists
if os.path.isfile(txt_name):
os.remove(txt_name)
assign_wcs_utils.create_completed_steps_txtfile(
txt_name, step_input_file)
# start the timer to compute the step running time of PTT
core_utils.start_end_PTT_time(txt_name,
start_time=PTT_start_time,
end_time=None)
msg = "\n Pipeline and PTT run times will be written in file: " + os.path.basename(
txt_name) + " in working directory. \n"
print(msg)
logging.info(msg)
if run_pipe_step:
# Create the logfile for PTT, but erase the previous one if it exists
PTTcalspec2_log = os.path.join(
output_directory,
'PTT_calspec2_' + detector + '_' + step + '.log')
if imaging_mode:
PTTcalspec2_log = PTTcalspec2_log.replace(
'calspec2', 'calimage2')
if os.path.isfile(PTTcalspec2_log):
os.remove(PTTcalspec2_log)
print(
"Information outputed to screen from PTT will be logged in file: ",
PTTcalspec2_log)
for handler in logging.root.handlers[:]:
logging.root.removeHandler(handler)
logging.basicConfig(filename=PTTcalspec2_log, level=logging.INFO)
logging.info(pipeline_version)
if change_filter_opaque:
logging.info(change_filter_opaque_msg)
# check that previous pipeline steps were run up to this point
core_utils.check_completed_steps(step, step_input_file)
if os.path.isfile(step_input_file):
msg = " *** Step " + step + " set to True"
print(msg)
logging.info(msg)
stp = AssignWcsStep()
if core_utils.check_MOS_true(inhdu):
if os.getcwd() != output_directory:
# copy the MSA shutter configuration file into the pytest directory
subprocess.run(["cp", msa_shutter_conf, "."])
# get the right configuration files to run the step
local_pipe_cfg_path = config.get("calwebb_spec2_input_file",
"local_pipe_cfg_path")
# start the timer to compute the step running time
print("running pipeline...")
start_time = time.time()
if local_pipe_cfg_path == "pipe_source_tree_code":
result = stp.call(step_input_file)
else:
result = stp.call(step_input_file,
config_file=local_pipe_cfg_path +
'/assign_wcs.cfg')
result.save(step_output_file)
# end the timer to compute the step running time
end_time = repr(time.time() - start_time) # this is in seconds
msg = "Step " + step + " took " + end_time + " seconds to finish"
print(msg)
logging.info(msg)
if core_utils.check_MOS_true(inhdu):
if TESTSDIR == os.path.dirname(msametfl):
# remove the copy of the MSA shutter configuration file
print("Removing MSA config file from: ", TESTSDIR)
subprocess.run(["rm", msametfl])
# rename and move the pipeline log file
try:
calspec2_pipelog = "calspec2_pipeline_" + step + "_" + detector + ".log"
if imaging_mode:
calspec2_pipelog = calspec2_pipelog.replace(
'calspec2', 'calimage2')
pytest_workdir = TESTSDIR
logfile = glob(pytest_workdir + "/" + pipelog)[0]
os.rename(logfile,
os.path.join(output_directory, calspec2_pipelog))
except IndexError:
print(
"\n* WARNING: Something went wrong. Could not find a ",
pipelog, " file \n")
else:
msg = "Skipping step. Input file " + step_input_file + " does not exit."
print(msg)
logging.info(msg)
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed, end_time)
pytest.skip("Skipping " + step +
" because the input file does not exist.")
else:
print("Skipping running pipeline step ", step)
# add the running time for this step
end_time = core_utils.get_stp_run_time_from_screenfile(
step, detector, output_directory)
if os.path.isfile(step_output_file):
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
step_completed = True
# add the running time for this step
core_utils.add_completed_steps(txt_name, step, outstep_file_suffix,
step_completed, end_time)
return hdul, step_output_file, msa_shutter_conf, truth_file, wcs_threshold_diff, save_wcs_plots, \
run_pytests, mode_used, compare_assign_wcs_and_extract_2d_with_esa
else:
step_completed = False
# add the running time for this step
core_utils.add_completed_steps(txt_name, step, outstep_file_suffix,
step_completed, end_time)
pytest.skip("Test skipped because input file " + step_output_file +
" does not exist.")
