def test_extract1d_miri(_bigdata):
"""
Regression test of extract_1d step performed on MIRI LRS fixed-slit data.
"""
suffix = 'extract_1d'
output_file_base, output_file = add_suffix('extract1d1_output.fits', suffix)
try:
os.remove(output_file)
except:
pass
Extract1dStep.call(_bigdata+'/miri/test_extract1d/jw00035001001_01101_00001_mirimage_photom.fits',
smoothing_length=0,
output_file=output_file_base, suffix=suffix
)
h = pf.open(output_file)
href = pf.open(_bigdata+'/miri/test_extract1d/jw00035001001_01101_00001_mirimage_x1d.fits')
newh = pf.HDUList([h['primary'],h['extract1d',1]])
newhref = pf.HDUList([href['primary'],href['extract1d',1]])
result = pf.diff.FITSDiff(newh,
newhref,
ignore_keywords = ['DATE','CAL_VER','CAL_VCS','CRDS_VER','CRDS_CTX'],
rtol = 0.00001
)
assert result.identical, result.report()
def test_extract1d_nirspec(_bigdata):
"""Regression test of extract_1d step performed on NIRSpec fixed slit data.
"""
output_file_base, output_file = add_suffix('extract1d1_output.fits', 'extract_1d')
try:
os.remove(output_file)
except:
pass
Extract1dStep.call(_bigdata + '/nirspec/test_extract_1d/jw00023001001_01101_00001_NRS1_cal.fits',
smoothing_length=0, bkg_order=0,
output_file=output_file_base, name='extract_1d')
h = fits.open(output_file)
href = fits.open(_bigdata + '/nirspec/test_extract_1d/jw00023001001_01101_00001_NRS1_spec.fits')
newh = fits.HDUList([h['primary'],
h[('extract1d',1)],
h[('extract1d',2)],
h[('extract1d',3)],
h[('extract1d',4)]])
newhref = fits.HDUList([href['primary'],
href[('extract1d',1)],
href[('extract1d',2)],
href[('extract1d',3)],
href[('extract1d',4)]])
result = fits.diff.FITSDiff(newh,
newhref,
ignore_keywords=['DATE','CAL_VER','CAL_VCS','CRDS_VER','CRDS_CTX'],
rtol=0.00001)
assert result.identical, result.report()
def test_extract1d_miri2(_bigdata):
"""
Regression test of extract_1d step performed on MIRI LRS slitless data.
"""
suffix = 'extract_1d'
output_file_base, output_file = add_suffix('extract1d2_output.fits',
suffix)
Extract1dStep.call(
_bigdata +
'/miri/test_extract1d/jw80600012001_02101_00003_mirimage_photom.fits',
smoothing_length=0,
output_file=output_file_base,
suffix=suffix)
h = fits.open(output_file)
href = fits.open(
_bigdata +
'/miri/test_extract1d/jw80600012001_02101_00003_mirimage_x1d.fits')
newh = fits.HDUList([
h['primary'], h['extract1d', 1], h['extract1d', 2], h['extract1d', 3],
h['extract1d', 4]
])
newhref = fits.HDUList([
href['primary'], href['extract1d', 1], href['extract1d', 2],
href['extract1d', 3], href['extract1d', 4]
])
result = fits.diff.FITSDiff(
newh,
newhref,
ignore_keywords=['DATE', 'CAL_VER', 'CAL_VCS', 'CRDS_VER', 'CRDS_CTX'],
rtol=0.00001)
assert result.identical, result.report()
def output_hdul(set_inandout_filenames, config):
set_inandout_filenames_info = core_utils.read_info4outputhdul(
config, set_inandout_filenames)
step, txt_name, step_input_file, step_output_file, run_calwebb_spec2, outstep_file_suffix = set_inandout_filenames_info
skip_runing_pipe_step = config.getboolean("tests_only", "_".join(
(step, "tests")))
stp = Extract1dStep()
# if run_calwebb_spec2 is True calwebb_spec2 will be called, else individual steps will be ran
step_completed = False
end_time = '0.0'
if run_calwebb_spec2:
# read the assign wcs fits file
local_step_output_file = core_utils.read_completion_to_full_run_map(
"full_run_map.txt", step)
hdul = core_utils.read_hdrfits(local_step_output_file,
info=False,
show_hdr=False)
# move the output file into the working directory
working_directory = config.get("calwebb_spec2_input_file",
"working_directory")
step_output_file = os.path.join(working_directory,
local_step_output_file)
print("Step product was saved as: ", step_output_file)
subprocess.run(["mv", local_step_output_file, step_output_file])
return hdul
else:
if config.getboolean("steps", step):
print("*** Step " + step + " set to True")
if os.path.isfile(step_input_file):
if not skip_runing_pipe_step:
# get the right configuration files to run the step
local_pipe_cfg_path = config.get(
"calwebb_spec2_input_file", "local_pipe_cfg_path")
# start the timer to compute the step running time
start_time = time.time()
if local_pipe_cfg_path == "pipe_source_tree_code":
result = stp.call(step_input_file)
else:
result = stp.call(step_input_file,
config_file=local_pipe_cfg_path +
'/extract_1d.cfg')
result.save(step_output_file)
# end the timer to compute the step running time
end_time = repr(time.time() -
start_time) # this is in seconds
print("Step " + step + " took " + end_time +
" seconds to finish")
step_completed = True
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed, end_time)
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
# get the total running time and print it in the file
total_time = repr(
core_utils.get_time_to_run_pipeline(txt_name))
total_time_min = repr(round(float(total_time) / 60.0, 2))
print("The total time for the pipeline to run was " +
total_time + " seconds.")
#print (" ( = "+total_time_min+" minutes )")
line2write = "{:<20} {:<20} {:<20} {:<20}".format(
'', '', 'total_time ',
total_time + ' =' + total_time_min + 'min')
print(line2write)
with open(txt_name, "a") as tf:
tf.write(line2write + "\n")
return hdul
else:
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed, end_time)
pytest.skip("Skipping " + step +
" because the input file does not exist.")
else:
core_utils.add_completed_steps(txt_name, step, outstep_file_suffix,
step_completed, end_time)
pytest.skip("Skipping " + step +
". Step set to False in configuration file.")
def output_hdul(set_inandout_filenames, config):
# determine if the pipeline is to be run in full, per steps, or skipped
run_calwebb_spec2 = config.get("run_calwebb_spec2_in_full",
"run_calwebb_spec2")
if run_calwebb_spec2 == "skip":
print(
'\n * PTT finished processing run_calwebb_spec2 is set to skip. \n'
)
pytest.exit(
"Skipping pipeline run and tests for spec2, run_calwebb_spec2 is set to skip in PTT_config file."
)
elif "T" in run_calwebb_spec2:
run_calwebb_spec2 = True
else:
run_calwebb_spec2 = False
# get the general info
set_inandout_filenames_info = core_utils.read_info4outputhdul(
config, set_inandout_filenames)
step, txt_name, step_input_file, step_output_file, outstep_file_suffix = set_inandout_filenames_info
run_pipe_step = config.getboolean("run_pipe_steps", step)
# determine which tests are to be run
extract_1d_completion_tests = config.getboolean(
"run_pytest", "_".join((step, "completion", "tests")))
extract_1d_reffile_tests = config.getboolean(
"run_pytest", "_".join((step, "reffile", "tests")))
# extract_1d_validation_tests = config.getboolean("run_pytest", "_".join((step, "validation", "tests")))
run_pytests = [extract_1d_completion_tests,
extract_1d_reffile_tests] #, extract_1d_validation_tests]
# Make sure at this point the MSA shutter configuration file is removed from the calwebb_spec2_pytests directory
msa_shutter_conf = config.get("benchmark_intermediary_products",
"msa_conf_name")
msametfl = os.path.basename(msa_shutter_conf)
# remove the copy of the MSA shutter configuration file
inhdu = core_utils.read_hdrfits(step_input_file,
info=False,
show_hdr=False)
if core_utils.check_MOS_true(inhdu):
if TESTSDIR == os.path.dirname(msametfl):
print("Removing MSA config file from: ", TESTSDIR)
os.remove(msametfl)
# check if processing an image, then set proper variables
imaging_mode = False
mode_used = config.get("calwebb_spec2_input_file", "mode_used").lower()
if mode_used in ('image', 'confirm', 'taconfirm', 'wata', 'msata', 'bota',
'focus', 'mimf'):
run_calwebb_spec2 = True
imaging_mode = True
# if run_calwebb_spec2 is True calwebb_spec2 will be called, else individual steps will be ran
step_completed = False
end_time = '0.0'
# Get the detector used
output_directory = config.get("calwebb_spec2_input_file",
"output_directory")
initial_input_file = config.get("calwebb_spec2_input_file", "input_file")
initial_input_file = os.path.join(output_directory, initial_input_file)
if os.path.isfile(initial_input_file):
detector = fits.getval(initial_input_file, "DETECTOR", 0)
else:
pytest.skip(
"Skipping " + step +
" because the initial input file given in PTT_config.cfg does not exist."
)
# set the name of the pipeline log to be read for times and renamed at the end for book-keeping
pipelog = "pipeline_" + detector + ".log"
# check if the filter is to be changed
change_filter_opaque = config.getboolean("calwebb_spec2_input_file",
"change_filter_opaque")
if change_filter_opaque:
is_filter_opaque, step_input_filename = change_filter_opaque2science.change_filter_opaque(
step_input_file, step=step)
if is_filter_opaque:
filter_opaque_msg = "With FILTER=OPAQUE, the calwebb_spec2 will run up to the extract_2d step. " \
"Extract_1d pytest now set to Skip."
print(filter_opaque_msg)
core_utils.add_completed_steps(txt_name, step, outstep_file_suffix,
step_completed, end_time)
#core_utils.convert_html2pdf() # convert the html report into a pdf file
# move the final reporting files to the working directory
core_utils.move_txt_files_2workdir(config, detector)
pytest.skip("Skipping " + step + " because FILTER=OPAQUE.")
if run_calwebb_spec2:
# read the output header
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
# end the timer to compute the step running time of PTT
PTT_end_time = time.time()
core_utils.start_end_PTT_time(txt_name,
start_time=None,
end_time=PTT_end_time)
# move the final reporting files to the working directory
core_utils.move_txt_files_2workdir(config, detector)
return hdul, step_output_file, run_pytests
else:
if run_pipe_step:
# Create the logfile for PTT, but erase the previous one if it exists
PTTcalspec2_log = os.path.join(
output_directory,
'PTT_calspec2_' + detector + '_' + step + '.log')
if imaging_mode:
PTTcalspec2_log = PTTcalspec2_log.replace(
'calspec2', 'calimage2')
if os.path.isfile(PTTcalspec2_log):
os.remove(PTTcalspec2_log)
print(
"Information outputed to screen from PTT will be logged in file: ",
PTTcalspec2_log)
for handler in logging.root.handlers[:]:
logging.root.removeHandler(handler)
logging.basicConfig(filename=PTTcalspec2_log, level=logging.INFO)
# print pipeline version
import jwst
pipeline_version = "\n *** Using jwst pipeline version: " + jwst.__version__ + " *** \n"
print(pipeline_version)
logging.info(pipeline_version)
if change_filter_opaque:
logging.info(filter_opaque_msg)
if os.path.isfile(step_input_file):
msg = " *** Step " + step + " set to True"
print(msg)
logging.info(msg)
stp = Extract1dStep()
# check that previous pipeline steps were run up to this point
core_utils.check_completed_steps(step, step_input_file)
# get the right configuration files to run the step
local_pipe_cfg_path = config.get("calwebb_spec2_input_file",
"local_pipe_cfg_path")
# start the timer to compute the step running time
start_time = time.time()
if local_pipe_cfg_path == "pipe_source_tree_code":
result = stp.call(step_input_file)
else:
result = stp.call(step_input_file,
config_file=local_pipe_cfg_path +
'/extract_1d.cfg')
result.save(step_output_file)
# end the timer to compute the step running time
end_time = repr(time.time() - start_time) # this is in seconds
msg = "Step " + step + " took " + end_time + " seconds to finish"
print(msg)
logging.info(msg)
# rename and move the pipeline log file
calspec2_pilelog = "calspec2_pipeline_" + step + "_" + detector + ".log"
if imaging_mode:
calspec2_pilelog = calspec2_pilelog.replace(
'calspec2', 'calimage2')
os.rename(pipelog,
os.path.join(output_directory, calspec2_pilelog))
else:
msg = " The input file does not exist. Skipping step."
print(msg)
logging.info(msg)
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed, end_time)
#core_utils.convert_html2pdf() # convert the html report into a pdf file
# end the timer to compute the step running time of PTT
PTT_end_time = time.time()
core_utils.start_end_PTT_time(txt_name,
start_time=None,
end_time=PTT_end_time)
# move the final reporting files to the working directory
core_utils.move_txt_files_2workdir(config, detector)
# skip the test if input file does not exist
pytest.skip("Skipping " + step +
" because the input file does not exist.")
else:
msg = "Skipping running pipeline step " + step
print(msg)
logging.info(msg)
# get the running time for this step
end_time = core_utils.get_stp_run_time_from_screenfile(
step, detector, output_directory)
# add the running time for this step
if os.path.isfile(step_output_file):
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
step_completed = True
core_utils.add_completed_steps(txt_name, step, outstep_file_suffix,
step_completed, end_time)
else:
step_completed = False
core_utils.add_completed_steps(txt_name, step, outstep_file_suffix,
step_completed, end_time)
# move the final reporting files to the working directory
core_utils.move_txt_files_2workdir(config, detector)
pytest.skip("Test skipped because input file " + step_output_file +
" does not exist.")
# get the total running time and print it in the file
total_time = repr(core_utils.get_time_to_run_pipeline(txt_name))
total_time_min = repr(round(float(total_time) / 60.0, 2))
msg = "\n\n **** The total time for the pipeline to run was " + total_time + " seconds."
print(msg)
logging.info(msg)
line2write = "{:<20} {:<20} {:<20} {:<20}".format(
'', '', 'total_time ',
total_time + ' =' + total_time_min + 'min')
print(line2write)
logging.info(line2write)
with open(txt_name, "a") as tf:
tf.write(line2write + "\n")
# convert the html report into a pdf file
#core_utils.convert_html2pdf()
# end the timer to compute the step running time of PTT
PTT_end_time = time.time()
core_utils.start_end_PTT_time(txt_name,
start_time=None,
end_time=PTT_end_time)
# move the final reporting text files to the working directory
core_utils.move_txt_files_2workdir(config, detector)
# rename and move the PTT log file
try:
calspec2_pipelog = "calspec2_pipeline_" + step + "_" + detector + ".log"
if imaging_mode:
calspec2_pipelog = calspec2_pipelog.replace(
'calspec2', 'calimage2')
pytest_workdir = TESTSDIR
logfile = glob(pytest_workdir + "/" + pipelog)[0]
os.rename(logfile, os.path.join(output_directory,
calspec2_pipelog))
except IndexError:
print("\n* WARNING: Something went wrong. Could not find a ",
pipelog, " file \n")
return hdul, step_output_file, run_pytests