def check_output_is_zero(output_hdul):
run_step = output_hdul[2]
step_input_file = output_hdul[3]
step_output_file = output_hdul[1]
# Only run test if data is IFU or MSA
inhdu = core_utils.read_hdrfits(step_input_file,
info=False,
show_hdr=False)
if core_utils.check_IFU_true(inhdu) or core_utils.check_MOS_true(inhdu):
if run_step:
# set specifics for the test
msa_imprint_structure = copy.deepcopy(step_input_file)
result_to_check = step_output_file.replace(".fits",
"_zerotest.fits")
# run the step with the specifics
stp = ImprintStep()
res = stp.call(step_input_file, msa_imprint_structure)
res.save(result_to_check)
# check that the end product of image - image is zero
c = fits.getdata(result_to_check)
substraction = sum(c.flatten())
result = False
if substraction == 0.0:
result = True
# erase test output file
subprocess.run(["rm", result_to_check])
return result
pytest.skip(
"Skipping validation check for imprint_subtract. Step set to False in configuration file."
)
def test_imprint_nirspec(_bigdata):
"""
Regression test of imprint step performed on NIRSpec MSA data.
"""
output_file_base, output_file = add_suffix('imprint1_output.fits', 'imprint')
try:
os.remove(output_file)
except:
pass
ImprintStep.call(_bigdata+'/nirspec/test_imprint/jw00038001001_01101_00001_NRS1_rate.fits',
_bigdata+'/nirspec/test_imprint/NRSMOS-MODEL-21_NRS1_rate.fits',
output_file=output_file_base, name='imprint')
h = pf.open(output_file)
href = pf.open(_bigdata+'/nirspec/test_imprint/jw00038001001_01101_00001_NRS1_imprint.fits')
newh = pf.HDUList([h['primary'],h['sci'],h['err'],h['dq']])
newhref = pf.HDUList([href['primary'],href['sci'],href['err'],href['dq']])
result = pf.diff.FITSDiff(newh, newhref,
ignore_keywords = ['DATE','CAL_VER','CAL_VCS','CRDS_VER','CRDS_CTX'],
rtol = 0.00001)
assert result.identical, result.report()
def check_output_is_zero(output_hdul):
"""
This test is a simple subtraction of the input file minus a copy of the input file (instead of the actual
MSA imprint file. The output is expected to be zero.
:param output_hdul: list
:return: result: boolean
"""
# output_hdul = hdul, step_output_file, step_input_file, run_pytests
step_input_file = output_hdul[2]
step_output_file = output_hdul[1]
# Only run test if data is IFU or MSA
inhdu = core_utils.read_hdrfits(step_input_file,
info=False,
show_hdr=False)
if core_utils.check_IFU_true(inhdu) or core_utils.check_MOS_true(inhdu):
# set specifics for the test
msa_imprint_structure = copy.deepcopy(step_input_file)
result_to_check = step_output_file.replace(".fits", "_zerotest.fits")
# run the step with the specifics
stp = ImprintStep()
res = stp.call(step_input_file, msa_imprint_structure)
res.save(result_to_check)
# check that the end product of image - image is zero
c = fits.getdata(result_to_check)
subtraction = sum(c.flatten())
result = False
if subtraction == 0.0:
result = True
# erase test output file
subprocess.run(["rm", result_to_check])
return result
def output_hdul(set_inandout_filenames, config):
set_inandout_filenames_info = core_utils.read_info4outputhdul(
config, set_inandout_filenames)
step, txt_name, step_input_file, step_output_file, run_calwebb_spec2, outstep_file_suffix = set_inandout_filenames_info
# determine which steps are to be run, if not run in full
run_pipe_step = config.getboolean("run_pipe_steps", step)
# determine which tests are to be run
imprint_subtract_completion_tests = config.getboolean(
"run_pytest", "_".join((step, "completion", "tests")))
imprint_subtract_numerical_tests = config.getboolean(
"run_pytest", "_".join((step, "numerical", "tests")))
#imprint_subtract_validation_tests = config.getboolean("run_pytest", "_".join((step, "validation", "tests")))
run_pytests = [
imprint_subtract_completion_tests, imprint_subtract_numerical_tests
] #, imprint_subtract_validation_tests]
end_time = '0.0'
# Only run step if data is IFU or MSA
working_directory = config.get("calwebb_spec2_input_file",
"working_directory")
initial_input_file = config.get("calwebb_spec2_input_file", "input_file")
initial_input_file = os.path.join(working_directory, initial_input_file)
detector = fits.getval(initial_input_file, "DETECTOR", 0)
calspec2_pilelog = "calspec2_pipeline_" + step + "_" + detector + ".log"
pytest_workdir = os.getcwd()
if os.path.isfile(initial_input_file):
inhdu = core_utils.read_hdrfits(initial_input_file,
info=False,
show_hdr=False)
else:
pytest.skip(
"Skipping " + step +
" because the initial input file given in PTT_config.cfg does not exist."
)
if core_utils.check_IFU_true(inhdu) or core_utils.check_MOS_true(inhdu):
# if run_calwebb_spec2 is True calwebb_spec2 will be called, else individual steps will be ran
step_completed = False
if run_calwebb_spec2:
if os.path.isfile(step_output_file):
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
else:
pytest.skip("Skipping " + step +
" because the output file does not exist.")
return hdul, step_output_file, run_pipe_step, step_input_file, run_pytests
else:
if run_pipe_step:
# Create the logfile for PTT, but erase the previous one if it exists
PTTcalspec2_log = os.path.join(
working_directory,
'PTT_calspec2_' + detector + '_' + step + '.log')
if os.path.isfile(PTTcalspec2_log):
os.remove(PTTcalspec2_log)
print(
"Information outputed to screen from PTT will be logged in file: ",
PTTcalspec2_log)
for handler in logging.root.handlers[:]:
logging.root.removeHandler(handler)
logging.basicConfig(filename=PTTcalspec2_log,
level=logging.INFO)
# print pipeline version
import jwst
pipeline_version = "\n *** Using jwst pipeline version: " + jwst.__version__ + " *** \n"
print(pipeline_version)
logging.info(pipeline_version)
if os.path.isfile(step_input_file):
msg = " The input file " + step_input_file + " exists... will run step " + step
print(msg)
logging.info(msg)
msa_imprint_structure = config.get(
"additional_arguments", "msa_imprint_structure")
msg = "msa_imprint_structure file: " + msa_imprint_structure
print(msg)
logging.info(msg)
if not os.path.isfile(msa_imprint_structure):
print(
" Need msa_imprint_structure file to continue. Step will be skipped."
)
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed,
end_time)
pytest.skip(
"Skipping " + step +
" because msa_imprint_structure file in the configuration file does not exist."
)
else:
msg = "*** Step " + step + " set to True"
print(msg)
logging.info(msg)
stp = ImprintStep()
# check that previous pipeline steps were run up to this point
core_utils.check_completed_steps(step, step_input_file)
# get the right configuration files to run the step
local_pipe_cfg_path = config.get(
"calwebb_spec2_input_file", "local_pipe_cfg_path")
# start the timer to compute the step running time
start_time = time.time()
if local_pipe_cfg_path == "pipe_source_tree_code":
result = stp.call(step_input_file,
msa_imprint_structure)
else:
result = stp.call(step_input_file,
msa_imprint_structure,
config_file=local_pipe_cfg_path +
'/imprint.cfg')
if result is not None:
result.save(step_output_file)
# end the timer to compute the step running time
end_time = repr(time.time() -
start_time) # this is in seconds
msg = "Step " + step + " took " + end_time + " seconds to finish"
print(msg)
logging.info(msg)
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
step_completed = True
else:
hdul = core_utils.read_hdrfits(step_input_file,
info=False,
show_hdr=False)
# rename and move the pipeline log file
try:
logfile = glob(pytest_workdir + "/pipeline.log")[0]
os.rename(
logfile,
os.path.join(working_directory,
calspec2_pilelog))
except:
IndexError
# add the running time for this step
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed,
end_time)
return hdul, step_output_file, run_pipe_step, step_input_file, run_pytests
else:
msg = " The input file does not exist. Skipping step."
print(msg)
logging.info(msg)
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed, end_time)
pytest.skip("Skipping " + step +
" because the input file does not exist.")
else:
msg = "Skipping running pipeline step " + step
print(msg)
logging.info(msg)
end_time = core_utils.get_stp_run_time_from_screenfile(
step, detector, working_directory)
if os.path.isfile(step_output_file):
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
step_completed = True
# add the running time for this step
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed, end_time)
return hdul, step_output_file, step_input_file, run_pytests
else:
step_completed = False
# add the running time for this step
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed, end_time)
pytest.skip()
else:
pytest.skip("Skipping " + step +
" because data is neither IFU or MOS.")
def output_hdul(set_inandout_filenames, config):
set_inandout_filenames_info = core_utils.read_info4outputhdul(
config, set_inandout_filenames)
step, txt_name, step_input_file, step_output_file, run_calwebb_spec2, outstep_file_suffix = set_inandout_filenames_info
skip_runing_pipe_step = config.getboolean("tests_only", "_".join(
(step, "tests")))
# Only run step if data is IFU or MSA
inhdu = core_utils.read_hdrfits(step_input_file,
info=False,
show_hdr=False)
end_time = '0.0'
run_step = config.getboolean("steps", step)
if core_utils.check_IFU_true(inhdu) or core_utils.check_MOS_true(inhdu):
stp = ImprintStep()
# if run_calwebb_spec2 is True calwebb_spec2 will be called, else individual steps will be ran
step_completed = False
if run_calwebb_spec2:
# read the assign wcs fits file
local_step_output_file = core_utils.read_completion_to_full_run_map(
"full_run_map.txt", step)
hdul = core_utils.read_hdrfits(local_step_output_file,
info=False,
show_hdr=False)
# move the output file into the working directory
working_directory = config.get("calwebb_spec2_input_file",
"working_directory")
step_output_file = os.path.join(working_directory,
local_step_output_file)
print("Step product was saved as: ", step_output_file)
subprocess.run(["mv", local_step_output_file, step_output_file])
return hdul, run_step, step_input_file, step_output_file
else:
if run_step:
print("*** Step " + step + " set to True")
if os.path.isfile(step_input_file):
print(" The input file ", step_input_file,
"exists... will run step " + step)
msa_imprint_structure = config.get(
"additional_arguments", "msa_imprint_structure")
print("msa_imprint_structure file: ",
msa_imprint_structure)
if not os.path.isfile(msa_imprint_structure):
print(
" Need msa_imprint_structure file to continue. Step will be skipped."
)
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed,
end_time)
pytest.skip(
"Skipping " + step +
" because msa_imprint_structure file in the configuration file does not exist."
)
else:
if not skip_runing_pipe_step:
# get the right configuration files to run the step
local_pipe_cfg_path = config.get(
"calwebb_spec2_input_file",
"local_pipe_cfg_path")
# start the timer to compute the step running time
start_time = time.time()
if local_pipe_cfg_path == "pipe_source_tree_code":
result = stp.call(step_input_file,
msa_imprint_structure)
else:
result = stp.call(
step_input_file,
msa_imprint_structure,
config_file=local_pipe_cfg_path +
'/imprint.cfg')
if result is not None:
result.save(step_output_file)
# end the timer to compute the step running time
end_time = repr(
time.time() -
start_time) # this is in seconds
print("Step " + step + " took " + end_time +
" seconds to finish")
hdul = core_utils.read_hdrfits(
step_output_file,
info=False,
show_hdr=False)
step_completed = True
else:
hdul = core_utils.read_hdrfits(step_input_file,
info=False,
show_hdr=False)
else:
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
step_completed = True
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed,
end_time)
return hdul, run_step, step_input_file, step_output_file
else:
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed, end_time)
pytest.skip("Skipping " + step +
" because the input file does not exist.")
else:
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed, end_time)
pytest.skip("Skipping " + step +
". Step set to False in configuration file.")
else:
pytest.skip("Skipping " + step +
" because data is neither IFU or MOS.")