def get_reference_files(datamodel):
"""Get the reference files associated with the step."""
refs = {}
step = AssignWcsStep()
for reftype in AssignWcsStep.reference_file_types:
val = step.get_reference_file(datamodel, reftype)
if val == 'N/A':
refs[reftype] = None
else:
refs[reftype] = val
return refs
def get_reference_files(datamodel):
refs = {}
step = AssignWcsStep()
for reftype in AssignWcsStep.reference_file_types:
val = step.get_reference_file(datamodel, reftype)
print(reftype, val)
if val.strip() == 'N/A':
refs[reftype] = None
else:
refs[reftype] = val
print(refs)
return refs
def run_msa_flagging_testing(input_file, msa_flagging_threshold=99.5, rate_obj=None,
stellarity=None, operability_ref=None, source_type=None,
save_figs=False, show_figs=True, debug=False):
"""
This is the validation function for the msa flagging step.
:param input_file: string, fits file output from the msa_flagging step
:param msa_flagging_threshold: float, percentage for all slits with more than 100 pixels
:param rate_obj: object, the stage 1 pipeline output object
:param stellarity: float, stellarity number fro 0.0 to 1.0
:param operability_ref: string, msa failed open - operability - reference file
:param source_type: string, options are point, extended, unknown
:param save_figs: boolean
:param show_figs: boolean
:param debug: boolean
:return:
FINAL_TEST_RESULT: boolean, True if smaller than or equal to threshold
result_msg: string, message with reason for passing, failing, or skipped
log_msgs: list, diagnostic strings to be printed in log
"""
# start the list of messages that will be added to the log file
log_msgs = []
# start the timer
msa_flagging_test_start_time = time.time()
# get the data model
if isinstance(input_file, str):
msaflag = datamodels.open(input_file)
else:
msaflag = input_file
if debug:
print('got MSA flagging datamodel!')
# set up generals for all the plots
font = {'weight': 'normal',
'size': 12}
matplotlib.rc('font', **font)
# plot full image
fig = plt.figure(figsize=(9, 9))
norm = ImageNormalize(msaflag.data, vmin=0., vmax=50., stretch=AsinhStretch())
plt.imshow(msaflag.data, norm=norm, aspect=1.0, origin='lower', cmap='viridis')
# Show and/or save figures
detector = msaflag.meta.instrument.detector
datadir = None
if save_figs:
file_basename = os.path.basename(input_file.replace("_msa_flagging.fits", ""))
datadir = os.path.dirname(input_file)
t = (file_basename, "MSA_flagging_full_detector.png")
plt_name = "_".join(t)
plt_name = os.path.join(datadir, plt_name)
plt.savefig(plt_name)
print('Figure saved as: ', plt_name)
if show_figs:
plt.show()
plt.close()
# read in DQ flags from MSA_flagging product
# find all pixels that have been flagged by this step as MSA_FAILED_OPEN -> DQ array value = 536870912
# https://jwst-pipeline.readthedocs.io/en/latest/jwst/references_general/references_general.html?highlight=536870912#data-quality-flags
dq_flag = 536870912
msaflag_1d = msaflag.dq.flatten()
index_opens = np.squeeze(np.asarray(np.where(msaflag_1d & dq_flag)))
if debug:
print("DQ array at 167, 1918: ", msaflag.dq[167, 1918])
# np.set_printoptions(threshold=sys.maxsize) # print all elements in array
print("Index where Failed Open shutters exist: ", np.shape(index_opens), index_opens)
# execute script that creates an MSA metafile for the failed open shutters
# read operability reference file
"""
crds_path = os.environ.get('CRDS_PATH')
if crds_path is None:
print("(msa_flagging_testing): The environment variable CRDS_PATH is not defined. To set it, follow the "
"instructions at: \n"
" https://github.com/spacetelescope/nirspec_pipe_testing_tool")
exit()
"""
crds_path = "https://jwst-crds.stsci.edu/unchecked_get/references/jwst/"
op_ref_file = "jwst_nirspec_msaoper_0001.json"
if operability_ref is None:
ref_file = os.path.join(crds_path, op_ref_file)
urllib.request.urlretrieve(ref_file, op_ref_file)
else:
op_ref_file = operability_ref
if "http" not in op_ref_file:
if not os.path.isfile(op_ref_file):
result_msg = "Skipping msa_flagging test because the operability reference file does not exist: " + \
op_ref_file
print(result_msg)
log_msgs.append(result_msg)
result = 'skip'
return result, result_msg, log_msgs
if debug:
print("Using this operability reference file: ", op_ref_file)
with open(op_ref_file) as f:
msaoper_dict = json.load(f)
msaoper = msaoper_dict["msaoper"]
# find the failed open shutters
failedopens = [(c["Q"], c["x"], c["y"]) for c in msaoper if c["state"] == 'open']
if debug:
print("Failed Open shutters: ", failedopens)
# unpack the list of tuples into separate lists for MSA quadrant, row, column locations
quads, allrows, allcols = zip(*failedopens)
# stellarity -- internal lamps are uniform illumination, so set to 0
# ** if considering a point source, need to change this to 1, or actual value if known
if source_type is None:
# srctyapt = fits.getval(input_file, 'SRCTYAPT') # previously used
srctyapt = msaflag.meta.target.source_type_apt
else:
srctyapt = source_type.upper()
if stellarity is None:
if "POINT" in srctyapt:
stellarity = 1.0
else:
stellarity = 0.0
else:
stellarity = float(stellarity)
# create MSA metafile with F/O shutters
if datadir is not None:
fometafile = os.path.join(datadir, 'fopens_metafile_msa.fits')
else:
fometafile = 'fopens_metafile_msa.fits'
if not os.path.isfile(fometafile):
pattnum = msaflag.meta.dither.position_number
create_metafile_fopens(fometafile, allcols, allrows, quads, stellarity, failedopens,
pattnum, save_fig=save_figs, show_fig=show_figs, debug=debug)
# run assign_wcs on the science exposure using F/O metafile
# change MSA metafile name in header to match the F/O metafile name
if isinstance(input_file, str):
rate_file = input_file.replace("msa_flagging", "rate")
if not os.path.isfile(rate_file):
# if a _rate.fits file does not exist try the usual name
rate_file = os.path.join(datadir, 'final_output_caldet1_'+detector+'.fits')
if not os.path.isfile(rate_file):
result_msg = "Skipping msa_flagging test because no rate fits file was found in directory: " + datadir
print(result_msg)
log_msgs.append(result_msg)
result = 'skip'
return result, result_msg, log_msgs
if debug:
print("Will run assign_wcs with new Failed Open fits file on this file: ", rate_file)
rate_mdl = datamodels.ImageModel(rate_file)
else:
rate_mdl = rate_obj
if debug:
print("MSA metadata file in initial rate file: ", rate_mdl.meta.instrument.msa_metadata_file)
rate_mdl.meta.instrument.msa_metadata_file = fometafile
if debug:
print("New MSA metadata file in rate file: ", rate_mdl.meta.instrument.msa_metadata_file)
# force the exp_type of this new model to MSA, even if IFU so that nrs_wcs_set_input pipeline function works
if "ifu" in msaflag.meta.exposure.type.lower():
rate_mdl.meta.exposure.type = 'NRS_MSASPEC'
# run assign_wcs; use +/-0.45 for the y-limits because the default is too big (0.6 including buffer)
stp = AssignWcsStep()
awcs_fo = stp.call(rate_mdl, slit_y_low=-0.45, slit_y_high=0.45)
# get the slits from the F/O processing run
slits_list = awcs_fo.meta.wcs.get_transform('gwa', 'slit_frame').slits
# prepare arrays to hold info needed for validation test
allsizes = np.zeros(len(slits_list))
allchecks = np.zeros(len(slits_list))
# loop over the slits and compare pixel bounds with the flagged pixels from the original product
for i, slit in enumerate(slits_list):
try:
name = slit.name
except AttributeError:
name = i
print("\nWorking with slit/slice: ", name)
if "IFU" not in msaflag.meta.exposure.type.upper():
print("Slit min and max in y direction: ", slit.ymin, slit.ymax)
# get the WCS object for this particular slit
wcs_slice = nirspec.nrs_wcs_set_input(awcs_fo, name)
# get the bounding box for the 2D subwindow, round to nearest integer, and convert to integer
bbox = np.rint(wcs_slice.bounding_box)
bboxint = bbox.astype(int)
print("bounding box rounded to next integer: ", bboxint)
i1 = bboxint[0, 0]
i2 = bboxint[0, 1]
i3 = bboxint[1, 0]
i4 = bboxint[1, 1]
# make array of pixel locations within bounding box
x, y = np.mgrid[i1:i2, i3:i4]
index_1d = np.ravel_multi_index([[y], [x]], (2048, 2048))
# get the slity WCS parameter to find which pixels are located in the actual spectrum
det2slit = wcs_slice.get_transform('detector', 'slit_frame')
slitx, slity, _ = det2slit(x, y)
print("Max value in slity array (ignoring NANs): ", np.nanmax(slity))
index_trace = np.squeeze(index_1d)[~np.isnan(slity)]
n_overlap = np.sum(np.isin(index_opens, index_trace))
overlap_percentage = round(n_overlap/index_trace.size*100., 1)
if debug:
print("Size of index_trace= ", index_trace.size)
print("Size of index_opens=", index_opens.size)
print("Sum of values found in index_opens and index_trace=", n_overlap)
msg = 'percentage of F/O trace that was flagged: ' + repr(overlap_percentage)
print(msg)
log_msgs.append(msg)
allchecks[i] = overlap_percentage
allsizes[i] = index_trace.size
# show 2D cutouts, with flagged pixels overlaid
# calculate wavelength, slit_y values for the subwindow
det2slit = wcs_slice.get_transform('detector', 'slit_frame')
slitx, slity, _ = det2slit(x, y)
# extract & display the F/O 2d subwindows from the msa_flagging sci image
fig = plt.figure(figsize=(19, 19))
subwin = msaflag.data[i3:i4, i1:i2].copy()
# set all pixels outside of the nominal shutter length to 0, inside to 1
subwin[np.isnan(slity.T)] = 0
subwin[~np.isnan(slity.T)] = 1
# find the pixels flagged by the msaflagopen step; set them to 1 and everything else to 0 for ease of display
subwin_dq = msaflag.dq[i3:i4, i1:i2].copy()
mask = np.zeros(subwin_dq.shape, dtype=bool)
mask[np.where(subwin_dq & 536870912)] = True
subwin_dq[mask] = 1
subwin_dq[~mask] = 0
# plot the F/O traces
vmax = np.max(msaflag.data[i3:i4, i1:i2])
norm = ImageNormalize(msaflag.data[i3:i4, i1:i2], vmin=0., vmax=vmax, stretch=AsinhStretch())
plt.imshow(msaflag.data[i3:i4, i1:i2], norm=norm, aspect=10.0, origin='lower', cmap='viridis',
label='MSA flagging data')
plt.imshow(subwin, aspect=20.0, origin='lower', cmap='Reds', alpha=0.3, label='Calculated F/O')
# overplot the flagged pixels in translucent grayscale
plt.imshow(subwin_dq, aspect=20.0, origin='lower', cmap='gray', alpha=0.3, label='Pipeline F/O')
if save_figs:
t = (file_basename, "FailedOpen_detector", detector, "slit", repr(name) + ".png")
plt_name = "_".join(t)
plt_name = os.path.join(datadir, plt_name)
plt.savefig(plt_name)
print('Figure saved as: ', plt_name)
if show_figs:
plt.show()
plt.close()
# validation: overlap should be >= msa_flagging_threshold percent for all slits with more than 100 pixels
FINAL_TEST_RESULT = False
if not isinstance(msa_flagging_threshold, float):
msa_flagging_threshold = float(msa_flagging_threshold)
if (allchecks[allsizes >= 100] >= msa_flagging_threshold).all():
FINAL_TEST_RESULT = True
else:
print("\n * One or more traces show msa_flagging match < ", repr(msa_flagging_threshold))
print(" See results above per trace. \n")
if FINAL_TEST_RESULT:
result_msg = "\n *** Final result for msa_flagging test will be reported as PASSED *** \n"
print(result_msg)
log_msgs.append(result_msg)
else:
result_msg = "\n *** Final result for msa_flagging test will be reported as FAILED *** \n"
print(result_msg)
log_msgs.append(result_msg)
# end the timer
msa_flagging_test_end_time = time.time() - msa_flagging_test_start_time
if msa_flagging_test_end_time >= 60.0:
msa_flagging_test_end_time = msa_flagging_test_end_time/60.0 # in minutes
msa_flagging_test_tot_time = "* MSA flagging validation test took ", repr(msa_flagging_test_end_time) + \
" minutes to finish."
if msa_flagging_test_end_time >= 60.0:
msa_flagging_test_end_time = msa_flagging_test_end_time/60. # in hours
msa_flagging_test_tot_time = "* MSA flagging validation test took ", repr(msa_flagging_test_end_time) + \
" hours to finish."
else:
msa_flagging_test_tot_time = "* MSA flagging validation test took ", repr(msa_flagging_test_end_time) + \
" seconds to finish."
print(msa_flagging_test_tot_time)
log_msgs.append(msa_flagging_test_tot_time)
# close the datamodel
msaflag.close()
rate_mdl.close()
return FINAL_TEST_RESULT, result_msg, log_msgs
def output_hdul(set_inandout_filenames, config):
set_inandout_filenames_info = core_utils.read_info4outputhdul(
config, set_inandout_filenames)
step, txt_name, step_input_file, step_output_file, run_calwebb_spec2, outstep_file_suffix = set_inandout_filenames_info
# start the timer to compute the step running time of PTT
PTT_start_time = time.time()
# check if the filter is to be changed
change_filter_opaque = config.getboolean("calwebb_spec2_input_file",
"change_filter_opaque")
if change_filter_opaque:
_, step_input_file = change_filter_opaque2science.change_filter_opaque(
step_input_file)
change_filter_opaque_msg = " * With FILTER=OPAQUE, the calwebb_spec2 will run up to the extract_2d step. Further steps will be skipped. \n"
print(change_filter_opaque_msg)
# determine if the pipeline is to be run in full
run_calwebb_spec2 = config.getboolean("run_calwebb_spec2_in_full",
"run_calwebb_spec2")
# determine which steps are to be run, if not run in full
run_pipe_step = config.getboolean("run_pipe_steps", step)
# determine which tests are to be run
assign_wcs_completion_tests = config.getboolean(
"run_pytest", "_".join((step, "completion", "tests")))
assign_wcs_reffile_tests = config.getboolean(
"run_pytest", "_".join((step, "reffile", "tests")))
assign_wcs_validation_tests = config.getboolean(
"run_pytest", "_".join((step, "validation", "tests")))
run_pytests = [
assign_wcs_completion_tests, assign_wcs_reffile_tests,
assign_wcs_validation_tests
]
# get other relevant info from PTT config file
esa_files_path = config.get("esa_intermediary_products", "esa_files_path")
wcs_threshold_diff = config.get("additional_arguments",
"wcs_threshold_diff")
save_wcs_plots = config.getboolean("additional_arguments",
"save_wcs_plots")
output_directory = config.get("calwebb_spec2_input_file",
"output_directory")
# Get the detector used
detector = fits.getval(step_input_file, "DETECTOR", 0)
# get main header from input file
inhdu = core_utils.read_hdrfits(step_input_file,
info=False,
show_hdr=False)
# if run_calwebb_spec2 is True calwebb_spec2 will be called, else individual steps will be ran
step_completed = False
end_time = '0.0'
# Check if data is IFU that the Image Model keyword is correct
mode_used = config.get("calwebb_spec2_input_file", "mode_used").lower()
if mode_used == "ifu":
DATAMODL = fits.getval(step_input_file, "DATAMODL", 0)
if DATAMODL != "IFUImageModel":
fits.setval(step_input_file, "DATAMODL", 0, value="IFUImageModel")
print("DATAMODL keyword changed to IFUImageModel.")
# get the shutter configuration file for MOS data only
msa_shutter_conf = "No shutter configuration file will be used."
if core_utils.check_MOS_true(inhdu):
msa_shutter_conf = config.get("esa_intermediary_products",
"msa_conf_name")
# check if the configuration shutter file name is in the header of the fits file and if not add it
msametfl = fits.getval(step_input_file, "MSAMETFL", 0)
if os.path.basename(msa_shutter_conf) != msametfl:
msametfl = os.path.basename(msa_shutter_conf)
fits.setval(step_input_file, "MSAMETFL", 0, value=msametfl)
# check if processing an image, then set proper variables
imaging_mode = False
if mode_used in ('image', 'confirm', 'taconfirm', 'wata', 'msata', 'bota',
'focus', 'mimf'):
run_calwebb_spec2 = True
imaging_mode = True
print(
'\n * Image processing will only be run in full with PTT. All intermediary products will be saved.'
)
print(
' -> For now, all pytests will be skipped since there are now image-specific routines yet. \n'
)
# TODO: add imaging tests
# get the name of the configuration file and run the pipeline
calwebb_spec2_cfg = config.get("run_calwebb_spec2_in_full",
"calwebb_spec2_cfg")
# copy the configuration file to create the pipeline log
stpipelogcfg = calwebb_spec2_cfg.replace("calwebb_spec2.cfg",
"stpipe-log.cfg")
subprocess.run(["cp", stpipelogcfg, os.getcwd()])
# run the pipeline
if run_calwebb_spec2:
# Create the logfile for PTT, but remove the previous log file
PTTcalspec2_log = os.path.join(output_directory,
'PTT_calspec2_' + detector + '.log')
if imaging_mode:
PTTcalspec2_log = PTTcalspec2_log.replace('calspec2', 'calimage2')
if os.path.isfile(PTTcalspec2_log):
os.remove(PTTcalspec2_log)
print(
"Information outputed to screen from PTT will be logged in file: ",
PTTcalspec2_log)
for handler in logging.root.handlers[:]:
logging.root.removeHandler(handler)
logging.basicConfig(filename=PTTcalspec2_log, level=logging.INFO)
logging.info(pipeline_version)
if change_filter_opaque:
logging.info(change_filter_opaque_msg)
run_calwebb_spec2_msg = " *** Will run pipeline in full ... "
print(run_calwebb_spec2_msg)
logging.info(run_calwebb_spec2_msg)
# create the map
txt_name = "full_run_map_" + detector + ".txt"
if os.path.isfile(txt_name):
os.remove(txt_name)
assign_wcs_utils.create_completed_steps_txtfile(
txt_name, step_input_file)
# start the timer to compute the step running time of PTT
core_utils.start_end_PTT_time(txt_name,
start_time=PTT_start_time,
end_time=None)
if mode_used == "bots":
calwebb_spec2_cfg = calwebb_spec2_cfg.replace(
"calwebb_spec2.cfg", "calwebb_tso-spec2.cfg")
print(
'\nUsing the following configuration file to run TSO pipeline:'
)
print(calwebb_spec2_cfg, '\n')
if imaging_mode:
calwebb_image2_cfg = calwebb_spec2_cfg.replace(
"calwebb_spec2.cfg", "calwebb_image2.cfg")
print(
'\nUsing the following configuration file to run imaging pipeline:'
)
print(calwebb_image2_cfg, '\n')
else:
print(
'\nUsing the following configuration file to run spectroscopic pipeline:'
)
print(calwebb_spec2_cfg, '\n')
input_file = config.get("calwebb_spec2_input_file", "input_file")
if "_uncal_rate" in input_file:
input_file = input_file.replace("_uncal_rate", "")
if "_uncal" in input_file:
input_file = input_file.replace("_uncal", "")
final_output_name = input_file.replace(".fits", "_cal.fits")
if core_utils.check_MOS_true(inhdu):
# copy the MSA shutter configuration file into the pytest directory
subprocess.run(["cp", msa_shutter_conf, "."])
# start the timer to compute the step running time
start_time = time.time()
# run the pipeline
print('Running pipeline... \n')
if not imaging_mode:
Spec2Pipeline.call(
step_input_file,
config_file=calwebb_spec2_cfg) #, logcfg=stpipelogcfg)
else:
Image2Pipeline.call(step_input_file,
config_file=calwebb_image2_cfg)
subprocess.run(["rm", "stpipe-log.cfg"])
# For the moment, the pipeline is using the wrong reference file for slit 400A1, so the
# file needs to be re-processed with the right reference file
if core_utils.check_FS_true(inhdu):
print(
"\n * WARNING: For the moment, the wrong reference file is being used for "
"processing slit 400A1. The file will be re-processed ")
# print(" $ jwst.pathloss.PathLossStep final_output_caldet1_NRS1_srctype.fits "
# "--override_pathloss=jwst-nirspec-a400.plrf.fits \n")
pathloss_400a1 = step_input_file.replace("srctype.fits",
"pathloss_400A1.fits")
reffile_400a1 = "jwst-nirspec-a400.plrf.fits"
print("Re-processing slit with new reference file: ",
reffile_400a1)
pl = PathLossStep()
pl.override_pathloss = reffile_400a1
pl.run(step_input_file)
subprocess.run([
"mv",
step_input_file.replace("srctype", "pathlossstep"),
pathloss_400a1
])
print("Saved pipeline re-processed file as: ", pathloss_400a1)
# end the timer to compute calwebb_spec2 running time
end_time = repr(time.time() - start_time) # this is in seconds
calspec2_time = " * Pipeline took " + end_time + " seconds to finish.\n"
print(calspec2_time)
logging.info(calspec2_time)
# remove the copy of the MSA shutter configuration file
if core_utils.check_MOS_true(inhdu):
subprocess.run(["rm", msametfl])
# add the detector string to the name of the files and move them to the working directory
core_utils.add_detector2filename(output_directory, step_input_file)
final_output_name_msg = "\nThe final pipeline product was saved in: " + final_output_name
print(final_output_name_msg)
logging.info(final_output_name_msg)
# read the assign wcs fits file
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
# scihdul = core_utils.read_hdrfits(step_output_file, info=False, show_hdr=False, ext=1)
# rename and move the pipeline log file
try:
calspec2_pipelog = "calspec2_pipeline_" + detector + ".log"
if imaging_mode:
calspec2_pipelog = calspec2_pipelog.replace(
'calspec2', 'calimage2')
path_where_pipeline_was_run = os.getcwd()
logfile = glob(path_where_pipeline_was_run + "/pipeline.log")[0]
print(logfile)
os.rename(logfile, os.path.join(output_directory,
calspec2_pipelog))
except IndexError:
print(
"\nWARNING: Something went wrong. Could not find a pipeline.log file \n"
)
# make sure we are able to find calspec2_pipelog either in the calwebb_spec2 directory or in the working dir
if not os.path.isfile(calspec2_pipelog):
calspec2_pipelog = os.path.join(output_directory, calspec2_pipelog)
# add the running time for all steps
step_running_times = core_utils.calculate_step_run_time(
calspec2_pipelog)
end_time_list = []
for stp in core_utils.step_string_dict:
if stp in step_running_times:
step_completed = True
step_time = step_running_times[stp]["run_time"]
out_suffix = core_utils.step_string_dict[stp]["suffix"]
core_utils.add_completed_steps(txt_name, stp, out_suffix,
step_completed, step_time)
end_time_list.append(step_time)
# print total running time in the text file and move it to the indicated directory
string2print = "pipeline_total_time"
if float(end_time) <= sum(end_time_list):
tot_time = repr(sum(end_time_list))
else:
tot_time = end_time
assign_wcs_utils.print_time2file(txt_name, tot_time, string2print)
PTT_runtimes_msg = "Pipeline and PTT run times written in file: " + os.path.basename(
txt_name) + " in working directory. \n"
print(PTT_runtimes_msg)
logging.info(PTT_runtimes_msg)
# move the final reporting text files to the working directory
core_utils.move_txt_files_2workdir(config, detector)
# end script for imaging case
if imaging_mode:
print('\nPTT finished processing imaging mode. \n')
pytest.exit(
"Skipping pytests for now because they need to be written for imaging mode."
)
return hdul, step_output_file, msa_shutter_conf, esa_files_path, wcs_threshold_diff, save_wcs_plots, run_pytests, mode_used
else:
# create the map but remove a previous one if it exists
if os.path.isfile(txt_name):
os.remove(txt_name)
assign_wcs_utils.create_completed_steps_txtfile(
txt_name, step_input_file)
# start the timer to compute the step running time of PTT
core_utils.start_end_PTT_time(txt_name,
start_time=PTT_start_time,
end_time=None)
msg = "\n Pipeline and PTT run times will be written in file: " + os.path.basename(
txt_name) + " in working directory. \n"
print(msg)
logging.info(msg)
if run_pipe_step:
# Create the logfile for PTT, but erase the previous one if it exists
PTTcalspec2_log = os.path.join(
output_directory,
'PTT_calspec2_' + detector + '_' + step + '.log')
if imaging_mode:
PTTcalspec2_log = PTTcalspec2_log.replace(
'calspec2', 'calimage2')
if os.path.isfile(PTTcalspec2_log):
os.remove(PTTcalspec2_log)
print(
"Information outputed to screen from PTT will be logged in file: ",
PTTcalspec2_log)
for handler in logging.root.handlers[:]:
logging.root.removeHandler(handler)
logging.basicConfig(filename=PTTcalspec2_log, level=logging.INFO)
logging.info(pipeline_version)
if change_filter_opaque:
logging.info(change_filter_opaque_msg)
# check that previous pipeline steps were run up to this point
core_utils.check_completed_steps(step, step_input_file)
if os.path.isfile(step_input_file):
msg = " *** Step " + step + " set to True"
print(msg)
logging.info(msg)
stp = AssignWcsStep()
if core_utils.check_MOS_true(inhdu):
# copy the MSA shutter configuration file into the pytest directory
subprocess.run(["cp", msa_shutter_conf, "."])
# get the right configuration files to run the step
local_pipe_cfg_path = config.get("calwebb_spec2_input_file",
"local_pipe_cfg_path")
# start the timer to compute the step running time
print("running pipeline...")
start_time = time.time()
if local_pipe_cfg_path == "pipe_source_tree_code":
result = stp.call(step_input_file)
else:
result = stp.call(step_input_file,
config_file=local_pipe_cfg_path +
'/assign_wcs.cfg')
result.save(step_output_file)
# end the timer to compute the step running time
end_time = repr(time.time() - start_time) # this is in seconds
msg = "Step " + step + " took " + end_time + " seconds to finish"
print(msg)
logging.info(msg)
if core_utils.check_MOS_true(inhdu):
# remove the copy of the MSA shutter configuration file
subprocess.run(["rm", msametfl])
# rename and move the pipeline log file
try:
calspec2_pipelog = "calspec2_pipeline_" + step + "_" + detector + ".log"
if imaging_mode:
calspec2_pipelog = calspec2_pipelog.replace(
'calspec2', 'calimage2')
pytest_workdir = TESTSDIR
logfile = glob(pytest_workdir + "/pipeline.log")[0]
os.rename(logfile,
os.path.join(output_directory, calspec2_pipelog))
except IndexError:
print(
"\n* WARNING: Something went wrong. Could not find a pipeline.log file \n"
)
else:
msg = "Skipping step. Input file " + step_input_file + " does not exit."
print(msg)
logging.info(msg)
core_utils.add_completed_steps(txt_name, step,
outstep_file_suffix,
step_completed, end_time)
pytest.skip("Skipping " + step +
" because the input file does not exist.")
else:
print("Skipping running pipeline step ", step)
# add the running time for this step
end_time = core_utils.get_stp_run_time_from_screenfile(
step, detector, output_directory)
if os.path.isfile(step_output_file):
hdul = core_utils.read_hdrfits(step_output_file,
info=False,
show_hdr=False)
step_completed = True
# add the running time for this step
core_utils.add_completed_steps(txt_name, step, outstep_file_suffix,
step_completed, end_time)
return hdul, step_output_file, msa_shutter_conf, esa_files_path, wcs_threshold_diff, save_wcs_plots, run_pytests, mode_used
else:
step_completed = False
# add the running time for this step
core_utils.add_completed_steps(txt_name, step, outstep_file_suffix,
step_completed, end_time)
pytest.skip("Test skipped because input file " + step_output_file +
" does not exist.")
def output_hdul(set_inandout_filenames, config):
set_inandout_filenames_info = core_utils.read_info4outputhdul(config, set_inandout_filenames)
step, txt_name, step_input_file, step_output_file, run_calwebb_spec2, outstep_file_suffix = set_inandout_filenames_info
stp = AssignWcsStep()
run_calwebb_spec2 = config.getboolean("run_calwebb_spec2_in_full", "run_calwebb_spec2")
skip_runing_pipe_step = config.getboolean("tests_only", "_".join((step, "tests")))
esa_files_path = config.get("esa_intermediary_products", "esa_files_path")
wcs_threshold_diff = config.get("additional_arguments", "wcs_threshold_diff")
save_wcs_plots = config.getboolean("additional_arguments", "save_wcs_plots")
# if run_calwebb_spec2 is True calwebb_spec2 will be called, else individual steps will be ran
step_completed = False
end_time = '0.0'
# get the MSA shutter configuration file full path only for MOS data
inhdu = core_utils.read_hdrfits(step_input_file, info=False, show_hdr=False)
if core_utils.check_MOS_true(inhdu):
msa_shutter_conf = config.get("esa_intermediary_products", "msa_conf_name")
# check if the configuration shutter file name is in the header of the fits file and if not add it
msametfl = fits.getval(step_input_file, "MSAMETFL", 0)
if os.path.basename(msa_shutter_conf) != msametfl:
msametfl = os.path.basename(msa_shutter_conf)
fits.setval(step_input_file, "MSAMETFL", 0, value=msametfl)
# run the pipeline
if run_calwebb_spec2:
print ("*** Will run calwebb_spec2... ")
# create the map
full_run_map = "full_run_map.txt"
assign_wcs_utils.create_map_from_full_run(full_run_map, step_input_file)
# get the name of the configuration file and run the pipeline
calwebb_spec2_cfg = config.get("run_calwebb_spec2_in_full", "calwebb_spec2_cfg")
input_file_basename_list = step_input_file.split("_")[:4]
input_file_basename = "_".join((input_file_basename_list))
final_output_name = "_".join((input_file_basename, "cal.fits"))
final_output_name_basename = os.path.basename(final_output_name)
if core_utils.check_MOS_true(inhdu):
# copy the MSA shutter configuration file into the pytest directory
subprocess.run(["cp", msa_shutter_conf, "."])
# start the timer to compute the step running time
start_time = time.time()
Spec2Pipeline.call(step_input_file, config_file=calwebb_spec2_cfg)
# end the timer to compute calwebb_spec2 running time
end_time = repr(time.time() - start_time) # this is in seconds
print(" * calwebb_spec2 took "+end_time+" seconds to finish.")
if core_utils.check_MOS_true(inhdu):
# remove the copy of the MSA shutter configuration file
subprocess.run(["rm", msametfl])
# move the output file into the working directory
print ("The final calwebb_spec2 product was saved in: ", final_output_name)
subprocess.run(["mv", final_output_name_basename, final_output_name])
# read the assign wcs fits file
step_output_file = core_utils.read_completion_to_full_run_map(full_run_map, step)
hdul = core_utils.read_hdrfits(step_output_file, info=True, show_hdr=True)
scihdul = core_utils.read_hdrfits(step_output_file, info=False, show_hdr=False, ext=1)
return hdul, scihdul, step_output_file, esa_files_path, wcs_threshold_diff, save_wcs_plots
else:
# create the Map of file names
assign_wcs_utils.create_completed_steps_txtfile(txt_name, step_input_file)
if config.getboolean("steps", step):
print ("*** Step "+step+" set to True")
if os.path.isfile(step_input_file):
if not skip_runing_pipe_step:
if core_utils.check_MOS_true(inhdu):
# copy the MSA shutter configuration file into the pytest directory
subprocess.run(["cp", msa_shutter_conf, "."])
# get the right configuration files to run the step
local_pipe_cfg_path = config.get("calwebb_spec2_input_file", "local_pipe_cfg_path")
# start the timer to compute the step running time
start_time = time.time()
if local_pipe_cfg_path == "pipe_source_tree_code":
result = stp.call(step_input_file)
else:
result = stp.call(step_input_file, config_file=local_pipe_cfg_path+'/assign_wcs.cfg')
result.save(step_output_file)
# end the timer to compute the step running time
end_time = repr(time.time() - start_time) # this is in seconds
print("Step "+step+" took "+end_time+" seconds to finish")
if core_utils.check_MOS_true(inhdu):
# remove the copy of the MSA shutter configuration file
subprocess.run(["rm", msametfl])
step_completed = True
core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time)
hdul = core_utils.read_hdrfits(step_output_file, info=False, show_hdr=False, ext=0)
scihdul = core_utils.read_hdrfits(step_output_file, info=False, show_hdr=False, ext=1)
return hdul, scihdul, step_output_file, esa_files_path, wcs_threshold_diff, save_wcs_plots
else:
print("Skipping step. Intput file "+step_input_file+" does not exit.")
core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time)
pytest.skip("Skipping "+step+" because the input file does not exist.")
else:
core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time)
pytest.skip("Skipping "+step+". Step set to False in configuration file.")