def get_reference_files(datamodel): """Get the reference files associated with the step.""" refs = {} step = AssignWcsStep() for reftype in AssignWcsStep.reference_file_types: val = step.get_reference_file(datamodel, reftype) if val == 'N/A': refs[reftype] = None else: refs[reftype] = val return refs
def get_reference_files(datamodel): refs = {} step = AssignWcsStep() for reftype in AssignWcsStep.reference_file_types: val = step.get_reference_file(datamodel, reftype) print(reftype, val) if val.strip() == 'N/A': refs[reftype] = None else: refs[reftype] = val print(refs) return refs
def run_msa_flagging_testing(input_file, msa_flagging_threshold=99.5, rate_obj=None, stellarity=None, operability_ref=None, source_type=None, save_figs=False, show_figs=True, debug=False): """ This is the validation function for the msa flagging step. :param input_file: string, fits file output from the msa_flagging step :param msa_flagging_threshold: float, percentage for all slits with more than 100 pixels :param rate_obj: object, the stage 1 pipeline output object :param stellarity: float, stellarity number fro 0.0 to 1.0 :param operability_ref: string, msa failed open - operability - reference file :param source_type: string, options are point, extended, unknown :param save_figs: boolean :param show_figs: boolean :param debug: boolean :return: FINAL_TEST_RESULT: boolean, True if smaller than or equal to threshold result_msg: string, message with reason for passing, failing, or skipped log_msgs: list, diagnostic strings to be printed in log """ # start the list of messages that will be added to the log file log_msgs = [] # start the timer msa_flagging_test_start_time = time.time() # get the data model if isinstance(input_file, str): msaflag = datamodels.open(input_file) else: msaflag = input_file if debug: print('got MSA flagging datamodel!') # set up generals for all the plots font = {'weight': 'normal', 'size': 12} matplotlib.rc('font', **font) # plot full image fig = plt.figure(figsize=(9, 9)) norm = ImageNormalize(msaflag.data, vmin=0., vmax=50., stretch=AsinhStretch()) plt.imshow(msaflag.data, norm=norm, aspect=1.0, origin='lower', cmap='viridis') # Show and/or save figures detector = msaflag.meta.instrument.detector datadir = None if save_figs: file_basename = os.path.basename(input_file.replace("_msa_flagging.fits", "")) datadir = os.path.dirname(input_file) t = (file_basename, "MSA_flagging_full_detector.png") plt_name = "_".join(t) plt_name = os.path.join(datadir, plt_name) plt.savefig(plt_name) print('Figure saved as: ', plt_name) if show_figs: plt.show() plt.close() # read in DQ flags from MSA_flagging product # find all pixels that have been flagged by this step as MSA_FAILED_OPEN -> DQ array value = 536870912 # https://jwst-pipeline.readthedocs.io/en/latest/jwst/references_general/references_general.html?highlight=536870912#data-quality-flags dq_flag = 536870912 msaflag_1d = msaflag.dq.flatten() index_opens = np.squeeze(np.asarray(np.where(msaflag_1d & dq_flag))) if debug: print("DQ array at 167, 1918: ", msaflag.dq[167, 1918]) # np.set_printoptions(threshold=sys.maxsize) # print all elements in array print("Index where Failed Open shutters exist: ", np.shape(index_opens), index_opens) # execute script that creates an MSA metafile for the failed open shutters # read operability reference file """ crds_path = os.environ.get('CRDS_PATH') if crds_path is None: print("(msa_flagging_testing): The environment variable CRDS_PATH is not defined. To set it, follow the " "instructions at: \n" " https://github.com/spacetelescope/nirspec_pipe_testing_tool") exit() """ crds_path = "https://jwst-crds.stsci.edu/unchecked_get/references/jwst/" op_ref_file = "jwst_nirspec_msaoper_0001.json" if operability_ref is None: ref_file = os.path.join(crds_path, op_ref_file) urllib.request.urlretrieve(ref_file, op_ref_file) else: op_ref_file = operability_ref if "http" not in op_ref_file: if not os.path.isfile(op_ref_file): result_msg = "Skipping msa_flagging test because the operability reference file does not exist: " + \ op_ref_file print(result_msg) log_msgs.append(result_msg) result = 'skip' return result, result_msg, log_msgs if debug: print("Using this operability reference file: ", op_ref_file) with open(op_ref_file) as f: msaoper_dict = json.load(f) msaoper = msaoper_dict["msaoper"] # find the failed open shutters failedopens = [(c["Q"], c["x"], c["y"]) for c in msaoper if c["state"] == 'open'] if debug: print("Failed Open shutters: ", failedopens) # unpack the list of tuples into separate lists for MSA quadrant, row, column locations quads, allrows, allcols = zip(*failedopens) # stellarity -- internal lamps are uniform illumination, so set to 0 # ** if considering a point source, need to change this to 1, or actual value if known if source_type is None: # srctyapt = fits.getval(input_file, 'SRCTYAPT') # previously used srctyapt = msaflag.meta.target.source_type_apt else: srctyapt = source_type.upper() if stellarity is None: if "POINT" in srctyapt: stellarity = 1.0 else: stellarity = 0.0 else: stellarity = float(stellarity) # create MSA metafile with F/O shutters if datadir is not None: fometafile = os.path.join(datadir, 'fopens_metafile_msa.fits') else: fometafile = 'fopens_metafile_msa.fits' if not os.path.isfile(fometafile): pattnum = msaflag.meta.dither.position_number create_metafile_fopens(fometafile, allcols, allrows, quads, stellarity, failedopens, pattnum, save_fig=save_figs, show_fig=show_figs, debug=debug) # run assign_wcs on the science exposure using F/O metafile # change MSA metafile name in header to match the F/O metafile name if isinstance(input_file, str): rate_file = input_file.replace("msa_flagging", "rate") if not os.path.isfile(rate_file): # if a _rate.fits file does not exist try the usual name rate_file = os.path.join(datadir, 'final_output_caldet1_'+detector+'.fits') if not os.path.isfile(rate_file): result_msg = "Skipping msa_flagging test because no rate fits file was found in directory: " + datadir print(result_msg) log_msgs.append(result_msg) result = 'skip' return result, result_msg, log_msgs if debug: print("Will run assign_wcs with new Failed Open fits file on this file: ", rate_file) rate_mdl = datamodels.ImageModel(rate_file) else: rate_mdl = rate_obj if debug: print("MSA metadata file in initial rate file: ", rate_mdl.meta.instrument.msa_metadata_file) rate_mdl.meta.instrument.msa_metadata_file = fometafile if debug: print("New MSA metadata file in rate file: ", rate_mdl.meta.instrument.msa_metadata_file) # force the exp_type of this new model to MSA, even if IFU so that nrs_wcs_set_input pipeline function works if "ifu" in msaflag.meta.exposure.type.lower(): rate_mdl.meta.exposure.type = 'NRS_MSASPEC' # run assign_wcs; use +/-0.45 for the y-limits because the default is too big (0.6 including buffer) stp = AssignWcsStep() awcs_fo = stp.call(rate_mdl, slit_y_low=-0.45, slit_y_high=0.45) # get the slits from the F/O processing run slits_list = awcs_fo.meta.wcs.get_transform('gwa', 'slit_frame').slits # prepare arrays to hold info needed for validation test allsizes = np.zeros(len(slits_list)) allchecks = np.zeros(len(slits_list)) # loop over the slits and compare pixel bounds with the flagged pixels from the original product for i, slit in enumerate(slits_list): try: name = slit.name except AttributeError: name = i print("\nWorking with slit/slice: ", name) if "IFU" not in msaflag.meta.exposure.type.upper(): print("Slit min and max in y direction: ", slit.ymin, slit.ymax) # get the WCS object for this particular slit wcs_slice = nirspec.nrs_wcs_set_input(awcs_fo, name) # get the bounding box for the 2D subwindow, round to nearest integer, and convert to integer bbox = np.rint(wcs_slice.bounding_box) bboxint = bbox.astype(int) print("bounding box rounded to next integer: ", bboxint) i1 = bboxint[0, 0] i2 = bboxint[0, 1] i3 = bboxint[1, 0] i4 = bboxint[1, 1] # make array of pixel locations within bounding box x, y = np.mgrid[i1:i2, i3:i4] index_1d = np.ravel_multi_index([[y], [x]], (2048, 2048)) # get the slity WCS parameter to find which pixels are located in the actual spectrum det2slit = wcs_slice.get_transform('detector', 'slit_frame') slitx, slity, _ = det2slit(x, y) print("Max value in slity array (ignoring NANs): ", np.nanmax(slity)) index_trace = np.squeeze(index_1d)[~np.isnan(slity)] n_overlap = np.sum(np.isin(index_opens, index_trace)) overlap_percentage = round(n_overlap/index_trace.size*100., 1) if debug: print("Size of index_trace= ", index_trace.size) print("Size of index_opens=", index_opens.size) print("Sum of values found in index_opens and index_trace=", n_overlap) msg = 'percentage of F/O trace that was flagged: ' + repr(overlap_percentage) print(msg) log_msgs.append(msg) allchecks[i] = overlap_percentage allsizes[i] = index_trace.size # show 2D cutouts, with flagged pixels overlaid # calculate wavelength, slit_y values for the subwindow det2slit = wcs_slice.get_transform('detector', 'slit_frame') slitx, slity, _ = det2slit(x, y) # extract & display the F/O 2d subwindows from the msa_flagging sci image fig = plt.figure(figsize=(19, 19)) subwin = msaflag.data[i3:i4, i1:i2].copy() # set all pixels outside of the nominal shutter length to 0, inside to 1 subwin[np.isnan(slity.T)] = 0 subwin[~np.isnan(slity.T)] = 1 # find the pixels flagged by the msaflagopen step; set them to 1 and everything else to 0 for ease of display subwin_dq = msaflag.dq[i3:i4, i1:i2].copy() mask = np.zeros(subwin_dq.shape, dtype=bool) mask[np.where(subwin_dq & 536870912)] = True subwin_dq[mask] = 1 subwin_dq[~mask] = 0 # plot the F/O traces vmax = np.max(msaflag.data[i3:i4, i1:i2]) norm = ImageNormalize(msaflag.data[i3:i4, i1:i2], vmin=0., vmax=vmax, stretch=AsinhStretch()) plt.imshow(msaflag.data[i3:i4, i1:i2], norm=norm, aspect=10.0, origin='lower', cmap='viridis', label='MSA flagging data') plt.imshow(subwin, aspect=20.0, origin='lower', cmap='Reds', alpha=0.3, label='Calculated F/O') # overplot the flagged pixels in translucent grayscale plt.imshow(subwin_dq, aspect=20.0, origin='lower', cmap='gray', alpha=0.3, label='Pipeline F/O') if save_figs: t = (file_basename, "FailedOpen_detector", detector, "slit", repr(name) + ".png") plt_name = "_".join(t) plt_name = os.path.join(datadir, plt_name) plt.savefig(plt_name) print('Figure saved as: ', plt_name) if show_figs: plt.show() plt.close() # validation: overlap should be >= msa_flagging_threshold percent for all slits with more than 100 pixels FINAL_TEST_RESULT = False if not isinstance(msa_flagging_threshold, float): msa_flagging_threshold = float(msa_flagging_threshold) if (allchecks[allsizes >= 100] >= msa_flagging_threshold).all(): FINAL_TEST_RESULT = True else: print("\n * One or more traces show msa_flagging match < ", repr(msa_flagging_threshold)) print(" See results above per trace. \n") if FINAL_TEST_RESULT: result_msg = "\n *** Final result for msa_flagging test will be reported as PASSED *** \n" print(result_msg) log_msgs.append(result_msg) else: result_msg = "\n *** Final result for msa_flagging test will be reported as FAILED *** \n" print(result_msg) log_msgs.append(result_msg) # end the timer msa_flagging_test_end_time = time.time() - msa_flagging_test_start_time if msa_flagging_test_end_time >= 60.0: msa_flagging_test_end_time = msa_flagging_test_end_time/60.0 # in minutes msa_flagging_test_tot_time = "* MSA flagging validation test took ", repr(msa_flagging_test_end_time) + \ " minutes to finish." if msa_flagging_test_end_time >= 60.0: msa_flagging_test_end_time = msa_flagging_test_end_time/60. # in hours msa_flagging_test_tot_time = "* MSA flagging validation test took ", repr(msa_flagging_test_end_time) + \ " hours to finish." else: msa_flagging_test_tot_time = "* MSA flagging validation test took ", repr(msa_flagging_test_end_time) + \ " seconds to finish." print(msa_flagging_test_tot_time) log_msgs.append(msa_flagging_test_tot_time) # close the datamodel msaflag.close() rate_mdl.close() return FINAL_TEST_RESULT, result_msg, log_msgs
def output_hdul(set_inandout_filenames, config): set_inandout_filenames_info = core_utils.read_info4outputhdul( config, set_inandout_filenames) step, txt_name, step_input_file, step_output_file, run_calwebb_spec2, outstep_file_suffix = set_inandout_filenames_info # start the timer to compute the step running time of PTT PTT_start_time = time.time() # check if the filter is to be changed change_filter_opaque = config.getboolean("calwebb_spec2_input_file", "change_filter_opaque") if change_filter_opaque: _, step_input_file = change_filter_opaque2science.change_filter_opaque( step_input_file) change_filter_opaque_msg = " * With FILTER=OPAQUE, the calwebb_spec2 will run up to the extract_2d step. Further steps will be skipped. \n" print(change_filter_opaque_msg) # determine if the pipeline is to be run in full run_calwebb_spec2 = config.getboolean("run_calwebb_spec2_in_full", "run_calwebb_spec2") # determine which steps are to be run, if not run in full run_pipe_step = config.getboolean("run_pipe_steps", step) # determine which tests are to be run assign_wcs_completion_tests = config.getboolean( "run_pytest", "_".join((step, "completion", "tests"))) assign_wcs_reffile_tests = config.getboolean( "run_pytest", "_".join((step, "reffile", "tests"))) assign_wcs_validation_tests = config.getboolean( "run_pytest", "_".join((step, "validation", "tests"))) run_pytests = [ assign_wcs_completion_tests, assign_wcs_reffile_tests, assign_wcs_validation_tests ] # get other relevant info from PTT config file esa_files_path = config.get("esa_intermediary_products", "esa_files_path") wcs_threshold_diff = config.get("additional_arguments", "wcs_threshold_diff") save_wcs_plots = config.getboolean("additional_arguments", "save_wcs_plots") output_directory = config.get("calwebb_spec2_input_file", "output_directory") # Get the detector used detector = fits.getval(step_input_file, "DETECTOR", 0) # get main header from input file inhdu = core_utils.read_hdrfits(step_input_file, info=False, show_hdr=False) # if run_calwebb_spec2 is True calwebb_spec2 will be called, else individual steps will be ran step_completed = False end_time = '0.0' # Check if data is IFU that the Image Model keyword is correct mode_used = config.get("calwebb_spec2_input_file", "mode_used").lower() if mode_used == "ifu": DATAMODL = fits.getval(step_input_file, "DATAMODL", 0) if DATAMODL != "IFUImageModel": fits.setval(step_input_file, "DATAMODL", 0, value="IFUImageModel") print("DATAMODL keyword changed to IFUImageModel.") # get the shutter configuration file for MOS data only msa_shutter_conf = "No shutter configuration file will be used." if core_utils.check_MOS_true(inhdu): msa_shutter_conf = config.get("esa_intermediary_products", "msa_conf_name") # check if the configuration shutter file name is in the header of the fits file and if not add it msametfl = fits.getval(step_input_file, "MSAMETFL", 0) if os.path.basename(msa_shutter_conf) != msametfl: msametfl = os.path.basename(msa_shutter_conf) fits.setval(step_input_file, "MSAMETFL", 0, value=msametfl) # check if processing an image, then set proper variables imaging_mode = False if mode_used in ('image', 'confirm', 'taconfirm', 'wata', 'msata', 'bota', 'focus', 'mimf'): run_calwebb_spec2 = True imaging_mode = True print( '\n * Image processing will only be run in full with PTT. All intermediary products will be saved.' ) print( ' -> For now, all pytests will be skipped since there are now image-specific routines yet. \n' ) # TODO: add imaging tests # get the name of the configuration file and run the pipeline calwebb_spec2_cfg = config.get("run_calwebb_spec2_in_full", "calwebb_spec2_cfg") # copy the configuration file to create the pipeline log stpipelogcfg = calwebb_spec2_cfg.replace("calwebb_spec2.cfg", "stpipe-log.cfg") subprocess.run(["cp", stpipelogcfg, os.getcwd()]) # run the pipeline if run_calwebb_spec2: # Create the logfile for PTT, but remove the previous log file PTTcalspec2_log = os.path.join(output_directory, 'PTT_calspec2_' + detector + '.log') if imaging_mode: PTTcalspec2_log = PTTcalspec2_log.replace('calspec2', 'calimage2') if os.path.isfile(PTTcalspec2_log): os.remove(PTTcalspec2_log) print( "Information outputed to screen from PTT will be logged in file: ", PTTcalspec2_log) for handler in logging.root.handlers[:]: logging.root.removeHandler(handler) logging.basicConfig(filename=PTTcalspec2_log, level=logging.INFO) logging.info(pipeline_version) if change_filter_opaque: logging.info(change_filter_opaque_msg) run_calwebb_spec2_msg = " *** Will run pipeline in full ... " print(run_calwebb_spec2_msg) logging.info(run_calwebb_spec2_msg) # create the map txt_name = "full_run_map_" + detector + ".txt" if os.path.isfile(txt_name): os.remove(txt_name) assign_wcs_utils.create_completed_steps_txtfile( txt_name, step_input_file) # start the timer to compute the step running time of PTT core_utils.start_end_PTT_time(txt_name, start_time=PTT_start_time, end_time=None) if mode_used == "bots": calwebb_spec2_cfg = calwebb_spec2_cfg.replace( "calwebb_spec2.cfg", "calwebb_tso-spec2.cfg") print( '\nUsing the following configuration file to run TSO pipeline:' ) print(calwebb_spec2_cfg, '\n') if imaging_mode: calwebb_image2_cfg = calwebb_spec2_cfg.replace( "calwebb_spec2.cfg", "calwebb_image2.cfg") print( '\nUsing the following configuration file to run imaging pipeline:' ) print(calwebb_image2_cfg, '\n') else: print( '\nUsing the following configuration file to run spectroscopic pipeline:' ) print(calwebb_spec2_cfg, '\n') input_file = config.get("calwebb_spec2_input_file", "input_file") if "_uncal_rate" in input_file: input_file = input_file.replace("_uncal_rate", "") if "_uncal" in input_file: input_file = input_file.replace("_uncal", "") final_output_name = input_file.replace(".fits", "_cal.fits") if core_utils.check_MOS_true(inhdu): # copy the MSA shutter configuration file into the pytest directory subprocess.run(["cp", msa_shutter_conf, "."]) # start the timer to compute the step running time start_time = time.time() # run the pipeline print('Running pipeline... \n') if not imaging_mode: Spec2Pipeline.call( step_input_file, config_file=calwebb_spec2_cfg) #, logcfg=stpipelogcfg) else: Image2Pipeline.call(step_input_file, config_file=calwebb_image2_cfg) subprocess.run(["rm", "stpipe-log.cfg"]) # For the moment, the pipeline is using the wrong reference file for slit 400A1, so the # file needs to be re-processed with the right reference file if core_utils.check_FS_true(inhdu): print( "\n * WARNING: For the moment, the wrong reference file is being used for " "processing slit 400A1. The file will be re-processed ") # print(" $ jwst.pathloss.PathLossStep final_output_caldet1_NRS1_srctype.fits " # "--override_pathloss=jwst-nirspec-a400.plrf.fits \n") pathloss_400a1 = step_input_file.replace("srctype.fits", "pathloss_400A1.fits") reffile_400a1 = "jwst-nirspec-a400.plrf.fits" print("Re-processing slit with new reference file: ", reffile_400a1) pl = PathLossStep() pl.override_pathloss = reffile_400a1 pl.run(step_input_file) subprocess.run([ "mv", step_input_file.replace("srctype", "pathlossstep"), pathloss_400a1 ]) print("Saved pipeline re-processed file as: ", pathloss_400a1) # end the timer to compute calwebb_spec2 running time end_time = repr(time.time() - start_time) # this is in seconds calspec2_time = " * Pipeline took " + end_time + " seconds to finish.\n" print(calspec2_time) logging.info(calspec2_time) # remove the copy of the MSA shutter configuration file if core_utils.check_MOS_true(inhdu): subprocess.run(["rm", msametfl]) # add the detector string to the name of the files and move them to the working directory core_utils.add_detector2filename(output_directory, step_input_file) final_output_name_msg = "\nThe final pipeline product was saved in: " + final_output_name print(final_output_name_msg) logging.info(final_output_name_msg) # read the assign wcs fits file hdul = core_utils.read_hdrfits(step_output_file, info=False, show_hdr=False) # scihdul = core_utils.read_hdrfits(step_output_file, info=False, show_hdr=False, ext=1) # rename and move the pipeline log file try: calspec2_pipelog = "calspec2_pipeline_" + detector + ".log" if imaging_mode: calspec2_pipelog = calspec2_pipelog.replace( 'calspec2', 'calimage2') path_where_pipeline_was_run = os.getcwd() logfile = glob(path_where_pipeline_was_run + "/pipeline.log")[0] print(logfile) os.rename(logfile, os.path.join(output_directory, calspec2_pipelog)) except IndexError: print( "\nWARNING: Something went wrong. Could not find a pipeline.log file \n" ) # make sure we are able to find calspec2_pipelog either in the calwebb_spec2 directory or in the working dir if not os.path.isfile(calspec2_pipelog): calspec2_pipelog = os.path.join(output_directory, calspec2_pipelog) # add the running time for all steps step_running_times = core_utils.calculate_step_run_time( calspec2_pipelog) end_time_list = [] for stp in core_utils.step_string_dict: if stp in step_running_times: step_completed = True step_time = step_running_times[stp]["run_time"] out_suffix = core_utils.step_string_dict[stp]["suffix"] core_utils.add_completed_steps(txt_name, stp, out_suffix, step_completed, step_time) end_time_list.append(step_time) # print total running time in the text file and move it to the indicated directory string2print = "pipeline_total_time" if float(end_time) <= sum(end_time_list): tot_time = repr(sum(end_time_list)) else: tot_time = end_time assign_wcs_utils.print_time2file(txt_name, tot_time, string2print) PTT_runtimes_msg = "Pipeline and PTT run times written in file: " + os.path.basename( txt_name) + " in working directory. \n" print(PTT_runtimes_msg) logging.info(PTT_runtimes_msg) # move the final reporting text files to the working directory core_utils.move_txt_files_2workdir(config, detector) # end script for imaging case if imaging_mode: print('\nPTT finished processing imaging mode. \n') pytest.exit( "Skipping pytests for now because they need to be written for imaging mode." ) return hdul, step_output_file, msa_shutter_conf, esa_files_path, wcs_threshold_diff, save_wcs_plots, run_pytests, mode_used else: # create the map but remove a previous one if it exists if os.path.isfile(txt_name): os.remove(txt_name) assign_wcs_utils.create_completed_steps_txtfile( txt_name, step_input_file) # start the timer to compute the step running time of PTT core_utils.start_end_PTT_time(txt_name, start_time=PTT_start_time, end_time=None) msg = "\n Pipeline and PTT run times will be written in file: " + os.path.basename( txt_name) + " in working directory. \n" print(msg) logging.info(msg) if run_pipe_step: # Create the logfile for PTT, but erase the previous one if it exists PTTcalspec2_log = os.path.join( output_directory, 'PTT_calspec2_' + detector + '_' + step + '.log') if imaging_mode: PTTcalspec2_log = PTTcalspec2_log.replace( 'calspec2', 'calimage2') if os.path.isfile(PTTcalspec2_log): os.remove(PTTcalspec2_log) print( "Information outputed to screen from PTT will be logged in file: ", PTTcalspec2_log) for handler in logging.root.handlers[:]: logging.root.removeHandler(handler) logging.basicConfig(filename=PTTcalspec2_log, level=logging.INFO) logging.info(pipeline_version) if change_filter_opaque: logging.info(change_filter_opaque_msg) # check that previous pipeline steps were run up to this point core_utils.check_completed_steps(step, step_input_file) if os.path.isfile(step_input_file): msg = " *** Step " + step + " set to True" print(msg) logging.info(msg) stp = AssignWcsStep() if core_utils.check_MOS_true(inhdu): # copy the MSA shutter configuration file into the pytest directory subprocess.run(["cp", msa_shutter_conf, "."]) # get the right configuration files to run the step local_pipe_cfg_path = config.get("calwebb_spec2_input_file", "local_pipe_cfg_path") # start the timer to compute the step running time print("running pipeline...") start_time = time.time() if local_pipe_cfg_path == "pipe_source_tree_code": result = stp.call(step_input_file) else: result = stp.call(step_input_file, config_file=local_pipe_cfg_path + '/assign_wcs.cfg') result.save(step_output_file) # end the timer to compute the step running time end_time = repr(time.time() - start_time) # this is in seconds msg = "Step " + step + " took " + end_time + " seconds to finish" print(msg) logging.info(msg) if core_utils.check_MOS_true(inhdu): # remove the copy of the MSA shutter configuration file subprocess.run(["rm", msametfl]) # rename and move the pipeline log file try: calspec2_pipelog = "calspec2_pipeline_" + step + "_" + detector + ".log" if imaging_mode: calspec2_pipelog = calspec2_pipelog.replace( 'calspec2', 'calimage2') pytest_workdir = TESTSDIR logfile = glob(pytest_workdir + "/pipeline.log")[0] os.rename(logfile, os.path.join(output_directory, calspec2_pipelog)) except IndexError: print( "\n* WARNING: Something went wrong. Could not find a pipeline.log file \n" ) else: msg = "Skipping step. Input file " + step_input_file + " does not exit." print(msg) logging.info(msg) core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time) pytest.skip("Skipping " + step + " because the input file does not exist.") else: print("Skipping running pipeline step ", step) # add the running time for this step end_time = core_utils.get_stp_run_time_from_screenfile( step, detector, output_directory) if os.path.isfile(step_output_file): hdul = core_utils.read_hdrfits(step_output_file, info=False, show_hdr=False) step_completed = True # add the running time for this step core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time) return hdul, step_output_file, msa_shutter_conf, esa_files_path, wcs_threshold_diff, save_wcs_plots, run_pytests, mode_used else: step_completed = False # add the running time for this step core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time) pytest.skip("Test skipped because input file " + step_output_file + " does not exist.")
def output_hdul(set_inandout_filenames, config): set_inandout_filenames_info = core_utils.read_info4outputhdul(config, set_inandout_filenames) step, txt_name, step_input_file, step_output_file, run_calwebb_spec2, outstep_file_suffix = set_inandout_filenames_info stp = AssignWcsStep() run_calwebb_spec2 = config.getboolean("run_calwebb_spec2_in_full", "run_calwebb_spec2") skip_runing_pipe_step = config.getboolean("tests_only", "_".join((step, "tests"))) esa_files_path = config.get("esa_intermediary_products", "esa_files_path") wcs_threshold_diff = config.get("additional_arguments", "wcs_threshold_diff") save_wcs_plots = config.getboolean("additional_arguments", "save_wcs_plots") # if run_calwebb_spec2 is True calwebb_spec2 will be called, else individual steps will be ran step_completed = False end_time = '0.0' # get the MSA shutter configuration file full path only for MOS data inhdu = core_utils.read_hdrfits(step_input_file, info=False, show_hdr=False) if core_utils.check_MOS_true(inhdu): msa_shutter_conf = config.get("esa_intermediary_products", "msa_conf_name") # check if the configuration shutter file name is in the header of the fits file and if not add it msametfl = fits.getval(step_input_file, "MSAMETFL", 0) if os.path.basename(msa_shutter_conf) != msametfl: msametfl = os.path.basename(msa_shutter_conf) fits.setval(step_input_file, "MSAMETFL", 0, value=msametfl) # run the pipeline if run_calwebb_spec2: print ("*** Will run calwebb_spec2... ") # create the map full_run_map = "full_run_map.txt" assign_wcs_utils.create_map_from_full_run(full_run_map, step_input_file) # get the name of the configuration file and run the pipeline calwebb_spec2_cfg = config.get("run_calwebb_spec2_in_full", "calwebb_spec2_cfg") input_file_basename_list = step_input_file.split("_")[:4] input_file_basename = "_".join((input_file_basename_list)) final_output_name = "_".join((input_file_basename, "cal.fits")) final_output_name_basename = os.path.basename(final_output_name) if core_utils.check_MOS_true(inhdu): # copy the MSA shutter configuration file into the pytest directory subprocess.run(["cp", msa_shutter_conf, "."]) # start the timer to compute the step running time start_time = time.time() Spec2Pipeline.call(step_input_file, config_file=calwebb_spec2_cfg) # end the timer to compute calwebb_spec2 running time end_time = repr(time.time() - start_time) # this is in seconds print(" * calwebb_spec2 took "+end_time+" seconds to finish.") if core_utils.check_MOS_true(inhdu): # remove the copy of the MSA shutter configuration file subprocess.run(["rm", msametfl]) # move the output file into the working directory print ("The final calwebb_spec2 product was saved in: ", final_output_name) subprocess.run(["mv", final_output_name_basename, final_output_name]) # read the assign wcs fits file step_output_file = core_utils.read_completion_to_full_run_map(full_run_map, step) hdul = core_utils.read_hdrfits(step_output_file, info=True, show_hdr=True) scihdul = core_utils.read_hdrfits(step_output_file, info=False, show_hdr=False, ext=1) return hdul, scihdul, step_output_file, esa_files_path, wcs_threshold_diff, save_wcs_plots else: # create the Map of file names assign_wcs_utils.create_completed_steps_txtfile(txt_name, step_input_file) if config.getboolean("steps", step): print ("*** Step "+step+" set to True") if os.path.isfile(step_input_file): if not skip_runing_pipe_step: if core_utils.check_MOS_true(inhdu): # copy the MSA shutter configuration file into the pytest directory subprocess.run(["cp", msa_shutter_conf, "."]) # get the right configuration files to run the step local_pipe_cfg_path = config.get("calwebb_spec2_input_file", "local_pipe_cfg_path") # start the timer to compute the step running time start_time = time.time() if local_pipe_cfg_path == "pipe_source_tree_code": result = stp.call(step_input_file) else: result = stp.call(step_input_file, config_file=local_pipe_cfg_path+'/assign_wcs.cfg') result.save(step_output_file) # end the timer to compute the step running time end_time = repr(time.time() - start_time) # this is in seconds print("Step "+step+" took "+end_time+" seconds to finish") if core_utils.check_MOS_true(inhdu): # remove the copy of the MSA shutter configuration file subprocess.run(["rm", msametfl]) step_completed = True core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time) hdul = core_utils.read_hdrfits(step_output_file, info=False, show_hdr=False, ext=0) scihdul = core_utils.read_hdrfits(step_output_file, info=False, show_hdr=False, ext=1) return hdul, scihdul, step_output_file, esa_files_path, wcs_threshold_diff, save_wcs_plots else: print("Skipping step. Intput file "+step_input_file+" does not exit.") core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time) pytest.skip("Skipping "+step+" because the input file does not exist.") else: core_utils.add_completed_steps(txt_name, step, outstep_file_suffix, step_completed, end_time) pytest.skip("Skipping "+step+". Step set to False in configuration file.")