def compare(cls, reference_file, test_file, atol=None, rtol=None): import astropy from astropy.io.fits.diff import FITSDiff from astropy.utils.introspection import minversion if minversion(astropy, '2.0'): diff = FITSDiff(reference_file, test_file, rtol=rtol) else: diff = FITSDiff(reference_file, test_file, tolerance=rtol) return diff.identical, diff.report()
def test_partially_identical_files3(self): """ Test files that have some identical HDUs but a different extension name. """ phdu = PrimaryHDU() ehdu = ImageHDU(name='FOO') hdula = HDUList([phdu, ehdu]) ehdu = BinTableHDU(name='BAR') ehdu.header['EXTVER'] = 2 ehdu.header['EXTLEVEL'] = 3 hdulb = HDUList([phdu, ehdu]) diff = FITSDiff(hdula, hdulb) assert not diff.identical assert diff.diff_hdus[0][0] == 1 hdu_diff = diff.diff_hdus[0][1] assert hdu_diff.diff_extension_types == ('IMAGE', 'BINTABLE') assert hdu_diff.diff_extnames == ('FOO', 'BAR') assert hdu_diff.diff_extvers == (1, 2) assert hdu_diff.diff_extlevels == (1, 3) report = diff.report() assert 'Extension types differ' in report assert 'a: IMAGE\n b: BINTABLE' in report assert 'Extension names differ' in report assert 'a: FOO\n b: BAR' in report assert 'Extension versions differ' in report assert 'a: 1\n b: 2' in report assert 'Extension levels differ' in report assert 'a: 1\n b: 2' in report
def test_nircam_setpointing_tsimg(_jail, rtdata, fitsdiff_default_kwargs): """ Regression test of the set_telescope_pointing script on a level-1b NIRCam TSO imaging file. """ # Get SIAF PRD database file siaf_path = rtdata.get_data("common/prd.db") rtdata.get_data( "nircam/tsimg/jw00312006001_02102_00001-seg001_nrcb1_uncal.fits") # The add_wcs function overwrites its input, so output = input rtdata.output = rtdata.input # Call the WCS routine, using the ENGDB_Service try: add_wcs(rtdata.input, siaf_path=siaf_path) except ValueError: pytest.skip('Engineering Database not available.') rtdata.get_truth( "truth/test_nircam_setpointing/jw00312006001_02102_00001-seg001_nrcb1_uncal.fits" ) fitsdiff_default_kwargs['rtol'] = 1e-6 diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs) assert diff.identical, diff.report()
def test_miri_setpointing(_jail, rtdata, fitsdiff_default_kwargs): """ Regression test of the set_telescope_pointing script on a level-1b MIRI image. """ # Get SIAF PRD database file siaf_path = rtdata.get_data("common/prd.db") # Get the input level-1b file rtdata.get_data("miri/image/jw80600010001_02101_00001_mirimage_uncal.fits") # The add_wcs function overwrites its input, so output = input rtdata.output = rtdata.input # Call the WCS routine, using the ENGDB_Service # Note that there aren't any quaternion mnemonics in the ENGDB for the time # range of this exposure, so we set "allow_default" to tell add_wcs to use # default values for the pointing-related keywords. Keyword values retrieved # from the SIAF will be good. add_wcs(rtdata.input, allow_default=True, siaf_path=siaf_path) # Compare the results rtdata.get_truth( "truth/test_miri_setpointing/jw80600010001_02101_00001_mirimage_uncal.fits" ) fitsdiff_default_kwargs['rtol'] = 1e-6 diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs) assert diff.identical, diff.report()
def test_nirspec_ifu_mbkg_user(rtdata, fitsdiff_default_kwargs): """Test NIRSpec IFU data with a user-supplied background file.""" # Get user-supplied background user_background = "prism_bkg_x1d.fits" rtdata.get_data(f"nirspec/ifu/{user_background}") # Get input data rtdata.get_data("nirspec/ifu/prism_sci_bkg_cal.fits") collect_pipeline_cfgs("config") args = [ "config/master_background.cfg", rtdata.input, "--user_background", user_background ] Step.from_cmdline(args) output = "prism_sci_bkg_master_background.fits" rtdata.output = output # Get the truth file rtdata.get_truth(f"truth/test_nirspec_ifu_mbkg_user/{output}") # Compare the results diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs) assert diff.identical, diff.report()
def test_miri_lrs_slitless_tso3( run_spec3_pipeline, generate_tso3_asn, rtdata_module, fitsdiff_default_kwargs, step_suffix, filename_extension, is_product, diff_astropy_tables, ): """Compare the output of a MIRI LRS slitless calwebb_tso3 step.""" rtdata = rtdata_module _, asn_id = generate_tso3_asn if is_product: output_filename = f"{PRODUCT_NAME}_{step_suffix}.{filename_extension}" else: output_filename = f"{DATASET_ID}_{asn_id}_{step_suffix}.{filename_extension}" rtdata.output = output_filename rtdata.get_truth(f"truth/test_miri_lrs_slitless_tso3/{output_filename}") if filename_extension == "fits": diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs) assert diff.identical, diff.report() else: diff = diff_astropy_tables(rtdata.output, rtdata.truth) assert len(diff) == 0, "\n".join(diff)
def test_level_1(out_dir): l0 = test_data.products.L0_LightCurve_fits l1 = Level1(out_dir / 'LB', out_dir) res = sorted(l1.process_fits_files(files=l0)) assert len(res) == 2 # test for https://github.com/i4Ds/STIXCore/issues/180 # TODO remove when solved lc1 = Product(res[0]) lc2 = Product(res[1]) t = np.hstack((np.array(lc1.data['time']), (np.array(lc2.data['time'])))) td = np.hstack( (np.array(lc1.data['timedel']), (np.array(lc2.data['timedel'])))) range(len(lc1.data['time']) - 3, len(lc1.data['time']) + 3) assert np.all((t[1:] - t[0:-1]) == td[0:-1]) # end test for https://github.com/i4Ds/STIXCore/issues/180 for fits in res: diff = FITSDiff( test_data.products.DIR / fits.name, fits, ignore_keywords=['CHECKSUM', 'DATASUM', 'DATE', 'VERS_SW']) if not diff.identical: print(diff.report()) assert diff.identical
def fits_osiris_allclose(a, b): """Assert that two OSIRIS fits files are close.""" a = fits.open(a) b = fits.open(b) try: del a[0].header['COMMENT'] del b[0].header['COMMENT'] report = StringIO() diff = FITSDiff(a, b, ignore_keywords=["COMMENT"], ignore_comments=["SIMPLE"], ignore_fields=[], ignore_blanks=True, ignore_blank_cards=True, tolerance=1e-5) diff.report(fileobj=report) assert diff.identical, report.getvalue() finally: a.close() b.close()
def test_image3_closedfile(run_image3_closedfile, rtdata, fitsdiff_default_kwargs): """Ensure production of Image3Pipeline output with data having closed file issues""" rtdata.output = 'jw00617-o082_t001_nircam_clear-f090w-sub320_i2d.fits' rtdata.get_truth('truth/test_nircam_image/jw00617-o082_t001_nircam_clear-f090w-sub320_i2d.fits') diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs) assert diff.identical, diff.report()
def fitsdiff(input1, input2, comment_excl_list='', value_excl_list='', field_excl_list='', maxdiff=10, delta=0.0, neglect_blanks=True, output=None): if isinstance(comment_excl_list, string_types): comment_excl_list = list_parse(comment_excl_list) if isinstance(value_excl_list, string_types): value_excl_list = list_parse(value_excl_list) if isinstance(field_excl_list, string_types): field_excl_list = list_parse(field_excl_list) diff = FITSDiff(input1, input2, ignore_keywords=value_excl_list, ignore_comments=comment_excl_list, ignore_fields=field_excl_list, numdiffs=maxdiff, tolerance=delta, ignore_blanks=neglect_blanks) if output is None: output = sys.stdout diff.report(output) return diff.identical
def test_miri_lrs_extract1d_from_cal(run_pipeline, rtdata_module, fitsdiff_default_kwargs): rtdata = rtdata_module rtdata.input = "jw00623032001_03102_00001_mirimage_cal.fits" Extract1dStep.call(rtdata.input, save_results=True) output = "jw00623032001_03102_00001_mirimage_extract1dstep.fits" rtdata.output = output rtdata.get_truth(f"truth/test_miri_lrs_slit_spec2/{output}") diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs) assert diff.identical, diff.report()
def test_miri_image3_i2d(run_image3, rtdata_module, fitsdiff_default_kwargs): rtdata = rtdata_module rtdata.input = "det_dithered_5stars_image3_asn.json" rtdata.output = "det_dithered_5stars_f770w_i2d.fits" rtdata.get_truth("truth/test_miri_image_stages/det_dithered_5stars_f770w_i2d.fits") fitsdiff_default_kwargs["rtol"] = 1e-4 diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs) assert diff.identical, diff.report()
def test_miri_image2(run_pipeline, fitsdiff_default_kwargs, output): """Regression test of calwebb_image2 pipeline performed on MIRI data.""" rtdata = run_pipeline rtdata.output = output rtdata.get_truth(os.path.join("truth/test_miri_image2_cal", output)) diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs) assert diff.identical, diff.report()
def _test_write_string_bytes_io(self, fileobj): """ Implemented for both test_write_stringio and test_write_bytesio. """ with fits.open(self.data('test0.fits')) as hdul: hdul.writeto(fileobj) hdul2 = fits.HDUList.fromstring(fileobj.getvalue()) assert FITSDiff(hdul, hdul2).identical
def test_nircam_image_stages12(run_pipelines, fitsdiff_default_kwargs, suffix): """Regression test of detector1 and image2 pipelines performed on NIRCam data.""" rtdata = run_pipelines rtdata.input = "jw42424001001_01101_00001_nrca5_uncal.fits" output = "jw42424001001_01101_00001_nrca5_" + suffix + ".fits" rtdata.output = output rtdata.get_truth("truth/test_nircam_image_stages/" + output) diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs) assert diff.identical, diff.report()
def test_spec3(run_spec3, fitsdiff_default_kwargs, output): """Regression test matching output files""" rtdata = run_spec3 rtdata.output = output rtdata.get_truth(os.path.join(TRUTH_PATH, rtdata.output)) diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs) assert diff.identical, diff.report()
def test_niriss_soss_stage3_x1dints(run_pipelines, fitsdiff_default_kwargs): rtdata = run_pipelines rtdata.input = "jw00625-o023_20191210t204036_tso3_001_asn.json" rtdata.output = "jw00625-o023_t001_niriss_clear-gr700xd-substrip256_x1dints.fits" rtdata.get_truth( "truth/test_niriss_soss_stages/jw00625-o023_t001_niriss_clear-gr700xd-substrip256_x1dints.fits" ) diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs) assert diff.identical, diff.report()
def test_nirspec_detector1(run_pipeline, fitsdiff_default_kwargs, output): """ Regression test of calwebb_detector1 pipeline performed on NIRSpec data. """ rtdata = run_pipeline rtdata.output = output rtdata.get_truth(os.path.join("truth/test_nirspec_subarray", output)) diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs) assert diff.identical, diff.report()
def test_spec2(rtdata_module, run_spec2, fitsdiff_default_kwargs, suffix): """Test ensuring the calwebb_tso-spec2 is operating appropriately for MIRI MRS TSO data""" rtdata = rtdata_module output = f"jw80600018001_02101_00003_mirifushort_{suffix}.fits" rtdata.output = output rtdata.get_truth(f"{TRUTH_PATH}/{output}") diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs) assert diff.identical, diff.report()
def test_fits_file_bytes_object(self): """ Testing when fits file is passed as bytes. """ with fits.open(self.data('tdim.fits').encode()) as hdulist: assert hdulist[0].filebytes() == 2880 assert hdulist[1].filebytes() == 5760 with fits.open(self.data('tdim.fits')) as hdulist2: assert FITSDiff(hdulist2, hdulist).identical is True
def test_nircam_tsgrism_stage3_x1dints(run_pipelines, fitsdiff_default_kwargs): rtdata = run_pipelines rtdata.input = "jw00721-o012_20191119t043909_tso3_001_asn.json" rtdata.output = "jw00721-o012_t004_nircam_f444w-grismr-subgrism256_x1dints.fits" rtdata.get_truth( "truth/test_nircam_tsgrism_stages/jw00721-o012_t004_nircam_f444w-grismr-subgrism256_x1dints.fits" ) diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs) assert diff.identical, diff.report()
def test_nircam_image_stage3_segm(run_image3pipeline, rtdata_module, fitsdiff_default_kwargs): """Test that segmentation map looks good for NIRCam imaging""" rtdata = rtdata_module rtdata.input = "jw42424-o002_20191220t214154_image3_001_asn.json" output = "jw42424-o002_t001_nircam_clear-f444w_segm.fits" rtdata.output = output rtdata.get_truth(f"truth/test_nircam_image_stages/{output}") diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs) assert diff.identical, diff.report()
def test_nircam_coron3_product(run_pipeline, suffix, fitsdiff_default_kwargs): """Check final products of calwebb_coron3""" rtdata = run_pipeline output = "jw99999-a3001_t1_nircam_f140m-maskbar_" + suffix + ".fits" rtdata.output = output rtdata.get_truth("truth/test_nircam_coron3/" + output) fitsdiff_default_kwargs['atol'] = 1e-5 diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs) assert diff.identical, diff.report()
def test_miri_lrs_slitless_spec2(run_spec2_pipeline, rtdata_module, fitsdiff_default_kwargs, step_suffix): """Compare the output of a MIRI LRS slitless calwebb_tso-spec2 step.""" rtdata = rtdata_module output_filename = f"{DATASET_ID}_{step_suffix}.fits" rtdata.output = output_filename rtdata.get_truth(f"truth/test_miri_lrs_slitless_spec2/{output_filename}") diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs) assert diff.identical, diff.report()
def test_nircam_tsimg_stage2(run_pipelines, fitsdiff_default_kwargs, suffix): """Regression test of tso-image2 pipeline performed on NIRCam TSIMG data.""" rtdata = run_pipelines rtdata.input = "jw00312006001_02102_00001-seg001_nrcb1_rateints.fits" output = "jw00312006001_02102_00001-seg001_nrcb1_" + suffix + ".fits" rtdata.output = output rtdata.get_truth("truth/test_nircam_tsimg_stage23/" + output) diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs) assert diff.identical, diff.report()
def test_miri_lrs_slitless_tso3_x1dints(run_tso3_pipeline, rtdata_module, fitsdiff_default_kwargs): """Compare the output of a MIRI LRS slitless calwebb_tso3 pipeline.""" rtdata = rtdata_module output_filename = f"{PRODUCT_NAME}_x1dints.fits" rtdata.output = output_filename rtdata.get_truth(f"truth/test_miri_lrs_slitless_tso3/{output_filename}") diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs) assert diff.identical, diff.report()
def test_fits_file_path_object(self): """ Testing when fits file is passed as pathlib.Path object #4412. """ fpath = pathlib.Path(self.data('tdim.fits')) with fits.open(fpath) as hdulist: assert hdulist[0].filebytes() == 2880 assert hdulist[1].filebytes() == 5760 with fits.open(self.data('tdim.fits')) as hdulist2: assert FITSDiff(hdulist2, hdulist).identical is True
def test_niriss_soss_stage2(run_pipelines, fitsdiff_default_kwargs, suffix): """Regression test of tso-spec2 pipeline performed on NIRISS SOSS data.""" rtdata = run_pipelines rtdata.input = "jw00625023001_03101_00001-seg001_nis_rateints.fits" output = "jw00625023001_03101_00001-seg001_nis_" + suffix + ".fits" rtdata.output = output rtdata.get_truth("truth/test_niriss_soss_stages/" + output) diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs) assert diff.identical, diff.report()
def test_miri_image_stage3_i2d(run_pipelines, fitsdiff_default_kwargs): rtdata = run_pipelines rtdata.input = "det_dithered_5stars_image3_asn.json" rtdata.output = "det_dithered_5stars_f770w_i2d.fits" rtdata.get_truth( "truth/test_miri_image_stages/det_dithered_5stars_f770w_i2d.fits") fitsdiff_default_kwargs['ignore_fields'] = ['date', 'filename'] fitsdiff_default_kwargs['ignore_keywords'] += ['naxis1', 'tform*'] diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs) assert diff.identical, diff.report()
def test_nircam_coron3_exp(run_pipeline, suffix, exposure, fitsdiff_default_kwargs): """Check intermediate results of calwebb_coron3""" rtdata = run_pipeline output = "jw9999947001_02102_" + exposure + "_nrcb3_a3001_" + suffix + ".fits" rtdata.output = output rtdata.get_truth("truth/test_nircam_coron3/" + output) fitsdiff_default_kwargs["atol"] = 1e-5 diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs) assert diff.identical, diff.report()