def test_image3_pipeline1(self): asn_name = "mosaic_long_asn.json" asn_file = self.get_data('test_calimage3', asn_name) for file in raw_from_asn(asn_file): self.get_data('test_calimage3', file) collect_pipeline_cfgs('config') args = [ 'config/calwebb_image3.cfg', asn_file, '--steps.tweakreg.skip=True', ] Step.from_cmdline(args) self.ignore_keywords += ['NAXIS1', 'TFORM*'] self.ignore_fields = self.ignore_keywords self.rtol = 0.0001 outputs = [('nrca5_47Tuc_subpix_dither1_newpos_a3001_crf.fits', 'nrca5_47Tuc_subpix_dither1_newpos_cal-a3001_ref.fits'), ('mosaic_long_i2d.fits', 'mosaic_long_i2d_ref.fits'), ('mosaic_long_cat.ecsv', 'mosaic_long_cat_ref.ecsv'), ] self.compare_outputs(outputs)
def test_miri_masterbg_lrs_dedicated(self): """Run masterbackground step on MIRI LRS association""" asn_file = self.get_data(*self.test_dir, 'miri_lrs_mbkg_dedicated_spec3_asn.json') for file in raw_from_asn(asn_file): self.get_data(*self.test_dir, file) collect_pipeline_cfgs('./config') result = MasterBackgroundStep.call( asn_file, config_file='config/master_background.cfg', save_background = True, save_results = True) # test 1 # loop over the background subtracted data and compare to truth files for model in result: assert model.meta.cal_step.master_background == 'COMPLETE' result_file = model.meta.filename.replace('cal', 'master_background') truth_file = self.get_data(*self.ref_loc, result_file) outputs = [(result_file, truth_file)] self.compare_outputs(outputs) # test 2 # compare the master background combined file to truth file master_combined_bkg_file = 'MIRI_LRS_seq1_MIRIMAGE_P750Lexp1_o001_masterbg.fits' truth_background = self.get_data(*self.ref_loc, master_combined_bkg_file) outputs = [(master_combined_bkg_file, truth_background)] self.compare_outputs(outputs)
def test_image3_pipeline2(self): """Regression test definitions for CALIMAGE3 pipeline. Regression test of calwebb_image3 pipeline on NIRCam simulated long-wave data with a 6-point dither. """ asn_file = self.get_data(self.test_dir, "jw10002-o001_20171116t191235_image3_002_asn.json") for file in raw_from_asn(asn_file): self.get_data(self.test_dir, file) collect_pipeline_cfgs('config') args = [ 'config/calwebb_image3.cfg', asn_file, '--steps.tweakreg.kernel_fwhm=2', '--steps.tweakreg.snr_threshold=5', '--steps.tweakreg.enforce_user_order=True', '--steps.tweakreg.searchrad=10', '--steps.tweakreg.fitgeometry=rscale' ] Step.from_cmdline(args) outputs = [('jw10002001001_01101_00004_nrcblong_o001_crf.fits', 'jw10002001001_01101_00004_nrcblong_o001_crf_ref.fits'), {'files':('jw10002-o001_t002_nircam_f444w_i2d.fits', 'jw10002-o001_t002_nircam_f444w_i2d_ref.fits'), 'pars':{'ignore_keywords':self.ignore_keywords+['NAXIS1','TFORM*'], 'ignore_fields':self.ignore_keywords, 'rtol':0.0001} } ] self.compare_outputs(outputs)
def test_nrs_fs_spec3(self): """ Regression test of calwebb_spec3 pipeline performed on NIRSpec fixed-slit data. """ cfg_dir = './cfgs' collect_pipeline_cfgs(cfg_dir) datapath = ['test_datasets', 'fss', '93045', 'level2b'] asn_file = self.get_data( *datapath, 'jw93045-o010_20180725t035735_spec3_001_asn.json') for file in raw_from_asn(asn_file): self.get_data(*datapath, file) args = [path.join(cfg_dir, 'calwebb_spec3.cfg'), asn_file] Step.from_cmdline(args) # Compare results outputs = [('jw00023001001_01101_00001_NRS1_cal.fits', 'jw00023001001_01101_00001_NRS1_cal_ref.fits'), ('jw00023001001_01101_00001_NRS1_s2d.fits', 'jw00023001001_01101_00001_NRS1_s2d_ref.fits'), ('jw00023001001_01101_00001_NRS1_x1d.fits', 'jw00023001001_01101_00001_NRS1_x1d_ref.fits')] self.compare_outputs(outputs)
def test_tso3_pipeline_nrc2(self): """Regression test of calwebb_tso3 pipeline on NIRCam simulated data. Scaled imaging mode outlier_detection will be tested here. """ asn_file = self.get_data( self.test_dir, "jw93065-a3002_20170511t111213_tso3_001_asn.json") for file in raw_from_asn(asn_file): self.get_data(self.test_dir, file) step = Tso3Pipeline() step.scale_detection = True step.outlier_detection.weight_type = 'exptime' step.outlier_detection.pixfrac = 1.0 step.outlier_detection.kernel = 'square' step.outlier_detection.fillval = 'INDEF' step.outlier_detection.nlow = 0 step.outlier_detection.nhigh = 0 step.outlier_detection.maskpt = 0.7 step.outlier_detection.grow = 1 step.outlier_detection.snr = '4.0 3.0' step.outlier_detection.scale = '0.5 0.4' step.outlier_detection.backg = 0.0 step.outlier_detection.save_intermediate_results = False step.outlier_detection.resample_data = False step.outlier_detection.good_bits = 4 step.extract_1d.smoothing_length = 0 step.extract_1d.bkg_order = 0 step.run(asn_file) outputs = [('jw93065002002_02101_00001_nrca1_a3002_crfints.fits', 'jw93065002002_02101_00001_nrca1_a3002_crfints_ref.fits', ['primary', 'sci', 'dq', 'err'])] self.compare_outputs(outputs)
def test_miri_masterbg_mrs_nodded(self): """Run masterbackground step on MIRI MRS association""" asn_file = self.get_data(*self.test_dir, 'miri_mrs_mbkg_spec3_asn.json') for file in raw_from_asn(asn_file): self.get_data(*self.test_dir, file) collect_pipeline_cfgs('./config') result = MasterBackgroundStep.call( asn_file, config_file='config/master_background.cfg', save_background=True, save_results=True, ) # test 1 # loop over the background subtracted data and compare to truth files # check that the cal_step master_background ran to complete for model in result: assert model.meta.cal_step.master_background == 'COMPLETE' truth_file = self.get_data(*self.ref_loc, model.meta.filename) outputs = [(model.meta.filename, truth_file)] self.compare_outputs(outputs) # test 2 # compare the master background combined file to truth file master_combined_bkg_file = 'MIRI_MRS_nod_seq1_MIRIFUSHORT_12SHORTexp1_o001_masterbg.fits' truth_background = self.get_data(*self.ref_loc, master_combined_bkg_file) outputs = [(master_combined_bkg_file, truth_background)] self.compare_outputs(outputs)
def test_save_source_only(self): """Test saving the source-based files only""" datapath = ['test_datasets', 'fss', '93045', 'level2b'] asn_file = self.get_data( *datapath, 'jw93045-o010_20180725t035735_spec3_001_asn.json') for file in raw_from_asn(asn_file): self.get_data(*datapath, file) pipe = Spec3Pipeline() pipe.mrs_imatch.skip = True pipe.outlier_detection.skip = True pipe.resample_spec.skip = True pipe.cube_build.skip = True pipe.extract_1d.skip = True pipe.run(asn_file) # Check resulting product with open(asn_file) as fh: asn = load_asn(fh) base_name = asn['products'][0]['name'] product_name = base_name.format(source_id='s00000') + '_cal.fits' output_files = glob('*') if product_name in output_files: output_files.remove(product_name) else: assert False
def test_image3_pipeline1(self): asn_name = "mosaic_long_asn.json" asn_file = self.get_data('test_calimage3', asn_name) for file in raw_from_asn(asn_file): self.get_data('test_calimage3', file) collect_pipeline_cfgs('config') args = [ 'config/calwebb_image3.cfg', asn_file, '--steps.tweakreg.skip=True', ] Step.from_cmdline(args) outputs = [(# Compare level-2c crf product 'nrca5_47Tuc_subpix_dither1_newpos_a3001_crf.fits', 'nrca5_47Tuc_subpix_dither1_newpos_cal-a3001_ref.fits'), {'files':(# Compare i2d product 'mosaic_long_i2d.fits', 'mosaic_long_i2d_ref.fits'), 'pars': {'ignore_keywords': self.ignore_keywords+['NAXIS1', 'TFORM*'], 'ignore_fields':self.ignore_keywords, 'rtol': 0.0001} }, ('mosaic_long_cat.ecsv', 'mosaic_long_cat_ref.ecsv'), ] self.compare_outputs(outputs)
def test_save_source_only(self): """Test saving the source-based files only""" datapath = ['test_datasets', 'fss', '93045', 'level2b'] asn_file = self.get_data(*datapath, 'jw93045-o010_20180725t035735_spec3_001_asn.json') for file in raw_from_asn(asn_file): self.get_data(*datapath, file) pipe = Spec3Pipeline() pipe.mrs_imatch.skip = True pipe.outlier_detection.skip = True pipe.resample_spec.skip = True pipe.cube_build.skip = True pipe.extract_1d.skip = True pipe.run(asn_file) # Check resulting product with open(asn_file) as fh: asn = load_asn(fh) base_name = asn['products'][0]['name'] product_name = base_name.format(source_id='ss400a1') + '_cal.fits' output_files = glob('*') if product_name in output_files: output_files.remove(product_name) else: assert False
def test_nirspec_masterbg_nodded(self): """Run masterbackground step on NIRSpec association""" asn_file = self.get_data(*self.test_dir, 'nirspec_spec3_asn.json') for file in raw_from_asn(asn_file): self.get_data(*self.test_dir, file) collect_pipeline_cfgs('./config') result = MasterBackgroundStep.call( asn_file, config_file='config/master_background.cfg', save_background=True, save_results=True) # test 1 # compare background subtracted data to truth files # check that the cal_step master_background ran to complete outputs = [] for model in result: assert model.meta.cal_step.master_background == 'COMPLETE' result_file = model.meta.filename.replace('cal', 'master_background') truth_file = self.get_data(*self.ref_loc, result_file) outputs.append((result_file, truth_file)) self.compare_outputs(outputs) # test 2 # compare the master background combined file to truth file master_combined_bkg_file = 'ifu_prism_source_off_fix_NRS1_o001_masterbg.fits' truth_background = self.get_data(*self.ref_loc, master_combined_bkg_file) outputs = [(master_combined_bkg_file, truth_background)] self.compare_outputs(outputs)
def test_tso3_pipeline_nrc2(self): """Regression test of calwebb_tso3 pipeline on NIRCam simulated data. Scaled imaging mode outlier_detection will be tested here. """ asn_file = self.get_data(self.test_dir, "jw93065-a3002_20170511t111213_tso3_001_asn.json") for file in raw_from_asn(asn_file): self.get_data(self.test_dir, file) step = Tso3Pipeline() step.scale_detection = True step.outlier_detection.weight_type = 'exptime' step.outlier_detection.pixfrac = 1.0 step.outlier_detection.kernel = 'square' step.outlier_detection.fillval = 'INDEF' step.outlier_detection.nlow = 0 step.outlier_detection.nhigh = 0 step.outlier_detection.maskpt = 0.7 step.outlier_detection.grow = 1 step.outlier_detection.snr = '4.0 3.0' step.outlier_detection.scale = '0.5 0.4' step.outlier_detection.backg = 0.0 step.outlier_detection.save_intermediate_results = False step.outlier_detection.resample_data = False step.outlier_detection.good_bits = 4 step.extract_1d.smoothing_length = 0 step.extract_1d.bkg_order = 0 step.run(asn_file) outputs = [('jw93065002002_02101_00001_nrca1_a3002_crfints.fits', 'jw93065002002_02101_00001_nrca1_a3002_crfints_ref.fits', ['primary', 'sci', 'dq', 'err']) ] self.compare_outputs(outputs)
def test_nrs_fs_spec3(self): """ Regression test of calwebb_spec3 pipeline performed on NIRSpec fixed-slit data. """ cfg_dir = './cfgs' collect_pipeline_cfgs(cfg_dir) datapath = ['test_datasets', 'fss', '93045', 'level2b'] asn_file = self.get_data(*datapath, 'jw93045-o010_20180725t035735_spec3_001_asn.json') for file in raw_from_asn(asn_file): self.get_data(*datapath, file) args = [ path.join(cfg_dir, 'calwebb_spec3.cfg'), asn_file ] Step.from_cmdline(args) # Compare results outputs = [('jw00023001001_01101_00001_NRS1_cal.fits', 'jw00023001001_01101_00001_NRS1_cal_ref.fits'), ('jw00023001001_01101_00001_NRS1_s2d.fits', 'jw00023001001_01101_00001_NRS1_s2d_ref.fits'), ('jw00023001001_01101_00001_NRS1_x1d.fits', 'jw00023001001_01101_00001_NRS1_x1d_ref.fits') ] self.compare_outputs(outputs)
def test_image3_pipeline1(self): asn_name = "mosaic_long_asn.json" asn_file = self.get_data('test_calimage3', asn_name) for file in raw_from_asn(asn_file): self.get_data('test_calimage3', file) collect_pipeline_cfgs('config') args = [ 'config/calwebb_image3.cfg', asn_file, '--steps.tweakreg.skip=True', ] Step.from_cmdline(args) outputs = [ ( # Compare level-2c crf product 'nrca5_47Tuc_subpix_dither1_newpos_a3001_crf.fits', 'nrca5_47Tuc_subpix_dither1_newpos_cal-a3001_ref.fits'), { 'files': ( # Compare i2d product 'mosaic_long_i2d.fits', 'mosaic_long_i2d_ref.fits'), 'pars': { 'ignore_keywords': self.ignore_keywords + ['NAXIS1', 'TFORM*'], 'ignore_fields': self.ignore_keywords, 'rtol': 0.0001 } } ] self.compare_outputs(outputs)
def test_miri_masterbg_lrs_dedicated(self): """Run masterbackground step on MIRI LRS association""" asn_file = self.get_data(*self.test_dir, 'miri_lrs_mbkg_dedicated_spec3_asn.json') for file in raw_from_asn(asn_file): self.get_data(*self.test_dir, file) collect_pipeline_cfgs('./config') result = MasterBackgroundStep.call( asn_file, config_file='config/master_background.cfg', save_background=True, save_results=True) # test 1 # loop over the background subtracted data and compare to truth files for model in result: assert model.meta.cal_step.master_background == 'COMPLETE' result_file = model.meta.filename.replace('cal', 'master_background') truth_file = self.get_data(*self.ref_loc, result_file) outputs = [(result_file, truth_file)] self.compare_outputs(outputs) # test 2 # compare the master background combined file to truth file master_combined_bkg_file = 'MIRI_LRS_seq1_MIRIMAGE_P750Lexp1_o001_masterbg.fits' truth_background = self.get_data(*self.ref_loc, master_combined_bkg_file) outputs = [(master_combined_bkg_file, truth_background)] self.compare_outputs(outputs)
def test_coron3_1(self): """Regression test of calwebb_coron3 pipeline. Test will be performed on NIRCam simulated data. """ asn_name = 'jw99999-a3001_20170327t121212_coron3_001_asn.json' override_psfmask = 'jwst_nircam_psfmask_somb.fits' # get a local copy of the inputs asn_file = self.get_data('test_coron3', asn_name) psfmask_file = self.get_data('test_coron3', override_psfmask) for file in raw_from_asn(asn_file): self.get_data('test_coron3', file) pipe = Coron3Pipeline() pipe.align_refs.override_psfmask = psfmask_file pipe.outlier_detection.resample_data = False pipe.run(asn_file) outputs = [ ( # Compare psfstack product 'jw99999-a3001_t1_nircam_f140m-maskbar_psfstack.fits', 'jw99999-a3001_t1_nircam_f140m-maskbar_psfstack_ref.fits'), ( # Compare psfalign product 'jw9999947001_02102_00001_nrcb3_a3001_psfalign.fits', 'jw99999-a3001_t1_nircam_f140m-maskbar_psfalign_ref.fits'), ( # Compare psfsub product 'jw9999947001_02102_00001_nrcb3_a3001_psfsub.fits', 'jw9999947001_02102_00001_nrcb3_psfsub_ref.fits'), ( # Compare level-2c products 'jw9999947001_02102_00001_nrcb3_a3001_crfints.fits', 'jw9999947001_02102_00001_nrcb3_a3001_crfints_ref.fits'), ('jw9999947001_02102_00002_nrcb3_a3001_crfints.fits', 'jw9999947001_02102_00002_nrcb3_a3001_crfints_ref.fits'), { 'files': ( # Compare i2d product 'jw99999-a3001_t1_nircam_f140m-maskbar_i2d.fits', 'jw99999-a3001_t1_nircam_f140m-maskbar_i2d_ref.fits'), 'pars': { 'ignore_hdus': self.ignore_hdus + ['HDRTAB'] } }, { 'files': ( # Compare the HDRTAB in the i2d product 'jw99999-a3001_t1_nircam_f140m-maskbar_i2d.fits[hdrtab]', 'jw99999-a3001_t1_nircam_f140m-maskbar_i2d_ref.fits[hdrtab]' ), 'pars': { 'ignore_keywords': self.ignore_keywords + ['NAXIS1', 'TFORM*'], 'ignore_fields': self.ignore_keywords } } ] self.compare_outputs(outputs)
def test_coron3_1(self): """Regression test of calwebb_coron3 pipeline. Test will be performed on NIRCam simulated data. """ asn_name = 'jw99999-a3001_20170327t121212_coron3_001_asn.json' override_psfmask = 'jwst_nircam_psfmask_somb.fits' # get a local copy of the inputs asn_file = self.get_data('test_coron3', asn_name) psfmask_file = self.get_data('test_coron3', override_psfmask) for file in raw_from_asn(asn_file): self.get_data('test_coron3', file) pipe = Coron3Pipeline() pipe.align_refs.override_psfmask = psfmask_file pipe.outlier_detection.resample_data = False pipe.run(asn_file) outputs = [( # Compare psfstack product 'jw99999-a3001_t1_nircam_f140m-maskbar_psfstack.fits', 'jw99999-a3001_t1_nircam_f140m-maskbar_psfstack_ref.fits' ), ( # Compare psfalign product 'jw9999947001_02102_00001_nrcb3_a3001_psfalign.fits', 'jw99999-a3001_t1_nircam_f140m-maskbar_psfalign_ref.fits' ), ( # Compare psfsub product 'jw9999947001_02102_00001_nrcb3_a3001_psfsub.fits', 'jw9999947001_02102_00001_nrcb3_psfsub_ref.fits' ), ( # Compare level-2c products 'jw9999947001_02102_00001_nrcb3_a3001_crfints.fits', 'jw9999947001_02102_00001_nrcb3_a3001_crfints_ref.fits' ), ( 'jw9999947001_02102_00002_nrcb3_a3001_crfints.fits', 'jw9999947001_02102_00002_nrcb3_a3001_crfints_ref.fits' ), {'files':( # Compare i2d product 'jw99999-a3001_t1_nircam_f140m-maskbar_i2d.fits', 'jw99999-a3001_t1_nircam_f140m-maskbar_i2d_ref.fits' ), 'pars': {'ignore_hdus':self.ignore_hdus+['HDRTAB']} }, {'files':( # Compare the HDRTAB in the i2d product 'jw99999-a3001_t1_nircam_f140m-maskbar_i2d.fits[hdrtab]', 'jw99999-a3001_t1_nircam_f140m-maskbar_i2d_ref.fits[hdrtab]' ), 'pars': {'ignore_keywords':self.ignore_keywords+['NAXIS1', 'TFORM*'], 'ignore_fields':self.ignore_keywords} } ] self.compare_outputs(outputs)
def test_miri_masterbackground_mrs(self): """Run masterbackground step on MIRI LRS association""" asn_file = self.get_data(*self.test_dir, 'miri_mrs_mbkg_0304_spec3_asn.json') for file in raw_from_asn(asn_file): self.get_data(*self.test_dir, file) collect_pipeline_cfgs('./config') result = MasterBackgroundStep.call( asn_file, config_file='config/master_background.cfg', save_background=True ) for model in result: assert model.meta.cal_step.master_background == 'COMPLETE'
def test_spec3_pipeline1(self): """ Regression test of calwebb_spec3 pipeline on simulated MIRI MRS dithered data. """ asn_file = self.get_data(self.test_dir, 'test_asn4.json') for file in raw_from_asn(asn_file): self.get_data(self.test_dir, file) step = Spec3Pipeline() step.save_bsub = False step.mrs_imatch.suffix = 'mrs_imatch' step.mrs_imatch.bkg_degree = 1 step.mrs_imatch.subtract = False step.outlier_detection.skip = True step.output_use_model = True step.resample_spec.save_results = True step.resample_spec.suffix = 's2d' step.cube_build.save_results = True step.cube_build.suffix = 's3d' step.extract_1d.save_results = True step.extract_1d.suffix = 'x1d' step.run(asn_file) outputs = [(# Compare cube product 1 'det_image_ch1-short_s3d.fits', 'det_image_ch1-short_s3d_ref.fits', ['primary', 'sci', 'err', 'dq', 'wmap']), (# Compare cube product 2 'det_image_ch2-short_s3d.fits', 'det_image_ch2-short_s3d_ref.fits', ['primary', 'sci', 'err', 'dq', 'wmap']), (# Compare x1d product 1 'det_image_ch1-short_x1d.fits', 'det_image_ch1-short_x1d_ref.fits', ['primary', 'extract1d']), (# Compare x1d product 2 'det_image_ch2-short_x1d.fits', 'det_image_ch2-short_x1d_ref.fits', ['primary', 'extract1d']) ] self.compare_outputs(outputs)
def test_spec3_pipeline1(self): """ Regression test of calwebb_spec3 pipeline on simulated MIRI MRS dithered data. """ asn_file = self.get_data(self.test_dir, 'test_asn4.json') for file in raw_from_asn(asn_file): self.get_data(self.test_dir, file) step = Spec3Pipeline() step.save_bsub = False step.mrs_imatch.suffix = 'mrs_imatch' step.mrs_imatch.bkg_degree = 1 step.mrs_imatch.subtract = False step.outlier_detection.skip = True step.output_use_model = True step.resample_spec.save_results = True step.resample_spec.suffix = 's2d' step.cube_build.save_results = True step.cube_build.suffix = 's3d' step.extract_1d.save_results = True step.extract_1d.suffix = 'x1d' step.run(asn_file) outputs = [ ( # Compare cube product 1 'det_image_ch1-short_s3d.fits', 'det_image_ch1-short_s3d_ref.fits', ['primary', 'sci', 'err', 'dq', 'wmap']), ( # Compare cube product 2 'det_image_ch2-short_s3d.fits', 'det_image_ch2-short_s3d_ref.fits', ['primary', 'sci', 'err', 'dq', 'wmap']), ( # Compare x1d product 1 'det_image_ch1-short_x1d.fits', 'det_image_ch1-short_x1d_ref.fits', ['primary', 'extract1d']), ( # Compare x1d product 2 'det_image_ch2-short_x1d.fits', 'det_image_ch2-short_x1d_ref.fits', ['primary', 'extract1d']) ] self.compare_outputs(outputs)
def test_ami_pipeline(self): """ Regression test of the AMI pipeline performed on NIRISS AMI data. """ asn_file = self.get_data(self.test_dir, 'test_lg1_asn.json') for file in raw_from_asn(asn_file): self.get_data(self.test_dir, file) pipe = Ami3Pipeline() pipe.save_averages = True pipe.ami_analyze.oversample = 3 pipe.ami_analyze.rotation = 1.49 pipe.run(asn_file) outputs = [('test_targ_aminorm.fits', 'ami_pipeline_targ_lgnorm.fits'), ] self.compare_outputs(outputs, rtol=0.00001, ignore_hdus=['ASDF', 'HDRTAB'])
def test_tso3_pipeline_nis(self): """Regression test of calwebb_tso3 on NIRISS SOSS simulated data. """ asn_file = self.get_data( self.test_dir, "jw87600-a3001_20170527t111213_tso3_001_asn.json") for file in raw_from_asn(asn_file): self.get_data(self.test_dir, file) Tso3Pipeline.call(asn_file) outputs = [ # Compare level-2c product ('jw87600024001_02101_00001_nis_a3001_crfints.fits', 'jw87600-a3001_t1_niriss_clear-gr700xd_crfints_ref.fits', ['primary', 'sci', 'dq', 'err']), # Compare level-3 product ('jw87600-a3001_t1_niriss_clear-gr700xd_x1dints.fits', 'jw87600-a3001_t1_niriss_clear-gr700xd_x1dints_ref.fits', ['primary', 'extract1d']) ] self.compare_outputs(outputs)
def test_nrs_msa_spec3(self): """ Regression test of calwebb_spec3 pipeline performed on NIRSpec MSA data """ cfg_dir = './cfgs' collect_pipeline_cfgs(cfg_dir) asn_file = self.get_data(*self.test_dir, 'single_asn.json') for file in raw_from_asn(asn_file): self.get_data(*self.test_dir, file) args = [str(Path(cfg_dir) / 'calwebb_spec3.cfg'), asn_file] Step.from_cmdline(args) # Compare results truths = self.data_glob(*self.ref_loc, glob='*.fits') outputs = [(Path(output_file).name, ) * 2 for output_file in truths] self.compare_outputs(outputs)
def test_wfs_combine2(self): """ Regression test of wfs_combine using do_refine=True """ asn_file = self.get_data(self.test_dir, 'wfs_3sets_asn3.json') for file in raw_from_asn(asn_file): self.get_data(self.test_dir, file) WfsCombineStep.call(asn_file, do_refine=True) outputs = [('test_wfscom3_wfscmb.fits', 'test_wfscom_do_ref.fits'), ('test_wfscom3a_wfscmb.fits', 'test_wfscoma_do_ref.fits'), ('test_wfscom3b_wfscmb.fits', 'test_wfscomb_do_ref.fits') ] self.compare_outputs(outputs)
def test_nis_wfss_spec2(self): """ Regression test of calwebb_spec2 pipeline performed on NIRISS WFSS data. """ # Collect data asn_file = self.get_data(self.test_dir, 'jw87600-a3001_20171109T145456_spec2_001_asn.json') for file in raw_from_asn(asn_file): self.get_data(self.test_dir, file) # Run the step collect_pipeline_cfgs('cfgs') Spec2Pipeline.call(asn_file, config_file='cfgs/calwebb_spec2.cfg', save_bsub=True) # Test results. outputs = [('jw87600017001_02101_00002_nis_cal.fits', 'jw87600017001_02101_00002_nis_cal_ref.fits'), ('jw87600017001_02101_00002_nis_x1d.fits', 'jw87600017001_02101_00002_nis_x1d_ref.fits')] self.compare_outputs(outputs)
def test_tso3_pipeline_nis(self): """Regression test of calwebb_tso3 on NIRISS SOSS simulated data. """ asn_file = self.get_data(self.test_dir, "jw87600-a3001_20170527t111213_tso3_001_asn.json") for file in raw_from_asn(asn_file): self.get_data(self.test_dir, file) Tso3Pipeline.call(asn_file) outputs = [# Compare level-2c product ('jw87600024001_02101_00001_nis_a3001_crfints.fits', 'jw87600-a3001_t1_niriss_clear-gr700xd_crfints_ref.fits', ['primary', 'sci', 'dq', 'err']), # Compare level-3 product ('jw87600-a3001_t1_niriss_clear-gr700xd_x1dints.fits', 'jw87600-a3001_t1_niriss_clear-gr700xd_x1dints_ref.fits', ['primary', 'extract1d']) ] self.compare_outputs(outputs)
def test_wfs_combine(self): """ Regression test of wfs_combine using do_refine=False (default) Association table has 3 (identical) pairs of input files to combine """ asn_file = self.get_data(self.test_dir, 'wfs_3sets_asn.json') for file in raw_from_asn(asn_file): self.get_data(self.test_dir, file) WfsCombineStep.call(asn_file) outputs = [('test_wfscom_wfscmb.fits', 'test_wfscom.fits'), ('test_wfscoma_wfscmb.fits', 'test_wfscoma.fits'), ('test_wfscomb_wfscmb.fits', 'test_wfscomb.fits') ] self.compare_outputs(outputs)
def test_nis_wfss_spec2(self): """ Regression test of calwebb_spec2 pipeline performed on NIRISS WFSS data. """ asn_file = self.get_data(self.test_dir, 'jw87600-a3001_20171109T145456_spec2_001_asn.json') for file in raw_from_asn(asn_file): self.get_data(self.test_dir, file) pipe = Spec2Pipeline() pipe.save_bsub = True pipe.save_results = True pipe.resample_spec.save_results = True pipe.extract_1d.save_results = True pipe.run(asn_file) outputs = [('jw87600017001_02101_00002_nis_cal.fits', 'jw87600017001_02101_00002_nis_cal_ref.fits'), ('jw87600017001_02101_00002_nis_x1d.fits', 'jw87600017001_02101_00002_nis_x1d_ref.fits')] self.compare_outputs(outputs)
def test_image3_pipeline2(self): """Regression test definitions for CALIMAGE3 pipeline. Regression test of calwebb_image3 pipeline on NIRCam simulated long-wave data with a 6-point dither. """ asn_file = self.get_data( self.test_dir, "jw10002-o001_20171116t191235_image3_002_asn.json") for file in raw_from_asn(asn_file): self.get_data(self.test_dir, file) collect_pipeline_cfgs('config') args = [ 'config/calwebb_image3.cfg', asn_file, '--steps.tweakreg.kernel_fwhm=2', '--steps.tweakreg.snr_threshold=5', '--steps.tweakreg.enforce_user_order=True', '--steps.tweakreg.searchrad=10', '--steps.tweakreg.fitgeometry=rscale' ] Step.from_cmdline(args) outputs = [('jw10002001001_01101_00004_nrcblong_o001_crf.fits', 'jw10002001001_01101_00004_nrcblong_o001_crf_ref.fits'), { 'files': ('jw10002-o001_t002_nircam_f444w_i2d.fits', 'jw10002-o001_t002_nircam_f444w_i2d_ref.fits'), 'pars': { 'ignore_keywords': self.ignore_keywords + ['NAXIS1', 'TFORM*'], 'ignore_fields': self.ignore_keywords, 'rtol': 0.0001 } }] self.compare_outputs(outputs)
def test_miri_lrs_bkgnod(self): """ Regression test of calwebb_spec2 pipeline performed on an association of nodded MIRI LRS fixed-slit exposures. """ asn_file = self.get_data(self.test_dir, 'lrs_bkgnod_asn.json') for file in raw_from_asn(asn_file): self.get_data(self.test_dir, file) step = Spec2Pipeline() step.save_bsub=True, step.save_results=True step.resample_spec.save_results = True step.cube_build.save_results = True step.extract_1d.save_results = True step.run(asn_file) outputs = [('test_lrs1_bsub.fits', 'test_lrs1_bsub_ref.fits', ['primary','sci','err','dq']), ('test_lrs2_bsub.fits','test_lrs2_bsub_ref.fits', ['primary','sci','err','dq']), ('test_lrs3_bsub.fits','test_lrs3_bsub_ref.fits', ['primary','sci','err','dq']), ('test_lrs4_bsub.fits','test_lrs4_bsub_ref.fits', ['primary','sci','err','dq']), ('test_lrs1_cal.fits', 'test_lrs1_cal_ref.fits', ['primary','sci','err','dq','relsens']), ('test_lrs2_cal.fits', 'test_lrs2_cal_ref.fits', ['primary','sci','err','dq','relsens']), ('test_lrs3_cal.fits', 'test_lrs3_cal_ref.fits', ['primary','sci','err','dq','relsens']), ('test_lrs4_cal.fits', 'test_lrs4_cal_ref.fits', ['primary','sci','err','dq','relsens']) ] self.compare_outputs(outputs)