def _save_stack(self, stack_arr, stack_name, master_hdr): CCDData.write(stack_arr, os.path.join(self.output_directory, stack_name), hdu_mask=None, hdu_uncertainty=None, overwrite=True) f = fits.open(os.path.join(self.output_directory, stack_name), mode='update') f[0].header = master_hdr f.verify('silentfix') f.flush() self.info('Saving stack {} finished'.format(stack_name))
class FitsFileIOAndOps(TestCase): def setUp(self): self.fake_image = CCDData(data=np.ones((100, 100)), meta=fits.Header(), unit='adu') self.file_name = 'sample_file.fits' self.target_non_zero = 4 self.current_directory = os.getcwd() self.full_path = os.path.join(self.current_directory, self.file_name) self.parent_file = 'parent_file.fits' self.fake_image.header.set('CCDSUM', value='1 1', comment='Fake values') self.fake_image.header.set('OBSTYPE', value='OBJECT', comment='Fake values') self.fake_image.header.set('GSP_FNAM', value=self.file_name, comment='Fake values') self.fake_image.header.set('GSP_PNAM', value=self.parent_file, comment='Fake values') self.fake_image.write(self.full_path, overwrite=False) def test_write_fits(self): self.assertTrue(os.path.isfile(self.full_path)) os.remove(self.full_path) write_fits(ccd=self.fake_image, full_path=self.full_path, parent_file=self.parent_file, overwrite=False) self.assertTrue(os.path.isfile(self.full_path)) def test_read_fits(self): self.recovered_fake_image = read_fits(self.full_path) self.assertIsInstance(self.recovered_fake_image, CCDData) def test_image_overscan(self): data_value = 100. overscan_value = 0.1 # alter overscan region to a lower number self.fake_image.data *= data_value self.fake_image.data[:, 0:5] = overscan_value overscan_region = '[1:6,:]' self.assertEqual(self.fake_image.data[:, 6:99].mean(), data_value) self.assertEqual(self.fake_image.data[:, 0:5].mean(), overscan_value) self.fake_image = image_overscan(ccd=self.fake_image, overscan_region=overscan_region) self.assertEqual(self.fake_image.data[:, 6:99].mean(), data_value - overscan_value) self.assertEqual(self.fake_image.header['GSP_OVER'], overscan_region) def test_image_overscan_none(self): new_fake_image = image_overscan(ccd=self.fake_image, overscan_region=None) self.assertEqual(new_fake_image, self.fake_image) def test_image_trim(self): self.assertEqual(self.fake_image.data.shape, (100, 100)) trim_section = '[1:50,:]' self.fake_image = image_trim(ccd=self.fake_image, trim_section=trim_section, trim_type='trimsec') self.assertEqual(self.fake_image.data.shape, (100, 50)) self.assertEqual(self.fake_image.header['GSP_TRIM'], trim_section) def test_save_extracted_target_zero(self): self.fake_image.header.set('GSP_FNAM', value=self.file_name) same_fake_image = save_extracted(ccd=self.fake_image, destination=self.current_directory, prefix='e', target_number=0) self.assertEqual(same_fake_image, self.fake_image) self.assertTrue(os.path.isfile('e' + self.file_name)) def test_save_extracted_target_non_zero(self): self.fake_image.header.set('GSP_FNAM', value=self.file_name) same_fake_image = save_extracted(ccd=self.fake_image, destination=self.current_directory, prefix='e', target_number=self.target_non_zero) self.assertEqual(same_fake_image, self.fake_image) self.assertTrue(os.path.isfile('e' + re.sub('.fits', '_target_{:d}.fits'.format( self.target_non_zero), self.file_name))) def test_save_extracted_target_zero_comp(self): self.fake_image.header.set('GSP_FNAM', value=self.file_name) self.fake_image.header.set('OBSTYPE', value='COMP') self.fake_image.header.set('GSP_EXTR', value='100.00:101.00') same_fake_image = save_extracted(ccd=self.fake_image, destination=self.current_directory, prefix='e', target_number=0) self.assertEqual(same_fake_image, self.fake_image) self.assertTrue(os.path.isfile(self.fake_image.header['GSP_FNAM'])) def tearDown(self): files_to_remove = [self.full_path, self.fake_image.header['GSP_FNAM']] for _file in files_to_remove: if os.path.isfile(_file): os.unlink(_file)
class MasterFlatTest(TestCase): def setUp(self): # create a master flat self.master_flat = CCDData(data=np.ones((100, 100)), meta=fits.Header(), unit='adu') self.master_flat.header.set('GRATING', value='RALC_1200-BLUE') self.master_flat.header.set('SLIT', value='0.84" long slit') self.master_flat.header.set('FILTER2', value='<NO FILTER>') self.master_flat.header.set('WAVMODE', value='1200 m2') self.master_flat_name = 'master_flat_1200m2.fits' # expected master flat to be retrieved by get_best_flat self.reference_flat_name = 'master_flat_1200m2_0.84_dome.fits' # location of sample flats self.flat_path = 'goodman_pipeline/data/test_data/master_flat' slit = re.sub('[A-Za-z" ]', '', self.master_flat.header['SLIT']) self.flat_name_base = re.sub('.fits', '_' + slit + '*.fits', self.master_flat_name) # save a master flat with some random structure. self.master_flat_name_norm = 'flat_to_normalize.fits' # add a bias level self.master_flat.data += 300. # add noise self.master_flat.data += np.random.random_sample( self.master_flat.data.shape) self.master_flat.write(os.path.join(self.flat_path, self.master_flat_name_norm), overwrite=False) def tearDown(self): full_path = os.path.join(self.flat_path, self.master_flat_name_norm) self.assertTrue(os.path.isfile(full_path)) if os.path.isfile(full_path): os.unlink(full_path) self.assertFalse(os.path.isfile(full_path)) # remove normalized flat norm_flat = re.sub('flat_to_', 'norm_flat_to_', full_path) if os.path.isfile(norm_flat): os.unlink(norm_flat) self.assertFalse(os.path.isfile(norm_flat)) def test_get_best_flat(self): # print(self.flat_name_base) master_flat, master_flat_name = get_best_flat( flat_name=self.flat_name_base, path=self.flat_path) self.assertIsInstance(master_flat, CCDData) self.assertEqual(os.path.basename(master_flat_name), self.reference_flat_name) def test_get_best_flat_fail(self): # Introduce an error that will never produce a result. wrong_flat_name = re.sub('1200m2', '1300m2', self.flat_name_base) master_flat, master_flat_name = get_best_flat( flat_name=wrong_flat_name, path=self.flat_path) self.assertIsNone(master_flat) self.assertIsNone(master_flat_name) def test_normalize_master_flat(self): methods = ['mean', 'simple', 'full'] for method in methods: self.assertNotAlmostEqual(self.master_flat.data.mean(), 1.) normalized_flat, normalized_flat_name = normalize_master_flat( master=self.master_flat, name=os.path.join(self.flat_path, self.master_flat_name_norm), method=method) self.assertAlmostEqual(normalized_flat.data.mean(), 1., delta=0.001) self.assertEqual(normalized_flat.header['GSP_NORM'], method) self.assertIn('norm_', normalized_flat_name)
#reduce the arc frames for filename in ic1.files_filtered(obstype='Arc', isiarm='Blue arm'): hdu = fits.open(ic1.location + filename) ccd = CCDData(hdu[1].data, header=hdu[0].header + hdu[1].header, unit=u.adu) #this has to be fixed as the bias section does not include the whole section that will be trimmed ccd = ccdproc.subtract_overscan(ccd, median=True, overscan_axis=0, fits_section='[1:966,4105:4190]') ccd = ccdproc.trim_image(ccd, fits_section=ccd.header['TRIMSEC']) ccd = ccdproc.subtract_bias(ccd, master_bias_blue) ccd = ccdproc.flat_correct(ccd, master_flat_blue) ccd.data = ccd.data.T ccd.write('arc_' + filename, clobber=True) red_flat_list = [] for filename in ic1.files_filtered(obstype='Arc', isiarm='Red arm'): hdu = fits.open(ic1.location + filename) ccd = CCDData(hdu[1].data, header=hdu[0].header + hdu[1].header, unit=u.adu) #this has to be fixed as the bias section does not include the whole section that will be trimmed ccd = ccdproc.subtract_overscan(ccd, median=True, overscan_axis=0, fits_section='[1:966,4105:4190]') ccd = ccdproc.trim_image(ccd, fits_section=ccd.header['TRIMSEC']) ccd = ccdproc.subtract_bias(ccd, master_bias_red) ccd = ccdproc.flat_correct(ccd, master_flat_red)
def create_fake_spectroscopic_data(self): if os.path.isdir(self.raw_path): card_values = [{ 'obstype': 'OBJECT', 'object': 'NGC2070', 'obsra': '16:23:24.285', 'obsdec': '-39:12:53.954' }, { 'obstype': 'OBJECT', 'object': 'NGC2070', 'obsra': '16:24:34.285', 'obsdec': '-39:13:53.954' }, { 'obstype': 'OBJECT', 'object': 'NGC2070', 'obsra': '16:24:34.285', 'obsdec': '-39:13:53.954' }, { 'obstype': 'OBJECT', 'object': 'NGC2070', 'obsra': '16:24:34.285', 'obsdec': '-39:13:53.954' }, { 'obstype': 'OBJECT', 'object': 'NGC2070', 'obsra': '16:25:34.285', 'obsdec': '-39:13:53.954' }, { 'obstype': 'OBJECT', 'object': 'NGC2070', 'obsra': '16:25:34.285', 'obsdec': '-39:13:53.954' }, { 'obstype': 'OBJECT', 'object': 'NGC2070', 'obsra': '16:25:34.285', 'obsdec': '-39:13:53.954' }, { 'obstype': 'OBJECT', 'object': 'NGC2070', 'obsra': '16:26:34.285', 'obsdec': '-39:13:53.954' }, { 'obstype': 'OBJECT', 'object': 'NGC2070', 'obsra': '16:27:34.285', 'obsdec': '-39:13:53.954' }, { 'obstype': 'OBJECT', 'object': 'NGC2070', 'obsra': '16:28:34.285', 'obsdec': '-39:13:53.954' }, { 'obstype': 'OBJECT', 'object': 'NGC2070', 'obsra': '16:23:34.285', 'obsdec': ' 39:13:53.954' }, { 'obstype': 'OBJECT', 'object': 'NGC2070', 'obsra': '16:23:34.285', 'obsdec': ' 39:13:53.954' }, { 'obstype': 'OBJECT', 'object': 'NGC2070', 'obsra': '16:23:34.285', 'obsdec': ' 39:13:53.954' }, { 'obstype': 'OBJECT', 'object': 'NGC2070', 'obsra': '16:23:34.285', 'obsdec': ' 39:13:53.954' }, { 'obstype': 'OBJECT', 'object': 'NGC2070', 'obsra': '16:23:34.285', 'obsdec': ' 39:13:53.954' }, { 'obstype': 'COMP', 'object': 'HgArNe', 'obsra': '16:22:34.285', 'obsdec': '-39:23:53.954' }, { 'obstype': 'COMP', 'object': 'HgArNe', 'obsra': '16:24:34.285', 'obsdec': '-39:13:53.954' }, { 'obstype': 'COMP', 'object': 'HgArNe', 'obsra': '16:24:34.285', 'obsdec': '-39:13:53.954' }, { 'obstype': 'COMP', 'object': 'HgArNe', 'obsra': '16:24:34.285', 'obsdec': '-39:13:53.954' }, { 'obstype': 'COMP', 'object': 'HgArNe', 'obsra': '16:25:34.285', 'obsdec': '-39:13:53.954' }, { 'obstype': 'COMP', 'object': 'HgArNe', 'obsra': '16:25:34.285', 'obsdec': '-39:13:53.954' }, { 'obstype': 'COMP', 'object': 'HgArNe', 'obsra': '16:25:34.285', 'obsdec': '-39:13:53.954' }, { 'obstype': 'COMP', 'object': 'HgArNe', 'obsra': '16:26:34.285', 'obsdec': '-39:13:53.954' }, { 'obstype': 'COMP', 'object': 'HgArNe', 'obsra': '16:27:34.285', 'obsdec': '-39:13:53.954' }, { 'obstype': 'COMP', 'object': 'HgArNe', 'obsra': '16:28:34.285', 'obsdec': '-39:13:53.954' }, { 'obstype': 'COMP', 'object': 'HgArNe', 'obsra': '16:23:34.285', 'obsdec': '39:13:53.954' }, { 'obstype': 'COMP', 'object': 'HgArNe', 'obsra': '16:23:34.285', 'obsdec': '39:13:53.954' }, { 'obstype': 'COMP', 'object': 'HgArNe', 'obsra': '16:23:34.285', 'obsdec': '39:13:53.954' }, { 'obstype': 'COMP', 'object': 'HgArNe', 'obsra': '16:23:34.285', 'obsdec': '39:13:53.954' }, { 'obstype': 'COMP', 'object': 'HgArNe', 'obsra': '16:23:34.285', 'obsdec': '39:13:53.954' }] for i in range(len(card_values)): ccd = CCDData(data=np.ones((3, 3)), meta=fits.Header(), unit='adu') ccd.header.set('DATE', value='2019-03-22', comment='nc') ccd.header.set('SLIT', value='1.0" long slit', comment='nc') ccd.header.set('DATE-OBS', value='2019-03-22T09:59:33.654', comment='nc') ccd.header.set('OBSTYPE', value=card_values[i]['obstype'], comment='nc') ccd.header.set('OBJECT', value=card_values[i]['object'], comment='nc') ccd.header.set('EXPTIME', value='10', comment='nc') ccd.header.set('OBSRA', value=card_values[i]['obsra'], comment='nc') ccd.header.set('OBSDEC', value=card_values[i]['obsdec'], comment='nc') ccd.header.set('GRATING', value='SYZY_400', comment='nc') ccd.header.set('CAM_TARG', value='16.1', comment='nc') ccd.header.set('GRT_TARG', value='7.5', comment='nc') ccd.header.set('FILTER', value='<NO FILTER>', comment='nc') ccd.header.set('FILTER2', value='GG455', comment='nc') ccd.header.set('GAIN', value='1.48', comment='nc') ccd.header.set('RDNOISE', value='3.89', comment='nc') ccd.header.set('INSTCONF', value='Red', comment='nc') ccd.header.set('WAVMODE', value='Spectroscopy', comment='nc') ccd.write( os.path.join(self.raw_path, 'test_file_{:03d}.fits'.format(i)))
class FitsFileIOAndOps(TestCase): def setUp(self): self.fake_image = CCDData(data=np.ones((100, 100)), meta=fits.Header(), unit='adu') self.file_name = 'sample_file.fits' self.target_non_zero = 4 self.current_directory = os.getcwd() self.full_path = os.path.join(self.current_directory, self.file_name) self.parent_file = 'parent_file.fits' self.fake_image.header.set('CCDSUM', value='1 1', comment='Fake values') self.fake_image.header.set('OBSTYPE', value='OBJECT', comment='Fake values') self.fake_image.header.set('GSP_FNAM', value=self.file_name, comment='Fake values') self.fake_image.header.set('GSP_PNAM', value=self.parent_file, comment='Fake values') self.fake_image.write(self.full_path, overwrite=False) def test_write_fits(self): self.assertTrue(os.path.isfile(self.full_path)) os.remove(self.full_path) write_fits(ccd=self.fake_image, full_path=self.full_path, parent_file=self.parent_file, overwrite=False) self.assertTrue(os.path.isfile(self.full_path)) def test_read_fits(self): self.recovered_fake_image = read_fits(self.full_path) self.assertIsInstance(self.recovered_fake_image, CCDData) def test_image_overscan(self): data_value = 100. overscan_value = 0.1 # alter overscan region to a lower number self.fake_image.data *= data_value self.fake_image.data[:, 0:5] = overscan_value overscan_region = '[1:6,:]' self.assertEqual(self.fake_image.data[:, 6:99].mean(), data_value) self.assertEqual(self.fake_image.data[:, 0:5].mean(), overscan_value) self.fake_image = image_overscan(ccd=self.fake_image, overscan_region=overscan_region) self.assertEqual(self.fake_image.data[:, 6:99].mean(), data_value - overscan_value) self.assertEqual(self.fake_image.header['GSP_OVER'], overscan_region) def test_image_overscan_none(self): new_fake_image = image_overscan(ccd=self.fake_image, overscan_region=None) self.assertEqual(new_fake_image, self.fake_image) def test_image_trim(self): self.assertEqual(self.fake_image.data.shape, (100, 100)) trim_section = '[1:50,:]' self.fake_image = image_trim(ccd=self.fake_image, trim_section=trim_section, trim_type='trimsec') self.assertEqual(self.fake_image.data.shape, (100, 50)) self.assertEqual(self.fake_image.header['GSP_TRIM'], trim_section) def test_save_extracted_target_zero(self): self.fake_image.header.set('GSP_FNAM', value=self.file_name) same_fake_image = save_extracted(ccd=self.fake_image, destination=self.current_directory, prefix='e', target_number=0) self.assertEqual(same_fake_image, self.fake_image) self.assertTrue(os.path.isfile('e' + self.file_name)) def test_save_extracted_target_non_zero(self): self.fake_image.header.set('GSP_FNAM', value=self.file_name) same_fake_image = save_extracted(ccd=self.fake_image, destination=self.current_directory, prefix='e', target_number=self.target_non_zero) self.assertEqual(same_fake_image, self.fake_image) self.assertTrue( os.path.isfile( 'e' + re.sub('.fits', '_target_{:d}.fits'.format( self.target_non_zero), self.file_name))) def test_save_extracted_target_zero_comp(self): self.fake_image.header.set('GSP_FNAM', value=self.file_name) self.fake_image.header.set('OBSTYPE', value='COMP') self.fake_image.header.set('GSP_EXTR', value='100.00:101.00') same_fake_image = save_extracted(ccd=self.fake_image, destination=self.current_directory, prefix='e', target_number=0) self.assertEqual(same_fake_image, self.fake_image) self.assertTrue(os.path.isfile(self.fake_image.header['GSP_FNAM'])) def tearDown(self): files_to_remove = [self.full_path, self.fake_image.header['GSP_FNAM']] for _file in files_to_remove: if os.path.isfile(_file): os.unlink(_file)
class MasterFlatTest(TestCase): def setUp(self): # create a master flat self.master_flat = CCDData(data=np.ones((100, 100)), meta=fits.Header(), unit='adu') self.master_flat.header.set('GRATING', value='RALC_1200-BLUE') self.master_flat.header.set('SLIT', value='0.84" long slit') self.master_flat.header.set('FILTER2', value='<NO FILTER>') self.master_flat.header.set('WAVMODE', value='1200 m2') self.master_flat_name = 'master_flat_1200m2.fits' # expected master flat to be retrieved by get_best_flat self.reference_flat_name = 'master_flat_1200m2_0.84_dome.fits' # location of sample flats self.flat_path = 'goodman_pipeline/data/test_data/master_flat' slit = re.sub('[A-Za-z" ]', '', self.master_flat.header['SLIT']) self.flat_name_base = re.sub('.fits', '_' + slit + '*.fits', self.master_flat_name) # save a master flat with some random structure. self.master_flat_name_norm = 'flat_to_normalize.fits' # add a bias level self.master_flat.data += 300. # add noise self.master_flat.data += np.random.random_sample( self.master_flat.data.shape) self.master_flat.write(os.path.join(self.flat_path, self.master_flat_name_norm), overwrite=False) def tearDown(self): full_path = os.path.join(self.flat_path, self.master_flat_name_norm) self.assertTrue(os.path.isfile(full_path)) if os.path.isfile(full_path): os.unlink(full_path) self.assertFalse(os.path.isfile(full_path)) # remove normalized flat norm_flat = re.sub('flat_to_', 'norm_flat_to_', full_path) if os.path.isfile(norm_flat): os.unlink(norm_flat) self.assertFalse(os.path.isfile(norm_flat)) def test_get_best_flat(self): # print(self.flat_name_base) master_flat, master_flat_name = get_best_flat( flat_name=self.flat_name_base, path=self.flat_path) self.assertIsInstance(master_flat, CCDData) self.assertEqual(os.path.basename(master_flat_name), self.reference_flat_name) def test_get_best_flat_fail(self): # Introduce an error that will never produce a result. wrong_flat_name = re.sub('1200m2', '1300m2', self.flat_name_base) master_flat, master_flat_name = get_best_flat( flat_name=wrong_flat_name, path=self.flat_path) self.assertIsNone(master_flat) self.assertIsNone(master_flat_name) def test_normalize_master_flat(self): methods = ['mean', 'simple', 'full'] for method in methods: self.assertNotAlmostEqual(self.master_flat.data.mean(), 1.) normalized_flat, normalized_flat_name = normalize_master_flat( master=self.master_flat, name=os.path.join(self.flat_path, self.master_flat_name_norm), method=method) self.assertAlmostEqual(normalized_flat.data.mean(), 1., delta=0.001) self.assertEqual(normalized_flat.header['GSP_NORM'], method) self.assertIn('norm_', normalized_flat_name)
ccd = ccdproc.subtract_bias(ccd, master_bias_blue) blue_flat_list.append(ccd) master_flat_blue = ccdproc.combine(blue_flat_list, method='median') master_flat_blue.write('master_flat_blue.fits', clobber=True) #reduce the arc frames for filename in ic1.files_filtered(obstype='Arc', isiarm='Blue arm'): hdu = fits.open(ic1.location + filename) ccd = CCDData(hdu[1].data, header=hdu[0].header+hdu[1].header, unit = u.adu) #this has to be fixed as the bias section does not include the whole section that will be trimmed ccd = ccdproc.subtract_overscan(ccd, median=True, overscan_axis=0, fits_section='[1:966,4105:4190]') ccd = ccdproc.trim_image(ccd, fits_section=ccd.header['TRIMSEC'] ) ccd = ccdproc.subtract_bias(ccd, master_bias_blue) ccd = ccdproc.flat_correct(ccd, master_flat_blue) ccd.data = ccd.data.T ccd.write('arc_'+filename, clobber=True) red_flat_list = [] for filename in ic1.files_filtered(obstype='Arc', isiarm='Red arm'): hdu = fits.open(ic1.location + filename) ccd = CCDData(hdu[1].data, header=hdu[0].header+hdu[1].header, unit = u.adu) #this has to be fixed as the bias section does not include the whole section that will be trimmed ccd = ccdproc.subtract_overscan(ccd, median=True, overscan_axis=0, fits_section='[1:966,4105:4190]') ccd = ccdproc.trim_image(ccd, fits_section=ccd.header['TRIMSEC'] ) ccd = ccdproc.subtract_bias(ccd, master_bias_red) ccd = ccdproc.flat_correct(ccd, master_flat_red) ccd.data = ccd.data.T ccd.write('arc_'+filename, clobber=True) #reduce the sky frames for filename in ic1.files_filtered(obstype='Sky', isiarm='Blue arm'):