def _save_stack(self, stack_arr, stack_name, master_hdr):
CCDData.write(stack_arr,
os.path.join(self.output_directory, stack_name),
hdu_mask=None,
hdu_uncertainty=None,
overwrite=True)
f = fits.open(os.path.join(self.output_directory, stack_name),
mode='update')
f[0].header = master_hdr
f.verify('silentfix')
f.flush()
self.info('Saving stack {} finished'.format(stack_name))
class FitsFileIOAndOps(TestCase):
def setUp(self):
self.fake_image = CCDData(data=np.ones((100, 100)),
meta=fits.Header(),
unit='adu')
self.file_name = 'sample_file.fits'
self.target_non_zero = 4
self.current_directory = os.getcwd()
self.full_path = os.path.join(self.current_directory, self.file_name)
self.parent_file = 'parent_file.fits'
self.fake_image.header.set('CCDSUM',
value='1 1',
comment='Fake values')
self.fake_image.header.set('OBSTYPE',
value='OBJECT',
comment='Fake values')
self.fake_image.header.set('GSP_FNAM',
value=self.file_name,
comment='Fake values')
self.fake_image.header.set('GSP_PNAM',
value=self.parent_file,
comment='Fake values')
self.fake_image.write(self.full_path, overwrite=False)
def test_write_fits(self):
self.assertTrue(os.path.isfile(self.full_path))
os.remove(self.full_path)
write_fits(ccd=self.fake_image,
full_path=self.full_path,
parent_file=self.parent_file,
overwrite=False)
self.assertTrue(os.path.isfile(self.full_path))
def test_read_fits(self):
self.recovered_fake_image = read_fits(self.full_path)
self.assertIsInstance(self.recovered_fake_image, CCDData)
def test_image_overscan(self):
data_value = 100.
overscan_value = 0.1
# alter overscan region to a lower number
self.fake_image.data *= data_value
self.fake_image.data[:, 0:5] = overscan_value
overscan_region = '[1:6,:]'
self.assertEqual(self.fake_image.data[:, 6:99].mean(), data_value)
self.assertEqual(self.fake_image.data[:, 0:5].mean(), overscan_value)
self.fake_image = image_overscan(ccd=self.fake_image,
overscan_region=overscan_region)
self.assertEqual(self.fake_image.data[:, 6:99].mean(),
data_value - overscan_value)
self.assertEqual(self.fake_image.header['GSP_OVER'], overscan_region)
def test_image_overscan_none(self):
new_fake_image = image_overscan(ccd=self.fake_image,
overscan_region=None)
self.assertEqual(new_fake_image, self.fake_image)
def test_image_trim(self):
self.assertEqual(self.fake_image.data.shape, (100, 100))
trim_section = '[1:50,:]'
self.fake_image = image_trim(ccd=self.fake_image,
trim_section=trim_section,
trim_type='trimsec')
self.assertEqual(self.fake_image.data.shape, (100, 50))
self.assertEqual(self.fake_image.header['GSP_TRIM'], trim_section)
def test_save_extracted_target_zero(self):
self.fake_image.header.set('GSP_FNAM', value=self.file_name)
same_fake_image = save_extracted(ccd=self.fake_image,
destination=self.current_directory,
prefix='e',
target_number=0)
self.assertEqual(same_fake_image, self.fake_image)
self.assertTrue(os.path.isfile('e' + self.file_name))
def test_save_extracted_target_non_zero(self):
self.fake_image.header.set('GSP_FNAM', value=self.file_name)
same_fake_image = save_extracted(ccd=self.fake_image,
destination=self.current_directory,
prefix='e',
target_number=self.target_non_zero)
self.assertEqual(same_fake_image, self.fake_image)
self.assertTrue(os.path.isfile('e' + re.sub('.fits',
'_target_{:d}.fits'.format(
self.target_non_zero),
self.file_name)))
def test_save_extracted_target_zero_comp(self):
self.fake_image.header.set('GSP_FNAM', value=self.file_name)
self.fake_image.header.set('OBSTYPE', value='COMP')
self.fake_image.header.set('GSP_EXTR', value='100.00:101.00')
same_fake_image = save_extracted(ccd=self.fake_image,
destination=self.current_directory,
prefix='e',
target_number=0)
self.assertEqual(same_fake_image, self.fake_image)
self.assertTrue(os.path.isfile(self.fake_image.header['GSP_FNAM']))
def tearDown(self):
files_to_remove = [self.full_path, self.fake_image.header['GSP_FNAM']]
for _file in files_to_remove:
if os.path.isfile(_file):
os.unlink(_file)
class MasterFlatTest(TestCase):
def setUp(self):
# create a master flat
self.master_flat = CCDData(data=np.ones((100, 100)),
meta=fits.Header(),
unit='adu')
self.master_flat.header.set('GRATING', value='RALC_1200-BLUE')
self.master_flat.header.set('SLIT', value='0.84" long slit')
self.master_flat.header.set('FILTER2', value='<NO FILTER>')
self.master_flat.header.set('WAVMODE', value='1200 m2')
self.master_flat_name = 'master_flat_1200m2.fits'
# expected master flat to be retrieved by get_best_flat
self.reference_flat_name = 'master_flat_1200m2_0.84_dome.fits'
# location of sample flats
self.flat_path = 'goodman_pipeline/data/test_data/master_flat'
slit = re.sub('[A-Za-z" ]',
'',
self.master_flat.header['SLIT'])
self.flat_name_base = re.sub('.fits',
'_' + slit + '*.fits',
self.master_flat_name)
# save a master flat with some random structure.
self.master_flat_name_norm = 'flat_to_normalize.fits'
# add a bias level
self.master_flat.data += 300.
# add noise
self.master_flat.data += np.random.random_sample(
self.master_flat.data.shape)
self.master_flat.write(os.path.join(self.flat_path,
self.master_flat_name_norm),
overwrite=False)
def tearDown(self):
full_path = os.path.join(self.flat_path,
self.master_flat_name_norm)
self.assertTrue(os.path.isfile(full_path))
if os.path.isfile(full_path):
os.unlink(full_path)
self.assertFalse(os.path.isfile(full_path))
# remove normalized flat
norm_flat = re.sub('flat_to_', 'norm_flat_to_', full_path)
if os.path.isfile(norm_flat):
os.unlink(norm_flat)
self.assertFalse(os.path.isfile(norm_flat))
def test_get_best_flat(self):
# print(self.flat_name_base)
master_flat, master_flat_name = get_best_flat(
flat_name=self.flat_name_base,
path=self.flat_path)
self.assertIsInstance(master_flat, CCDData)
self.assertEqual(os.path.basename(master_flat_name),
self.reference_flat_name)
def test_get_best_flat_fail(self):
# Introduce an error that will never produce a result.
wrong_flat_name = re.sub('1200m2', '1300m2', self.flat_name_base)
master_flat, master_flat_name = get_best_flat(
flat_name=wrong_flat_name,
path=self.flat_path)
self.assertIsNone(master_flat)
self.assertIsNone(master_flat_name)
def test_normalize_master_flat(self):
methods = ['mean', 'simple', 'full']
for method in methods:
self.assertNotAlmostEqual(self.master_flat.data.mean(), 1.)
normalized_flat, normalized_flat_name = normalize_master_flat(
master=self.master_flat,
name=os.path.join(self.flat_path,
self.master_flat_name_norm),
method=method)
self.assertAlmostEqual(normalized_flat.data.mean(), 1.,
delta=0.001)
self.assertEqual(normalized_flat.header['GSP_NORM'], method)
self.assertIn('norm_', normalized_flat_name)
#reduce the arc frames
for filename in ic1.files_filtered(obstype='Arc', isiarm='Blue arm'):
hdu = fits.open(ic1.location + filename)
ccd = CCDData(hdu[1].data,
header=hdu[0].header + hdu[1].header,
unit=u.adu)
#this has to be fixed as the bias section does not include the whole section that will be trimmed
ccd = ccdproc.subtract_overscan(ccd,
median=True,
overscan_axis=0,
fits_section='[1:966,4105:4190]')
ccd = ccdproc.trim_image(ccd, fits_section=ccd.header['TRIMSEC'])
ccd = ccdproc.subtract_bias(ccd, master_bias_blue)
ccd = ccdproc.flat_correct(ccd, master_flat_blue)
ccd.data = ccd.data.T
ccd.write('arc_' + filename, clobber=True)
red_flat_list = []
for filename in ic1.files_filtered(obstype='Arc', isiarm='Red arm'):
hdu = fits.open(ic1.location + filename)
ccd = CCDData(hdu[1].data,
header=hdu[0].header + hdu[1].header,
unit=u.adu)
#this has to be fixed as the bias section does not include the whole section that will be trimmed
ccd = ccdproc.subtract_overscan(ccd,
median=True,
overscan_axis=0,
fits_section='[1:966,4105:4190]')
ccd = ccdproc.trim_image(ccd, fits_section=ccd.header['TRIMSEC'])
ccd = ccdproc.subtract_bias(ccd, master_bias_red)
ccd = ccdproc.flat_correct(ccd, master_flat_red)
def create_fake_spectroscopic_data(self):
if os.path.isdir(self.raw_path):
card_values = [{
'obstype': 'OBJECT',
'object': 'NGC2070',
'obsra': '16:23:24.285',
'obsdec': '-39:12:53.954'
}, {
'obstype': 'OBJECT',
'object': 'NGC2070',
'obsra': '16:24:34.285',
'obsdec': '-39:13:53.954'
}, {
'obstype': 'OBJECT',
'object': 'NGC2070',
'obsra': '16:24:34.285',
'obsdec': '-39:13:53.954'
}, {
'obstype': 'OBJECT',
'object': 'NGC2070',
'obsra': '16:24:34.285',
'obsdec': '-39:13:53.954'
}, {
'obstype': 'OBJECT',
'object': 'NGC2070',
'obsra': '16:25:34.285',
'obsdec': '-39:13:53.954'
}, {
'obstype': 'OBJECT',
'object': 'NGC2070',
'obsra': '16:25:34.285',
'obsdec': '-39:13:53.954'
}, {
'obstype': 'OBJECT',
'object': 'NGC2070',
'obsra': '16:25:34.285',
'obsdec': '-39:13:53.954'
}, {
'obstype': 'OBJECT',
'object': 'NGC2070',
'obsra': '16:26:34.285',
'obsdec': '-39:13:53.954'
}, {
'obstype': 'OBJECT',
'object': 'NGC2070',
'obsra': '16:27:34.285',
'obsdec': '-39:13:53.954'
}, {
'obstype': 'OBJECT',
'object': 'NGC2070',
'obsra': '16:28:34.285',
'obsdec': '-39:13:53.954'
}, {
'obstype': 'OBJECT',
'object': 'NGC2070',
'obsra': '16:23:34.285',
'obsdec': ' 39:13:53.954'
}, {
'obstype': 'OBJECT',
'object': 'NGC2070',
'obsra': '16:23:34.285',
'obsdec': ' 39:13:53.954'
}, {
'obstype': 'OBJECT',
'object': 'NGC2070',
'obsra': '16:23:34.285',
'obsdec': ' 39:13:53.954'
}, {
'obstype': 'OBJECT',
'object': 'NGC2070',
'obsra': '16:23:34.285',
'obsdec': ' 39:13:53.954'
}, {
'obstype': 'OBJECT',
'object': 'NGC2070',
'obsra': '16:23:34.285',
'obsdec': ' 39:13:53.954'
}, {
'obstype': 'COMP',
'object': 'HgArNe',
'obsra': '16:22:34.285',
'obsdec': '-39:23:53.954'
}, {
'obstype': 'COMP',
'object': 'HgArNe',
'obsra': '16:24:34.285',
'obsdec': '-39:13:53.954'
}, {
'obstype': 'COMP',
'object': 'HgArNe',
'obsra': '16:24:34.285',
'obsdec': '-39:13:53.954'
}, {
'obstype': 'COMP',
'object': 'HgArNe',
'obsra': '16:24:34.285',
'obsdec': '-39:13:53.954'
}, {
'obstype': 'COMP',
'object': 'HgArNe',
'obsra': '16:25:34.285',
'obsdec': '-39:13:53.954'
}, {
'obstype': 'COMP',
'object': 'HgArNe',
'obsra': '16:25:34.285',
'obsdec': '-39:13:53.954'
}, {
'obstype': 'COMP',
'object': 'HgArNe',
'obsra': '16:25:34.285',
'obsdec': '-39:13:53.954'
}, {
'obstype': 'COMP',
'object': 'HgArNe',
'obsra': '16:26:34.285',
'obsdec': '-39:13:53.954'
}, {
'obstype': 'COMP',
'object': 'HgArNe',
'obsra': '16:27:34.285',
'obsdec': '-39:13:53.954'
}, {
'obstype': 'COMP',
'object': 'HgArNe',
'obsra': '16:28:34.285',
'obsdec': '-39:13:53.954'
}, {
'obstype': 'COMP',
'object': 'HgArNe',
'obsra': '16:23:34.285',
'obsdec': '39:13:53.954'
}, {
'obstype': 'COMP',
'object': 'HgArNe',
'obsra': '16:23:34.285',
'obsdec': '39:13:53.954'
}, {
'obstype': 'COMP',
'object': 'HgArNe',
'obsra': '16:23:34.285',
'obsdec': '39:13:53.954'
}, {
'obstype': 'COMP',
'object': 'HgArNe',
'obsra': '16:23:34.285',
'obsdec': '39:13:53.954'
}, {
'obstype': 'COMP',
'object': 'HgArNe',
'obsra': '16:23:34.285',
'obsdec': '39:13:53.954'
}]
for i in range(len(card_values)):
ccd = CCDData(data=np.ones((3, 3)),
meta=fits.Header(),
unit='adu')
ccd.header.set('DATE', value='2019-03-22', comment='nc')
ccd.header.set('SLIT', value='1.0" long slit', comment='nc')
ccd.header.set('DATE-OBS',
value='2019-03-22T09:59:33.654',
comment='nc')
ccd.header.set('OBSTYPE',
value=card_values[i]['obstype'],
comment='nc')
ccd.header.set('OBJECT',
value=card_values[i]['object'],
comment='nc')
ccd.header.set('EXPTIME', value='10', comment='nc')
ccd.header.set('OBSRA',
value=card_values[i]['obsra'],
comment='nc')
ccd.header.set('OBSDEC',
value=card_values[i]['obsdec'],
comment='nc')
ccd.header.set('GRATING', value='SYZY_400', comment='nc')
ccd.header.set('CAM_TARG', value='16.1', comment='nc')
ccd.header.set('GRT_TARG', value='7.5', comment='nc')
ccd.header.set('FILTER', value='<NO FILTER>', comment='nc')
ccd.header.set('FILTER2', value='GG455', comment='nc')
ccd.header.set('GAIN', value='1.48', comment='nc')
ccd.header.set('RDNOISE', value='3.89', comment='nc')
ccd.header.set('INSTCONF', value='Red', comment='nc')
ccd.header.set('WAVMODE', value='Spectroscopy', comment='nc')
ccd.write(
os.path.join(self.raw_path,
'test_file_{:03d}.fits'.format(i)))
class FitsFileIOAndOps(TestCase):
def setUp(self):
self.fake_image = CCDData(data=np.ones((100, 100)),
meta=fits.Header(),
unit='adu')
self.file_name = 'sample_file.fits'
self.target_non_zero = 4
self.current_directory = os.getcwd()
self.full_path = os.path.join(self.current_directory, self.file_name)
self.parent_file = 'parent_file.fits'
self.fake_image.header.set('CCDSUM',
value='1 1',
comment='Fake values')
self.fake_image.header.set('OBSTYPE',
value='OBJECT',
comment='Fake values')
self.fake_image.header.set('GSP_FNAM',
value=self.file_name,
comment='Fake values')
self.fake_image.header.set('GSP_PNAM',
value=self.parent_file,
comment='Fake values')
self.fake_image.write(self.full_path, overwrite=False)
def test_write_fits(self):
self.assertTrue(os.path.isfile(self.full_path))
os.remove(self.full_path)
write_fits(ccd=self.fake_image,
full_path=self.full_path,
parent_file=self.parent_file,
overwrite=False)
self.assertTrue(os.path.isfile(self.full_path))
def test_read_fits(self):
self.recovered_fake_image = read_fits(self.full_path)
self.assertIsInstance(self.recovered_fake_image, CCDData)
def test_image_overscan(self):
data_value = 100.
overscan_value = 0.1
# alter overscan region to a lower number
self.fake_image.data *= data_value
self.fake_image.data[:, 0:5] = overscan_value
overscan_region = '[1:6,:]'
self.assertEqual(self.fake_image.data[:, 6:99].mean(), data_value)
self.assertEqual(self.fake_image.data[:, 0:5].mean(), overscan_value)
self.fake_image = image_overscan(ccd=self.fake_image,
overscan_region=overscan_region)
self.assertEqual(self.fake_image.data[:, 6:99].mean(),
data_value - overscan_value)
self.assertEqual(self.fake_image.header['GSP_OVER'], overscan_region)
def test_image_overscan_none(self):
new_fake_image = image_overscan(ccd=self.fake_image,
overscan_region=None)
self.assertEqual(new_fake_image, self.fake_image)
def test_image_trim(self):
self.assertEqual(self.fake_image.data.shape, (100, 100))
trim_section = '[1:50,:]'
self.fake_image = image_trim(ccd=self.fake_image,
trim_section=trim_section,
trim_type='trimsec')
self.assertEqual(self.fake_image.data.shape, (100, 50))
self.assertEqual(self.fake_image.header['GSP_TRIM'], trim_section)
def test_save_extracted_target_zero(self):
self.fake_image.header.set('GSP_FNAM', value=self.file_name)
same_fake_image = save_extracted(ccd=self.fake_image,
destination=self.current_directory,
prefix='e',
target_number=0)
self.assertEqual(same_fake_image, self.fake_image)
self.assertTrue(os.path.isfile('e' + self.file_name))
def test_save_extracted_target_non_zero(self):
self.fake_image.header.set('GSP_FNAM', value=self.file_name)
same_fake_image = save_extracted(ccd=self.fake_image,
destination=self.current_directory,
prefix='e',
target_number=self.target_non_zero)
self.assertEqual(same_fake_image, self.fake_image)
self.assertTrue(
os.path.isfile(
'e' + re.sub('.fits', '_target_{:d}.fits'.format(
self.target_non_zero), self.file_name)))
def test_save_extracted_target_zero_comp(self):
self.fake_image.header.set('GSP_FNAM', value=self.file_name)
self.fake_image.header.set('OBSTYPE', value='COMP')
self.fake_image.header.set('GSP_EXTR', value='100.00:101.00')
same_fake_image = save_extracted(ccd=self.fake_image,
destination=self.current_directory,
prefix='e',
target_number=0)
self.assertEqual(same_fake_image, self.fake_image)
self.assertTrue(os.path.isfile(self.fake_image.header['GSP_FNAM']))
def tearDown(self):
files_to_remove = [self.full_path, self.fake_image.header['GSP_FNAM']]
for _file in files_to_remove:
if os.path.isfile(_file):
os.unlink(_file)
class MasterFlatTest(TestCase):
def setUp(self):
# create a master flat
self.master_flat = CCDData(data=np.ones((100, 100)),
meta=fits.Header(),
unit='adu')
self.master_flat.header.set('GRATING', value='RALC_1200-BLUE')
self.master_flat.header.set('SLIT', value='0.84" long slit')
self.master_flat.header.set('FILTER2', value='<NO FILTER>')
self.master_flat.header.set('WAVMODE', value='1200 m2')
self.master_flat_name = 'master_flat_1200m2.fits'
# expected master flat to be retrieved by get_best_flat
self.reference_flat_name = 'master_flat_1200m2_0.84_dome.fits'
# location of sample flats
self.flat_path = 'goodman_pipeline/data/test_data/master_flat'
slit = re.sub('[A-Za-z" ]', '', self.master_flat.header['SLIT'])
self.flat_name_base = re.sub('.fits', '_' + slit + '*.fits',
self.master_flat_name)
# save a master flat with some random structure.
self.master_flat_name_norm = 'flat_to_normalize.fits'
# add a bias level
self.master_flat.data += 300.
# add noise
self.master_flat.data += np.random.random_sample(
self.master_flat.data.shape)
self.master_flat.write(os.path.join(self.flat_path,
self.master_flat_name_norm),
overwrite=False)
def tearDown(self):
full_path = os.path.join(self.flat_path, self.master_flat_name_norm)
self.assertTrue(os.path.isfile(full_path))
if os.path.isfile(full_path):
os.unlink(full_path)
self.assertFalse(os.path.isfile(full_path))
# remove normalized flat
norm_flat = re.sub('flat_to_', 'norm_flat_to_', full_path)
if os.path.isfile(norm_flat):
os.unlink(norm_flat)
self.assertFalse(os.path.isfile(norm_flat))
def test_get_best_flat(self):
# print(self.flat_name_base)
master_flat, master_flat_name = get_best_flat(
flat_name=self.flat_name_base, path=self.flat_path)
self.assertIsInstance(master_flat, CCDData)
self.assertEqual(os.path.basename(master_flat_name),
self.reference_flat_name)
def test_get_best_flat_fail(self):
# Introduce an error that will never produce a result.
wrong_flat_name = re.sub('1200m2', '1300m2', self.flat_name_base)
master_flat, master_flat_name = get_best_flat(
flat_name=wrong_flat_name, path=self.flat_path)
self.assertIsNone(master_flat)
self.assertIsNone(master_flat_name)
def test_normalize_master_flat(self):
methods = ['mean', 'simple', 'full']
for method in methods:
self.assertNotAlmostEqual(self.master_flat.data.mean(), 1.)
normalized_flat, normalized_flat_name = normalize_master_flat(
master=self.master_flat,
name=os.path.join(self.flat_path, self.master_flat_name_norm),
method=method)
self.assertAlmostEqual(normalized_flat.data.mean(),
1.,
delta=0.001)
self.assertEqual(normalized_flat.header['GSP_NORM'], method)
self.assertIn('norm_', normalized_flat_name)
ccd = ccdproc.subtract_bias(ccd, master_bias_blue)
blue_flat_list.append(ccd)
master_flat_blue = ccdproc.combine(blue_flat_list, method='median')
master_flat_blue.write('master_flat_blue.fits', clobber=True)
#reduce the arc frames
for filename in ic1.files_filtered(obstype='Arc', isiarm='Blue arm'):
hdu = fits.open(ic1.location + filename)
ccd = CCDData(hdu[1].data, header=hdu[0].header+hdu[1].header, unit = u.adu)
#this has to be fixed as the bias section does not include the whole section that will be trimmed
ccd = ccdproc.subtract_overscan(ccd, median=True, overscan_axis=0, fits_section='[1:966,4105:4190]')
ccd = ccdproc.trim_image(ccd, fits_section=ccd.header['TRIMSEC'] )
ccd = ccdproc.subtract_bias(ccd, master_bias_blue)
ccd = ccdproc.flat_correct(ccd, master_flat_blue)
ccd.data = ccd.data.T
ccd.write('arc_'+filename, clobber=True)
red_flat_list = []
for filename in ic1.files_filtered(obstype='Arc', isiarm='Red arm'):
hdu = fits.open(ic1.location + filename)
ccd = CCDData(hdu[1].data, header=hdu[0].header+hdu[1].header, unit = u.adu)
#this has to be fixed as the bias section does not include the whole section that will be trimmed
ccd = ccdproc.subtract_overscan(ccd, median=True, overscan_axis=0, fits_section='[1:966,4105:4190]')
ccd = ccdproc.trim_image(ccd, fits_section=ccd.header['TRIMSEC'] )
ccd = ccdproc.subtract_bias(ccd, master_bias_red)
ccd = ccdproc.flat_correct(ccd, master_flat_red)
ccd.data = ccd.data.T
ccd.write('arc_'+filename, clobber=True)
#reduce the sky frames
for filename in ic1.files_filtered(obstype='Sky', isiarm='Blue arm'):