def makeMasterFlat(images, master_bias):
"""
Flats are corrected for their bias level (if
master_bias)
TODO: Finish docstring
"""
try:
fitsfile = 'master_flat.fits'
master_flat = CCDData.read(fitsfile, unit=u.adu)
return master_flat
except FileNotFoundError:
# empty list for the flats
flat_list = []
# create the master flat field
print('Reducing flats')
for f in images.files_filtered(imagetyp=FLAT_KEYWORD):
print(f)
with fits.open(f) as fitsfile:
data_exp = fitsfile[0].header[EXPTIME_KEYWORD]
ccd = CCDData.read(f, unit=u.adu)
if master_bias:
ccd = subtract_bias(ccd, master_bias)
else:
print('No master bias, skipping correction...')
flat_list.append(ccd)
try:
master_flat = combine(flat_list, method='median')
master_flat.write('master_flat.fits', clobber=True)
return master_flat
except IndexError:
print('There are no flats, skipping...')
master_flat = None
def combinedark(cal_dir="../Data/20181207/cals", mast_dir=".", filt=[]):
#Generates master dark file from calibration directory - uses max exptime dark images
#Get master bias file
if os.path.isfile(mast_dir + "/master/master_bias.FIT") != True:
print "No master bias file"
return False
master_bias = CCDData.read(mast_dir + "/master/master_bias.FIT")
#Generate image list
imagelist = gen.getimages(cal_dir, filt=filt)
full_dark_list = []
for img in imagelist:
ccd = CCDData.read(cal_dir + '/' + img, unit=u.adu)
if ccd.header["IMAGETYP"].strip() == "Dark Frame":
img_exptime = ccd.header["EXPTIME"]
ccd = ccdproc.subtract_bias(ccd, master_bias)
full_dark_list.append([ccd, img_exptime])
print img
#Select only those with max exptime
exptime = max(full_dark_list, key=lambda x: x[1])[1]
dark_list = [x[0] for x in full_dark_list if x[1] == exptime]
if len(dark_list) == 0:
print "ERROR: no dark files"
return False
#Generate master file
master_dark = ccdproc.combine(dark_list, method='median', dtype="float32")
master_dark.write(mast_dir + "/master/master_dark.FIT", overwrite=True)
'''
print "Created master_dark file with " + str(exptime) + " exptime"
'''
return True
def test___call___method_two_solution(self):
file_path_1 = os.path.join(os.getcwd(),
'goodman_pipeline/data/ref_comp',
'goodman_comp_1200M2_FeHeAr.fits')
file_path_2 = os.path.join(os.getcwd(),
'goodman_pipeline/data/ref_comp',
'goodman_comp_1200M2_CuHeAr.fits')
ref_lamp_1 = CCDData.read(file_path_1, unit='adu')
ref_lamp_2 = CCDData.read(file_path_2, unit='adu')
lamp_ccd_1 = write_fits(
ref_lamp_1,
os.path.join(os.getcwd(), 'test_comparison_lamp_1.fits'),
)
lamp_ccd_2 = write_fits(
ref_lamp_2,
os.path.join(os.getcwd(), 'test_comparison_lamp_2.fits'),
)
self.file_list.append('test_comparison_lamp_1.fits')
self.file_list.append('test_comparison_lamp_2.fits')
json_output = self.wc(ccd=self.ccd,
comp_list=[lamp_ccd_1, lamp_ccd_2],
save_data_to='',
reference_data='goodman_pipeline/data/ref_comp',
json_output=True)
# print(json.dumps(json_output, indent=4))
self.assertEqual(len(json_output['wavelength_solution']), 2)
for _solution in json_output['wavelength_solution']:
self.file_list.append(_solution['file_name'])
self.file_list.append(_solution['reference_lamp'])
def load_image(filename):
try:
data = CCDData.read(filename)
except(ValueError):
try:
data = CCDData.read(filename, unit = u.dyn)
except:
raise
return data
def load_image(filename):
try:
data = CCDData.read(filename)
except (ValueError):
try:
data = CCDData.read(filename, unit=u.dyn)
except:
raise
return data
def reduce_image(imagefile, dark=None, flat=None):
im = CCDData.read(imagefile, unit='adu')
if dark is not None:
dark = CCDData.read(dark, unit='adu')
im = im.subtract(dark)
if flat is not None:
# masterflat = CCDData.read(flat, unit='adu')
hdul = fits.open(flat)
masterflat = CCDData(data=hdul[0].data, uncertainty=None, meta=hdul[0].header, unit='adu')
im = flat_correct(im, masterflat)
return im
def combineflat(cal_dir="../Data/20181207/cals", mast_dir=".", filt=[], binning=2):
#Generates master flat file from calibration directory
#Get master bias and dark files
if os.path.isfile(mast_dir + "/master/master_bias.FIT") != True:
print "No master bias file"
return False
if os.path.isfile(mast_dir + "/master/master_dark.FIT") != True:
print "No master dark file"
return False
master_bias = CCDData.read(mast_dir + "/master/master_bias.FIT")
master_dark = CCDData.read(mast_dir + "/master/master_dark.FIT")
#Generate image list
imagelist = gen.getimages(cal_dir, filt=filt)
flat_list = []
for img in imagelist:
ccd = CCDData.read(cal_dir + '/' + img, unit=u.adu)
if ccd.header["IMAGETYP"].strip() == "FLAT":
#Rebin images if needed
if ccd.header["XBINNING"] > binning:
print "ERROR: Binning too low"
return False
elif ccd.header["XBINNING"] < binning:
ccd.data = gen.rebin(ccd.data, oldbin=ccd.header["XBINNING"], newbin=binning)
ccd.header["XBINNING"] = binning
ccd.header["YBINNING"] = binning
#Remove bias and dark effects
ccd = ccdproc.subtract_bias(ccd, master_bias)
ccd = ccdproc.subtract_dark(ccd, master_dark, \
dark_exposure=master_dark.header["EXPTIME"]*u.s, \
data_exposure=ccd.header["EXPTIME"]*u.s, scale=True)
ccd.data = ccd.data/np.median(ccd.data)
flat_list.append(ccd)
if len(flat_list) == 0:
print "ERROR: no flat files"
return False
#Generate master file
master_flat = ccdproc.combine(flat_list, method='median', dtype="float32")
master_flat.write(mast_dir + "/master/master_flat_" + ".FIT", \
overwrite=True)
'''
print "Created master_flat file for " + filter_name + " filter"
'''
return True
def reproject(filename,ref_header,skylist,explist):
#Open the image.
hdulist = fits.open(filename)
datacube = hdulist[0].data
header = hdulist[0].header
#Create a list with zeros for the reprojected datacube.
repro_datacube = [0] * len(datacube)
#Loop over the different frames in the datacube.
for i,data in enumerate(datacube):
#Create a CCDData class object.
data_ccd = CCDData(data,header=header,unit="count",wcs=wcs.WCS(header).celestial)
#Reproject the data to the reference data.
repro_datacube[i] = np.asarray(wcs_project(data_ccd,wcs.WCS(ref_header).celestial,target_shape=(ref_header['NAXIS2'],ref_header['NAXIS1'])))
#Temporary workaround to update the header of the image.
new_data = wcs_project(data_ccd,wcs.WCS(ref_header).celestial,target_shape=(ref_header['NAXIS2'],ref_header['NAXIS1']))
new_data.write(filename.replace(".img","_r.img"),format="fits",overwrite=True)
temp_hdu = fits.open(filename.replace(".img","_r.img"))
new_header = temp_hdu[0].header
#Append the reprojected datacube to the list and write it to a new image.
skylist.append(np.array(repro_datacube))
new_hdu = fits.PrimaryHDU(repro_datacube,new_header)
new_hdu.writeto(filename.replace(".img","_r.img"),overwrite=True)
#Reproject the exposure map.
data_exp_ccd = CCDData.read(filename.replace("nm_coilsszp_c","ex"),unit="count",hdu=1)
repro_data_exp = wcs_project(data_exp_ccd,wcs.WCS(ref_header).celestial,target_shape=(ref_header['NAXIS2'],ref_header['NAXIS1']))
#Append the reprojected data to a list and write it to a new image.
explist.append(np.array(repro_data_exp))
repro_data_exp.write(filename.replace("nm_coilsszp_c","ex_r"),format="fits",overwrite=True)
def __call__(self, in_file, save=False):
self.file = in_file
self.fig, self.ax = plt.subplots()
# read data and get its wavelength solution
ccd = CCDData.read(self.file, unit=u.adu)
wcs_reader = ReadWavelengthSolution(header=ccd.header,
data=ccd.data)
wavelength, intensity = wcs_reader()
manager = plt.get_current_fig_manager()
manager.window.showMaximized()
plt.title('{:s}\n{:s}'.format(self.file, ccd.header['OBJECT']))
self.ax.plot(wavelength, intensity, color='k', label='Data')
self.ax.axvline(6562.8, color='r')
self.ax.set_xlim((wavelength[0], wavelength[-1]))
self.ax.set_ylabel('Intensity (ADU)')
self.ax.set_xlabel('Wavelength (Angstrom)')
plt.legend(loc='best')
plt.subplots_adjust(left=0.05,
right=0.99,
top=0.96,
bottom=0.04,
hspace=0.17,
wspace=0.11)
# plt.tight_layout()
if not save:
self.fig.canvas.mpl_connect('key_press_event', self.key_pressed)
plt.show()
def reduce(filename):
if os.path.isfile(filename):
ccd = CCDData.read(filename, unit='adu')
raw_data = {}
raw_data["file_name"] = os.path.basename(filename)
raw_data["date"] = ccd.header['DATE']
raw_data["ut_time"] = ccd.header['UT']
raw_data["instrument"] = ccd.header['INSTRUME']
raw_data["camera"] = ccd.header['INSTCONF']
raw_data["object"] = ccd.header['OBJECT']
raw_data["obstype"] = ccd.header['OBSTYPE']
raw_data["ra"] = ccd.header['RA']
raw_data["dec"] = ccd.header['DEC']
raw_data['airmass'] = ccd.header['AIRMASS']
raw_data['seeing'] = ccd.header['SEEING']
raw_data['filter'] = ccd.header['FILTER']
raw_data['filter2'] = ccd.header['FILTER2']
raw_data['grating'] = ccd.header['GRATING']
raw_data['slit'] = ccd.header['SLIT']
raw_data['wavmode'] = ccd.header['WAVMODE']
raw_data['exptime'] = ccd.header['EXPTIME']
json_data = json.dumps(raw_data)
post_header = {"Content-type": "application/json"}
r = requests.post('http://api:8080/api/files',
data=json_data,
headers=post_header)
print(r.status_code, r.reason)
print(json_data)
return "Reducing {}".format(filename)
def __call__(self, in_file, save=False):
self.file = in_file
self.fig, self.ax = plt.subplots()
# read data and get its wavelength solution
ccd = CCDData.read(self.file, unit=u.adu)
wcs_reader = ReadWavelengthSolution(header=ccd.header, data=ccd.data)
wavelength, intensity = wcs_reader()
manager = plt.get_current_fig_manager()
manager.window.showMaximized()
plt.title('{:s}\n{:s}'.format(self.file, ccd.header['OBJECT']))
self.ax.plot(wavelength, intensity, color='k', label='Data')
self.ax.axvline(6562.8, color='r')
self.ax.set_xlim((wavelength[0], wavelength[-1]))
self.ax.set_ylabel('Intensity (ADU)')
self.ax.set_xlabel('Wavelength (Angstrom)')
plt.legend(loc='best')
plt.subplots_adjust(left=0.05,
right=0.99,
top=0.96,
bottom=0.04,
hspace=0.17,
wspace=0.11)
# plt.tight_layout()
if not save:
self.fig.canvas.mpl_connect('key_press_event', self.key_pressed)
plt.show()
def __init__(self, filename,
ccd_gain, ccd_readnoise, # ccd properties
oscan_idx, oscan_size, # overscan region
plot_path=None, zscaler=None, cmap=None, # for plotting
**read_kwargs):
# read CCD frame
self.ccd = CCDData.read(filename, **read_kwargs)
self.filename = filename
self._filename_base = path.splitext(path.basename(self.filename))[0]
self._obj_name = self.ccd.header['OBJECT']
# CCD properties
self.ccd_gain = ccd_gain
self.ccd_readnoise = ccd_readnoise
self.oscan_idx = oscan_idx
self.oscan_size = oscan_size
# Plot settings
self.plot_path = plot_path
if zscaler is None:
zscaler = ZScaleInterval(32768, krej=5., max_iterations=16)
self.zscaler = zscaler
if cmap is None:
cmap = 'Greys_r'
self.cmap = cmap
def photometry(path, basename, db, config, logger):
"""Master Photometry Program"""
imf = os.sep.join((path, basename)) + '.fits'
bgf = imf.replace('.fits', '.bg.fits')
catf = imf.replace('.fits', '.cat.fits')
# Background check
row = db.execute(
'''
SELECT count() FROM obs
WHERE bg IS NOT null AND filename=?
''', [basename]).fetchone()
count = row[0]
run_bg = any((count == 0, config.reprocess_bg, not os.path.exists(bgf)))
# Photometry check
run_phot = any((run_bg, not os.path.exists(catf), config.reprocess_phot))
run_cal = any((run_phot, config.reprocess_cal))
if not any((run_bg, run_phot, run_cal)):
return
logger.info(basename)
ccd = border_mask(CCDData.read(imf), config)
ccd.mask[np.isnan(ccd.data)] = True
if run_bg:
background.background(ccd, bgf, db, config, logger)
if run_phot:
catalog(ccd, bgf, catf, db, config, logger)
if run_cal:
calibrate(ccd, basename, imf, catf, db, config, logger)
def main():
parser = argparse.ArgumentParser(description='Perform LACosmic cleaning of images')
parser.add_argument('filenames',nargs='+',help='List of files to clean.')
parser.add_argument('-odir',metavar='outdir',required=True,type=str,help='Output directory for files.')
#parser.add_argument('-mode',choices=['lacosmic','median'],default='lacosmic',help='Specify mode of operation (default=lacosmic)')
parser.add_argument('-sclip',metavar='sigclip',type=float,default=5,help='Laplacian-to-noise limit for cosmic ray detection. Lower values will flag more pixels as cosmic rays (default=5).')
parser.add_argument('-sfrac',metavar='sigfrac',type=float,default=0.3,help='Fractional detection limit for neighboring pixels. For cosmic ray neighbor pixels, a Laplacian-to-noise detection limit of sigfrac * sigclip will be used. (default=0.3).')
parser.add_argument('-objlim',type=float,default=5,help='Minimum contrast between Laplacian image and the fine structure image. Increase this value if cores of bright stars are flagged as cosmic rays (default=5).')
parser.add_argument('-satlevel',type=float,default=65535,help='Saturation level of the image (electrons). This value is used to detect saturated stars and pixels at or above this level are added to the mask (default=65535)')
parser.add_argument('-niter',type=int,default=5,help='umber of iterations of the LA Cosmic algorithm to perform (default=5).')
#parser.add_argument('-thresh',metavar='threshold',type=float,default=5,help='Threshold for detecting cosmic rays [median] (default=5).')
#parser.add_argument('-mbox',type=float,default=11,help='Median box for detecting cosmic rays [mbox] (default=11).')
parser.add_argument('-njobs',type=int,default=1,help='Process images in parallel. "-1" is all CPUs (default=1).')
parser.add_argument('--c',action='store_true',help='Clobber (overwrite) on output')
args = parser.parse_args()
ccds = (CCDData.read(fname,unit='adu') for fname in args.filenames)
with Parallel(args.njobs,verbose=11) as parallel:
cleaned = parallel(delayed(cosmicray_lacosmic)(ccd,sigclip=args.sclip,sigfrac=args.sfrac,niter=args.niter,objlim=args.objlim,satlevel=args.satlevel) for ccd in ccds)
outfiles = (os.path.join(args.odir,os.path.basename(fname)) for fname in args.filenames)
for hdu,outfile in zip(cleaned,outfiles):
if isinstance(hdu,CCDData):
hdu = hdu.to_hdu(hdu_mask=None,hdu_uncertainty=None)
header = hdu[0].header
header.add_history('clean.py - %s' % Time(Time.now(),format='fits'))
header['CLEANED'] = (True,'Cleaned with LACosmics')
header['CLNMTHD'] = (CLNMTHD,'Method used to clean')
try:
hdu.writeto(outfile,overwrite=args.c)
except OSError as e:
raise OSError("File '%s' already exists. Re-run with --c flag to overwrite existing files." % outfile) from e
def __call__(self, *args, **kwargs):
for file_name in self.file_list:
ccd = CCDData.read(file_name, unit='adu')
if self.args.style == 'light':
plt.style.use('default')
elif self.args.style == 'dark':
plt.style.use('dark_background')
else:
plt.style.use('dark_background')
fig, ax = plt.subplots(figsize=(16, 9))
fig.canvas.set_window_title(file_name)
ax.set_title(file_name)
if ccd.header['NAXIS'] == 2:
zlow, zhigh = self.scale.get_limits(ccd.data)
im = ax.imshow(ccd.data,
cmap=self.args.cmap,
clim=(zlow, zhigh))
divider = make_axes_locatable(ax)
cax = divider.append_axes('right', size="3%", pad=0.05)
fig.colorbar(im, cax=cax)
elif ccd.header['NAXIS'] == 1:
wav, intens = self.wcs.read(ccd=ccd)
ax.plot(wav, intens)
ax.set_ylabel('Intensity')
ax.set_xlabel('Wavelength')
plt.tight_layout()
plt.show()
def ccdproc_images_filter(list_files, image_filter=None, master_flat=None, master_bias=None, fits_section=None, gain=None, readnoise=None,
error=False, sky=True, dout=None, cosmic=False, mbox=15, rbox=15, gbox=11, cleantype="medmask", cosmic_method='lacosmic',
sigclip=5, key_filter='filter', dfilter={'imagetyp':'LIGHT'}, mask=None, key_find='find', invert_find=False, **kwargs):
if error and (gain is None or readnoise is None):
print ('WARNING: You need to provide "gain" and "readnoise" to compute the error!')
return
if gain is not None and not isinstance(gain, u.Quantity):
gain = gain * u.electron / u.adu
if readnoise is not None and not isinstance(readnoise, u.Quantity):
readnoise = readnoise * u.electron
if dfilter is not None and key_filter is not None and image_filter is not None:
dfilter = addKeysListDict(dfilter, {key_filter: image_filter})
list_files = getListFiles(list_files, dfilter, mask, key_find=key_find, invert_find=invert_find)
dccd = {}
for filename in list_files:
ccd = CCDData.read(filename, unit= u.adu)
nccd = ccdproc.ccd_process(ccd, trim=fits_section, gain=gain, master_bias=master_bias, master_flat=master_flat, readnoise=readnoise, error=error)
for key in ccd.header:
if not key in nccd.header:
nccd.header[key] = ccd.header[key]
# Better get rid of the cosmic rays BEFORE subtracting the global sky background
if cosmic:
nccd = cleanCosmic(nccd, mbox=mbox, rbox=rbox, gbox=gbox, sigclip=sigclip, cleantype=cleantype, cosmic_method=cosmic_method)
if sky:
nccd = subtract_sky_ccd(nccd, **kwargs)
addKeyHdr(nccd.header, 'MBIAS', getFilename(master_bias))
addKeyHdr(nccd.header, 'MFLAT', getFilename(master_flat))
filename = 'c%s' % os.path.basename(filename)
dccd[filename] = nccd
filename = join_path(filename, dout)
nccd.header['FILENAME'] = os.path.basename(filename)
nccd.header['CCDVER'] = VERSION
nccd.header = ammendHeader(nccd.header)
nccd.write(filename, clobber=True)
return dccd
def create_master_bias(list_files, fitsfile=None, fits_section=None, gain=None, method='median',
dfilter={'imagetyp':'bias'}, mask=None, key_find='find', invert_find=False, sjoin=','):
if gain is not None and not isinstance(gain, u.Quantity):
gain = gain * u.electron / u.adu
lbias = []
list_files = getListFiles(list_files, dfilter, mask, key_find=key_find, invert_find=invert_find)
for filename in list_files:
ccd = CCDData.read(filename, unit= u.adu)
trimmed = True if fits_section is not None else False
ccd = ccdproc.trim_image(ccd, fits_section=fits_section, add_keyword={'trimmed': trimmed})
if gain is not None:
ccd = ccdproc.gain_correct(ccd, gain)
lbias.append(ccd)
combine = ccdproc.combine(lbias, method=method)
if gain is not None and not 'GAIN' in combine.header:
combine.header.set('GAIN', gain.value, gain.unit)
combine.header['CGAIN'] = True if gain is not None else False
combine.header['IMAGETYP'] = 'BIAS'
combine.header['CMETHOD'] = method
combine.header['CCDVER'] = VERSION
if sjoin is not None:
combine.header['LBIAS'] = sjoin.join([os.path.basename(fits) for fits in list_files])
combine.header['NBIAS'] = len(list_files)
if fitsfile is not None:
combine.header['FILENAME'] = os.path.basename(fitsfile)
combine.write(fitsfile, clobber=True)
return combine
def fits2CCDData(lfits, key_unit='BUNIT', key_file='FILENAME', unit=None, single=False):
lccd = []
if not isinstance(lfits, (tuple, list)):
lfits = [lfits]
for fits in lfits:
fits_unit = unit
if os.path.exists(fits):
hdr = pyfits.getheader(fits)
else:
print ('>>> WARNING: File "%s" NOT found' % os.path.basename(fits))
continue
if key_unit is not None and key_unit in hdr:
try:
fits_unit = eval('u.%s' % hdr[key_unit])
except:
pass
if fits_unit is None:
if key_unit is not None:
sys.exit('>>> Units NOT found in header ("%s") of image "%s". Specify one in "unit" variable' % (key_unit, os.path.basename(fits)))
else:
print ('>>> WARNING: "key_unit" not specified')
ccd = CCDData.read(fits, unit=fits_unit)
if key_file is not None and not key_file in ccd.header:
ccd.header[key_file] = os.path.basename(fits)
lccd.append(ccd)
if len(lccd) == 0:
print ('>>> WARNING: NO files found!')
return
if single and len(lccd) == 1:
lccd = lccd[0]
return lccd
def reduceframes(img_dir="../Data/20181207/imgs", mast_dir=".", mast_cal_dir=False):
#Removes effects from bias, dark, and flat master files and makes calibrated images
#Get master bias and dark frames - get flat later once have filter_name
if not mast_cal_dir:
mast_cal_dir = mast_dir
if os.path.isfile(mast_cal_dir + "/master/master_bias.FIT") != True:
print "No master bias file"
return False
if os.path.isfile(mast_cal_dir + "/master/master_dark.FIT") != True:
print "No master dark file"
return False
master_bias = CCDData.read(mast_cal_dir + "/master/master_bias.FIT")
master_dark = CCDData.read(mast_cal_dir + "/master/master_dark.FIT")
# if not os.path.exists(mast_dir + "/frames"):
# makedirs(mast_dir + "/frames")
#Reduce images
raw_image_names = gen.getimages(img_dir, filt=[])
for img in raw_image_names:
print img
ccd = CCDData.read(img_dir + '/' + img, unit=u.adu)
#Mask saturated pixels - not doing any more
'''mask_saturated = (ccd.data > 50000)
.data = np.array(ccd.data, dtype=np.float32)
.data[mask_saturated] = np.nan'''
ccd = ccdproc.subtract_bias(ccd, master_bias)
ccd = ccdproc.subtract_dark(ccd, master_dark, \
dark_exposure=master_dark.header["EXPTIME"]*u.s, \
data_exposure=ccd.header["EXPTIME"]*u.s, scale=True)
mean, background, std = sigma_clipped_stats(ccd.data, sigma=3.0, iters=5)
ccd.data = ccd.data - background
ccd.data = ccd.data/ccd.header["EXPTIME"]
ccd.unit = u.adu/u.s
#Add info about background and raw image name to header
ccd.header['SKY'] = background
ccd.header['RAWFILE'] = img
#Save calibrated frame
ccd.write(mast_dir + '/frames/' + img[:-4] + '-calibrated.FIT' , overwrite=True)
print "Created all calibrated frames in " + mast_dir + '/frames'
return True
def test_fit_linear(self):
test_file = os.path.join(self.data_path,
'goodman_comp_400M1_HgArNe.fits')
ccd = CCDData.read(test_file, unit='adu')
pixel, angstrom = self._recover_lines(ccd=ccd)
model = self.wcs.fit(physical=pixel,
wavelength=angstrom,
model_name='linear')
self.assertIsInstance(model, Model)
def test_fit_linear(self):
test_file = os.path.join(self.data_path,
'goodman_comp_400M1_HgArNe.fits')
ccd = CCDData.read(test_file, unit='adu')
pixel, angstrom = self._recover_lines(ccd=ccd)
model = self.wcs.fit(physical=pixel,
wavelength=angstrom,
model_name='linear')
self.assertIsInstance(model, Model)
def test_read__non_linear_legendre(self):
test_file = os.path.join(self.data_path,
'non-linear_fits_solution_legendre.fits')
self.assertTrue(os.path.isfile(test_file))
ccd = CCDData.read(test_file, unit='adu')
result = self.wcs.read(ccd=ccd)
self.assertIsInstance(self.wcs.model, Model)
self.assertEqual(self.wcs.model.__class__.__name__, 'Legendre1D')
def test_data_classifier_mixed_instconf(self):
sample_file = os.listdir(self.raw_path)[0]
raw_path_full = os.path.join(self.raw_path, sample_file)
recovered_ccd = CCDData.read(raw_path_full, unit='adu')
recovered_ccd.header['INSTCONF'] = 'Blue'
# recovered_ccd.header['WAVMODE'] = 'Imaging'
recovered_ccd.write(raw_path_full, overwrite=True)
with self.assertRaises(SystemExit):
self.data_classifier(raw_path=self.raw_path)
def test_read__non_linear_cspline(self):
test_file = os.path.join(self.data_path,
'non-linear_fits_solution_cubic-spline.fits')
self.assertTrue(os.path.isfile(test_file))
ccd = CCDData.read(test_file, unit='adu')
self.assertRaises(NotImplementedError, self.wcs.read, ccd)
self.assertRaisesRegex(NotImplementedError,
'Cubic spline is not implemented',
self.wcs.read, ccd)
def test_read__non_linear_legendre(self):
test_file = os.path.join(self.data_path,
'non-linear_fits_solution_legendre.fits')
self.assertTrue(os.path.isfile(test_file))
ccd = CCDData.read(test_file, unit='adu')
result = self.wcs.read(ccd=ccd)
self.assertIsInstance(self.wcs.model, Model)
self.assertEqual(self.wcs.model.__class__.__name__, 'Legendre1D')
def test_read__non_linear_cspline(self):
test_file = os.path.join(self.data_path,
'non-linear_fits_solution_cubic-spline.fits')
self.assertTrue(os.path.isfile(test_file))
ccd = CCDData.read(test_file, unit='adu')
self.assertRaises(NotImplementedError, self.wcs.read, ccd)
self.assertRaisesRegex(NotImplementedError,
'Cubic spline is not implemented',
self.wcs.read, ccd)
def test_read_gsp_wcs(self):
test_file = os.path.join(self.data_path,
'goodman_comp_400M1_HgArNe.fits')
self.assertTrue(os.path.isfile(test_file))
ccd = CCDData.read(test_file, unit='adu')
result = self.wcs.read_gsp_wcs(ccd=ccd)
self.assertIsInstance(result, list)
self.assertEqual(len(result), 2)
self.assertIsInstance(self.wcs.get_model(), Model)
def test_read__linear(self):
test_file = os.path.join(self.data_path, 'linear_fits_solution.fits')
self.assertTrue(os.path.isfile(test_file))
ccd = CCDData.read(test_file, unit='adu')
result = self.wcs.read(ccd=ccd)
self.assertIsInstance(result, list)
self.assertEqual(len(result), 2)
self.assertIsInstance(self.wcs.get_model(), Model)
def test_read_gsp_wcs(self):
test_file = os.path.join(self.data_path,
'goodman_comp_400M1_HgArNe.fits')
self.assertTrue(os.path.isfile(test_file))
ccd = CCDData.read(test_file, unit='adu')
result = self.wcs.read_gsp_wcs(ccd=ccd)
self.assertIsInstance(result, list)
self.assertEqual(len(result), 2)
self.assertIsInstance(self.wcs.get_model(), Model)
def test_read__invalid(self):
test_file = os.path.join(self.data_path, 'linear_fits_solution.fits')
self.assertTrue(os.path.isfile(test_file))
ccd = CCDData.read(test_file, unit='adu')
ccd.wcs.wcs.ctype[0] = 'INVALID'
self.assertRaisesRegex(NotImplementedError,
'CTYPE INVALID is not recognized',
self.wcs.read, ccd)
self.assertRaises(NotImplementedError, self.wcs.read, ccd)
def create_plot(mode):
data_path = os.path.dirname(__import__('goodman_lamps').__file__)
search_pattern = os.path.join(data_path,
'data/lamps/*{:s}*.fits'.format(mode))
print(data_path)
print(search_pattern)
file_list = glob.glob(search_pattern)
for _file in file_list:
print(_file)
for file_name in file_list:
fig, ax = plt.subplots(figsize=(20, 7))
ccd = CCDData.read(file_name, unit='adu')
line_list = LineList()
line_list.import_from_file(ccd=ccd)
wavelength, intensity = goodman_wcs.read_gsp_wcs(ccd=ccd)
top_lim = 1.4 * ccd.data.max()
bottom_lim = ccd.data.min() - 0.05 * ccd.data.max()
plt.ylim((bottom_lim, top_lim))
plt.xlim((wavelength[0], wavelength[-1]))
ax.set_title('{:s} - {:s}'.format(ccd.header['object'],
ccd.header['wavmode']))
ax.set_xlabel('Wavelength (Angstrom)')
ax.set_ylabel('Intensity (ADU)')
ax.plot(wavelength, intensity)
for pixel, wavelength, spectrum in line_list.lines:
text = '{:.4f} - {:s}'.format(wavelength, spectrum)
print(text)
plt.axvline(wavelength, alpha=0.1, color='k')
text_x = wavelength
text_y = np.max((ccd.data[int(np.floor(pixel))],
ccd.data[int(np.ceil(pixel))]))
y_offset = 0.05 * ccd.data.max()
plt.text(text_x,
text_y + y_offset,
text,
rotation=90,
verticalalignment='bottom',
horizontalalignment='center')
plt.tight_layout()
plt.show()
print('END. {}'.format(mode))
def test_fit_invalid(self):
test_file = os.path.join(self.data_path,
'goodman_comp_400M1_HgArNe.fits')
ccd = CCDData.read(test_file, unit='adu')
pixel, angstrom = self._recover_lines(ccd=ccd)
self.assertRaisesRegex(NotImplementedError,
'The model invalid is not implemented',
self.wcs.fit, pixel, angstrom, 'invalid')
self.assertRaises(NotImplementedError, self.wcs.fit, pixel, angstrom,
'invalid')
def test_data_classifier_mixed_technique(self):
sample_file = os.listdir(self.raw_path)[0]
raw_path_full = os.path.join(self.raw_path, sample_file)
recovered_ccd = CCDData.read(raw_path_full, unit='adu')
# recovered_ccd.header['INSTCONF'] = 'Blue'
recovered_ccd.header['WAVMODE'] = 'Imaging'
recovered_ccd.write(raw_path_full, overwrite=True)
self.data_classifier(raw_path=self.raw_path)
self.assertEqual('Spectroscopy', self.data_classifier.technique)
def _print_object(self, full_path):
if ".fits" in full_path:
try:
ccd = CCDData.read(full_path, unit='adu')
message = "NEW {:s}".format(full_path)
self.publisher.broadcast(message=message)
except IORegistryError:
self.log.warning("Unable to read file or is not a FITS file: "
"{:s}".format(os.path.basename(full_path)))
else:
self.log.info("File Created: {:s}".format(
os.path.basename(full_path)))
def test_read__linear(self):
test_file = os.path.join(self.data_path,
'linear_fits_solution.fits')
self.assertTrue(os.path.isfile(test_file))
ccd = CCDData.read(test_file, unit='adu')
result = self.wcs.read(ccd=ccd)
self.assertIsInstance(result, list)
self.assertEqual(len(result), 2)
self.assertIsInstance(self.wcs.get_model(), Model)
def test_data_classifier_all_imaging(self):
for _file in os.listdir(self.raw_path):
raw_path_full = os.path.join(self.raw_path, _file)
recovered_ccd = CCDData.read(raw_path_full, unit='adu')
# recovered_ccd.header['INSTCONF'] = 'Blue'
recovered_ccd.header['WAVMODE'] = 'Imaging'
recovered_ccd.write(raw_path_full, overwrite=True)
self.data_classifier(raw_path=self.raw_path)
self.assertEqual('Imaging', self.data_classifier.technique)
def test_read__log_linear(self):
test_file = os.path.join(self.data_path,
'log-linear_fits_solution.fits')
self.assertTrue(os.path.isfile(test_file))
ccd = CCDData.read(test_file, unit='adu')
#
# result = self.wcs.read(ccd=ccd)
#
# self.assertIsInstance(result, list)
# self.assertEqual(len(result), 2)
# self.assertIsInstance(self.wcs.get_model(), Model)
self.assertRaises(NotImplementedError, self.wcs.read, ccd)
def test_read__invalid(self):
test_file = os.path.join(self.data_path,
'linear_fits_solution.fits')
self.assertTrue(os.path.isfile(test_file))
ccd = CCDData.read(test_file, unit='adu')
ccd.wcs.wcs.ctype[0] = 'INVALID'
self.assertRaisesRegex(NotImplementedError,
'CTYPE INVALID is not recognized',
self.wcs.read,
ccd)
self.assertRaises(NotImplementedError, self.wcs.read, ccd)
def create_master_flat(filepath='../../../KeckData/MOSFIRE_FCS/',
flatfiles = ['m180130_0320.fits',
'm180130_0321.fits',
'm180130_0322.fits',
'm180130_0323.fits',
'm180130_0324.fits',],
darkfile = 'm180130_0001.fits',
):
dark = CCDData.read(os.path.join(filepath, darkfile), unit='adu')
flats = []
for i,file in enumerate(flatfiles):
flat = CCDData.read(os.path.join(filepath, file), unit='adu')
flat = flat.subtract(dark)
flats.append(flat)
flat_combiner = Combiner(flats)
flat_combiner.sigma_clipping()
scaling_func = lambda arr: 1/np.ma.average(arr)
flat_combiner.scaling = scaling_func
masterflat = flat_combiner.median_combine()
masterflat.write('masterflat.fits', overwrite=True)
def test_fit_chebyshev(self):
test_file = os.path.join(self.data_path,
'goodman_comp_400M1_HgArNe.fits')
ccd = CCDData.read(test_file, unit='adu')
pixel, angstrom = self._recover_lines(ccd=ccd)
model = self.wcs.fit(physical=pixel, wavelength=angstrom)
self.assertIsInstance(model, Model)
self.assertEqual(model.__class__.__name__, ccd.header['GSP_FUNC'])
self.assertEqual(model.degree, ccd.header['GSP_ORDR'])
for i in range(model.degree + 1):
self.assertAlmostEqual(model.__getattr__('c{:d}'.format(i)).value,
ccd.header['GSP_C{:03d}'.format(i)])
def test_fit_chebyshev(self):
test_file = os.path.join(self.data_path,
'goodman_comp_400M1_HgArNe.fits')
ccd = CCDData.read(test_file, unit='adu')
pixel, angstrom = self._recover_lines(ccd=ccd)
model = self.wcs.fit(physical=pixel, wavelength=angstrom)
self.assertIsInstance(model, Model)
self.assertEqual(model.__class__.__name__, ccd.header['GSP_FUNC'])
self.assertEqual(model.degree, ccd.header['GSP_ORDR'])
for i in range(model.degree + 1):
self.assertAlmostEqual(model.__getattr__('c{:d}'.format(i)).value,
ccd.header['GSP_C{:03d}'.format(i)])
def makeMasterBias(images):
"""
Make a master bias using all biases found in
images object
TODO: Finish docstring
"""
try:
master_bias = CCDData.read('master_bias.fits', unit=u.adu)
return master_bias
except FileNotFoundError:
bias_list = []
for f in images.files_filtered(imagetyp=BIAS_KEYWORD):
print(f)
ccd = CCDData.read(f, unit=u.adu)
bias_list.append(ccd)
try:
master_bias = combine(bias_list, method='median')
master_bias.write('master_bias.fits', clobber=True)
return master_bias
except IndexError:
return None
def __call__(self):
"""Run the tool for all the images matching `search_pattern`"""
for fits_file in self.file_list:
print(fits_file)
self.ccd = CCDData.read(fits_file, unit=u.adu)
if not self.threads:
# plot_thread = Thread(target=self._create_plot)
# plot_thread.start()
id_thread = Thread(target=self.identify_matching_line)
id_thread.start()
# self.identify_matching_line()
# self.threads.
self._create_plot()
id_thread.join()
self.ccd.write(fits_file, overwrite=True)
def __init__(self, reference_dir):
self.cleaned_list = []
reference_data = ReferenceData(reference_dir)
file_list = glob.glob(os.path.join(reference_dir, '*fits'))
for lfile in file_list:
self.fig, self.ax = plt.subplots()
ccd = CCDData.read(lfile, unit=u.adu)
read_wavelength = ReadWavelengthSolution(ccd.header, ccd.data)
wavelength, intensity = read_wavelength()
self.line_list = reference_data.get_line_list_by_name(
ccd.header['OBJECT'])
file_name = lfile.split('/')[-1]
pickle_file_name = re.sub('.fits', '_list.pkl', file_name)
manager = plt.get_current_fig_manager()
manager.window.showMaximized()
if os.path.isfile(pickle_file_name):
with open(pickle_file_name, 'rb') as pickled_file:
line_list = pickle.load(pickled_file)
for ref_line in line_list:
self.ax.axvline(ref_line, color='r', alpha=1)
else:
for ref_line in self.line_list:
self.ax.axvline(ref_line, color='r', alpha=.4)
self.ax.set_title(file_name)
self.ax.plot(wavelength, intensity, color='k')
self.ax.set_xlabel('Wavelength (Angstrom)')
self.ax.set_ylabel('Intensity (ADU)')
self.ax.set_xlim((wavelength[0], wavelength[-1]))
self.fig.canvas.mpl_connect('button_press_event', self.on_click)
plt.show()
if self.cleaned_list != []:
with open(pickle_file_name, 'wb') as list_file:
pickle.dump(self.cleaned_list,
list_file,
protocol=pickle.HIGHEST_PROTOCOL)
def test_fit_invalid(self):
test_file = os.path.join(self.data_path,
'goodman_comp_400M1_HgArNe.fits')
ccd = CCDData.read(test_file, unit='adu')
pixel, angstrom = self._recover_lines(ccd=ccd)
self.assertRaisesRegex(NotImplementedError,
'The model invalid is not implemented',
self.wcs.fit,
pixel,
angstrom,
'invalid')
self.assertRaises(NotImplementedError,
self.wcs.fit,
pixel,
angstrom,
'invalid')
def test_write_gsp_wcs(self):
test_file = os.path.join(self.data_path,
'goodman_comp_400M1_HgArNe.fits')
ccd = CCDData.read(test_file, unit='adu')
pixel, angstrom = self._recover_lines(ccd=ccd)
model = self.wcs.fit(physical=pixel, wavelength=angstrom)
self.assertIsInstance(model, Model)
blank_ccd = CCDData(data=np.ones(ccd.data.shape),
meta=fits.Header(),
unit='adu')
blank_ccd.header.set('GSP_WREJ', value=None, comment='empty')
new_ccd = self.wcs.write_gsp_wcs(ccd=blank_ccd, model=model)
self.assertEqual(new_ccd.header['GSP_FUNC'], ccd.header['GSP_FUNC'])
self.assertEqual(new_ccd.header['GSP_ORDR'], ccd.header['GSP_ORDR'])
self.assertEqual(new_ccd.header['GSP_NPIX'], ccd.header['GSP_NPIX'])
for i in range(model.degree + 1):
self.assertAlmostEqual(new_ccd.header['GSP_C{:03d}'.format(i)],
ccd.header['GSP_C{:03d}'.format(i)])
def dq_ccd_insert(filename, sdb):
"""Insert CCD information into the database
Parameters
----------
filename: str
Raw file name
sdb: sdb_user.mysql
Connection to the sdb database
"""
logic="FileName='%s'" % os.path.basename(filename)
FileData_Id = sdb.select('FileData_Id','FileData',logic)[0][0]
i = 0
record=sdb.select('FileData_Id', 'PipelineDataQuality_CCD', 'FileData_Id=%i and Extension=%i' % (FileData_Id, i))
update=False
if record: update=True
#lets measureme the statistics in a 200x200 box in each image
struct = CCDData.read(filename, unit='adu')
my,mx=struct.data.shape
dx1=int(mx*0.5)
dx2=min(mx,dx1+200)
dy1=int(my*0.5)
dy2=min(my,dy1+200)
mean,med,sig=stats.sigma_clipped_stats(struct.data[dy1:dy2,dx1:dx2], sigma=5, iters=5)
omean=None
orms=None
ins_cmd=''
if omean is not None: ins_cmd='OverscanMean=%s,' % omean
if orms is not None: ins_cmd+='OverscanRms=%s,' % orms
if mean is not None: ins_cmd+='BkgdMean=%f,' % mean
if sig is not None: ins_cmd+='BkgdRms=%f' % sig
if update:
ins_cmd=ins_cmd.rstrip(',')
sdb.update(ins_cmd, 'PipelineDataQuality_CCD', 'FileData_Id=%i and Extension=%i' % (FileData_Id, i))
else:
ins_cmd+=',FileData_Id=%i, Extension=%i' % (FileData_Id, i)
sdb.insert(ins_cmd, 'PipelineDataQuality_CCD')
def aller_test(image_list):
wcs = WCS()
assert isinstance(wcs, WCS)
all_data = []
all_dates = []
for file_name in image_list:
ccd = CCDData.read(file_name, unit=u.adu)
print("Master Flat used: {:s}".format(ccd.header['GSP_FLAT']))
print("{:s} : {:s}".format(ccd.header['DATE-OBS'], file_name))
# print(ccd.header["GSP_*"])
wav, intens = wcs.read(ccd=ccd)
all_data.append([wav, normalize_data(intens), ccd.header])
all_dates.append(ccd.header['DATE'])
plt.title(file_name)
plt.plot(wav, normalize_data(intens))
plt.xlabel("Wavelength (Angstrom)")
plt.ylabel("Intensity (ADU)")
plt.show()
fig, (ax1, ax2) = plt.subplots(1, 2, sharey=True)
plt.title("Normalized Spectrum")
ax1.axvline(4822.3787, label='4822.3787A', color='k', linestyle='--',
alpha=0.7)
ax2.axvline(4955.91, label='4955.91A', color='k', linestyle='--', alpha=0.7)
for i in range(len(all_data)):
ax1.plot(all_data[i][0], all_data[i][1], label=all_dates[i])
ax1.set_xlim(4817, 4826)
# ax1.axvline(4822.3787, label='4822.3787A', color='k', linestyle='--')
ax2.plot(all_data[i][0], all_data[i][1], label=all_dates[i])
ax2.set_xlim(4951, 4958)
ax1.set_xlabel("Wavelength (Angstrom)")
ax1.set_ylabel("Intensity (ADU)")
ax1.legend(loc='best')
plt.legend(loc='best')
plt.show()
return all_data
def astrometry_for_directory(directories,
destination=None,
no_log_destination=False,
blind=False):
"""
Add astrometry to files in list of directories
Parameters
----------
directories : str or list of str
Directory or directories whose FITS files are to be processed.
blind : bool, optional
Set to True to force blind astrometry. False by default because
blind astrometry is slow.
"""
for currentDir in directories:
images = ImageFileCollection(currentDir,
keywords=['imagetyp', 'object',
'wcsaxes', 'ra', 'dec'])
summary = images.summary_info
if len(summary) == 0:
continue
logger.debug('\n %s', '\n'.join(summary.pformat()))
lights = summary[((summary['imagetyp'] == 'LIGHT') &
(summary['wcsaxes'].mask))]
working_dir = destination if destination is not None else currentDir
if (not no_log_destination) and (destination is not None):
add_file_handlers(logger, working_dir, 'run_astrometry')
logger.debug('About to loop over %d files', len(lights['file']))
for light_file in lights['file']:
if ((destination is not None) and (destination != currentDir)):
src = path.join(currentDir, light_file)
shutil.copy(src, destination)
original_fname = path.join(working_dir, light_file)
img = CCDData.read(original_fname, unit='adu')
try:
ra = img.header['ra']
dec = img.header['dec']
ra_dec = (ra, dec)
except KeyError:
ra_dec = None
if (ra_dec is None) and (not blind):
root, ext = path.splitext(original_fname)
f = open(root + '.blind', 'wb')
f.close()
continue
astrometry = ast.add_astrometry(original_fname,
ra_dec=ra_dec,
note_failure=True,
overwrite=True)
with fits.open(original_fname,
do_not_scale_image_data=True) as f:
try:
del f[0].header['imageh'], f[0].header['imagew']
f.writeto(original_fname, clobber=True)
except KeyError:
pass
if astrometry and ra_dec is None:
root, ext = path.splitext(original_fname)
img_new = CCDData.read(original_fname, unit='adu')
# The ndmin below ensures center_pix has the right shape
# for WCS conversion.
center_pix = np.trunc(np.array(img_new.shape, ndmin=2) / 2)
print(center_pix[np.newaxis, :].shape)
ra_dec = \
img_new.wcs.all_pix2world(center_pix,
1)
ra_dec = ra_dec[0]
# RA/Dec are in degrees. Convert them to sexagesimal for
# output. Yuck, but makes it easier for existing code to
# handle.
# Note that FK5 is J2000.
coords = SkyCoord(*ra_dec, unit=(u.degree, u.degree),
frame='fk5')
img_new.header['RA'] = coords.ra.to_string(unit=u.hour,
sep=':')
img_new.header['DEC'] = coords.dec.to_string(sep=':')
img_new.write(original_fname, clobber=True)
def wave2Dfit(ccd, order_frame, soldir, outfile):
l_xarr,l_warr,l_oarr=read_arclines(ccd, order_frame, soldir)
xarr,farr,oarr=read_fits(ccd, order_frame, soldir)
fitting_app=QtGui.QApplication(sys.argv)
fitting_window=FitWavelengthWindow(l_xarr,l_warr,l_oarr,xarr,farr,oarr, outfile)
fitting_window.raise_()
fitting_app.exec_()
fitting_app.deleteLater()
return True
if __name__=='__main__':
parser = argparse.ArgumentParser()
parser.add_argument("spectrum_fits",help="Fits file with an extracted HRS spectrum",type=str)
parser.add_argument("order_frame",help="HRS order frame",type=str)
parser.add_argument("calibration_folder",help="Path to the lr/mr/hr calibration folder",type=str)
args=parser.parse_args()
ccd = CCDData.read(args.spectrum_fits)
order_frame = CCDData.read(args.order_frame, unit=u.adu)
soldir = args.calibration_folder
outfile = (args.spectrum_fits).replace('.fits', '_spec.fits')
wave2Dfit(ccd, order_frame, soldir, outfile)
import argparse
if __name__=='__main__':
parser = argparse.ArgumentParser(description='Process SALT HRS observations')
parser.add_argument('infile', help='SALT HRS image')
parser.add_argument('mccd', help='Master bias file')
parser.add_argument('--o', dest='order', help='Master order file')
parser.add_argument('--f', dest='flat', help='Master flat file')
parser.add_argument('-s', dest='oscan', default=False, action='store_true', help='Apply overscan correction')
parser.add_argument('-n', dest='cray', default=True, action='store_false', help='Do not cosmic ray clean')
args = parser.parse_args()
infile = args.infile
mccd = CCDData.read(args.mccd, unit='electron')
if os.path.basename(infile).startswith('H'):
ccd = blue_process(infile, masterbias=mccd, oscan_correct=args.oscan)
elif os.path.basename(infile).startswith('R'):
ccd = red_process(infile, masterbias=mccd, oscan_correct=args.oscan)
else:
exit('Are you sure this is an HRS file?')
if args.cray:
from astroscrappy import detect_cosmics
crmask, cleanarr = detect_cosmics(ccd.data, inmask=None, sigclip=4.5, sigfrac=0.3,
objlim=5.0, gain=1.0, readnoise=6.5,
satlevel=65536.0, pssl=0.0, niter=4,
sepmed=True, cleantype='meanmask', fsmode='median',
psfmodel='gauss', psffwhm=2.5, psfsize=7,
def write_bias_flat_arc(files):
"""Creates masterbias masterflat and masterarc files
Parameters
----------
files: list of str
List containing names of HRS files
Returns
-------
Writes HBIAS.fits RBIAS.fits HFLAT.fits RFLAT.fits HARC.fits and RARC.fits in the current directory
"""
#create a list of calibration frames
hbias_list = []
rbias_list = []
hflat_list = []
rflat_list = []
harc_list = []
rarc_list = []
for img in files:
data, header = fits.getdata(img, header=True)
if not ('DATASEC' in header):
#just adding 'DATASEC' keyword if missing
header['DATASEC']=header['AMPSEC']
fits.writeto(img, data, header, clobber=True)
if header['OBSTYPE']=='Bias' and header['DETNAM']=='HBDET':
hbias_list.append(img)
if header['OBSTYPE']=='Bias' and header['DETNAM']=='HRDET':
rbias_list.append(img)
if 'EXPTYPE' in header:
if header['EXPTYPE']=='Flat field' and header['DETNAM']=='HBDET':
hflat_list.append(img)
if header['EXPTYPE']=='Flat field' and header['DETNAM']=='HRDET':
rflat_list.append(img)
if header['CCDTYPE']=='Arc' and header['DETNAM']=='HBDET':
harc_list.append(img)
if header['CCDTYPE']=='Arc' and header['DETNAM']=='HRDET':
rarc_list.append(img)
else:
if header['CCDTYPE']=='Flat field' and header['DETNAM']=='HBDET':
hflat_list.append(img)
if header['CCDTYPE']=='Flat field' and header['DETNAM']=='HRDET':
rflat_list.append(img)
if header['CCDTYPE']=='Arc' and header['DETNAM']=='HBDET':
harc_list.append(img)
if header['CCDTYPE']=='Arc' and header['DETNAM']=='HRDET':
rarc_list.append(img)
if not(hbias_list==[]):
master_bluebias = create_masterbias(hbias_list)
master_bluebias.write('HBIAS.fits', clobber=True)
print("\nCreated blue masterbias\n")
#this step is just inserted for debugging purposes
master_bluebias = CCDData.read('HBIAS.fits', ignore_missing_end=True)
gc.collect()
if not(hflat_list==[]):
master_blueflat = create_masterflat(hflat_list) #if bias was created it should probably be "create_masterflat(hflat_list,masterbias=master_bluebias)", but it generates an error
master_blueflat.write('HFLAT.fits', clobber=True)
del master_blueflat
gc.collect()
print("\nCreated blue masterflat\n")
if not(harc_list==[]):
master_bluearc = create_masterflat(harc_list) #if bias was created it should probably be "create_masterflat(hflat_list,masterbias=master_bluebias)", but it generates an error
master_bluearc.write('HARC.fits', clobber=True)
del master_bluearc
gc.collect()
print("\nCreated blue masterarc\n")
del master_bluebias
gc.collect()
if not(rbias_list==[]):
master_redbias = create_masterbias(rbias_list)
master_redbias.write('RBIAS.fits', clobber=True)
print("\nCreated red masterbias\n")
#this step is just inserted for debugging purposes
master_redbias = CCDData.read('RBIAS.fits', ignore_missing_end=True)
gc.collect()
if not(rflat_list==[]):
master_redflat = create_masterflat(rflat_list)#if bias was created it should probably be "create_masterflat(rflat_list,masterbias=master_rluebias)", but it generates an error
master_redflat.write('RFLAT.fits', clobber=True)
del master_redflat
gc.collect()
print("\nCreated red masterflat\n")
if not(rarc_list==[]):
master_redarc = create_masterflat(rarc_list)
master_redarc.write('RARC.fits', clobber=True)
del master_redarc
gc.collect()
print("\nCreated red masterarc\n")
del master_redbias
gc.collect()
return True
def write_orderframe(f_limit_red=1000.0,f_limit_blue=1000.0,interactive=True):
"""Creates blue and red order frames
Parameters
----------
f_limit_red: float
Limiting value for detecting orders in the red flat. Value should be higher than counts of the background light, but lower than in most part of any order.
f_limit_blue: float
Same as f_limit_red, but for blue frame.
interactive: bool
If true the program will display the flat frames in DS9 and it will ask the user to enter values of f_limit_red and f_limit_blue
Returns
-------
Writes HNORM.fits RNORM.fits HORDER.fits and RORDER.fits in the current directory
"""
print("Creating normalized red flat")
if interactive:
import pyds9
d=pyds9.DS9()
d.set("file RFLAT.fits")
print("Flat opened in DS9")
check=True
while check:
f_limit_red=raw_input("Enter a count number that is higher than background light, but lower than in most part of any order: ")
try:
f_limit_red=float(f_limit_red)
check=False
except ValueError:
print("\nError: Entered value should be a float ")
master_redflat = CCDData.read('RFLAT.fits', ignore_missing_end=True)
image = nd.filters.maximum_filter(master_redflat, 10)
mask = (image > f_limit_red)
norm = normalize_image(image, mod.models.Legendre1D(10), mask=mask)
norm[norm < f_limit_red]=0
hdu = fits.PrimaryHDU(norm)
hdu.writeto('RNORM.fits', clobber=True)
#create the initial detection kernal
ys, xs = norm.shape
xc = int(xs/2.0)
norm[norm>500] = 500
ndata = norm[:,xc]
detect_kern = ndata[5:80]
#these remove light that has bleed at the edges and may need adjusting
norm[:,:20]=0
norm[:,4040:]=0
#detect orders in image
frame = create_orderframe(norm, 53, xc, detect_kern, y_start=8,
y_limit=3920, smooth_length=20)
hdu = fits.PrimaryHDU(frame)
hdu.writeto('RORDER.fits', clobber=True)
print("Creating normalized blue flat")
if interactive:
d.set("file HFLAT.fits")
print("Flat opened in DS9")
check=True
while check:
f_limit_blue=raw_input("Enter a count number that is higher than background light, but lower than in most part of any order: ")
try:
f_limit_blue=float(f_limit_blue)
check=False
except ValueError:
print("\nError: Entered value should be a float ")
#normalize the blue image--these values may need adjusting based on the image
master_blueflat = CCDData.read('HFLAT.fits', ignore_missing_end=True)
image = nd.filters.maximum_filter(master_blueflat, 10)
mask = (image > f_limit_blue)
norm = normalize_image(image, mod.models.Legendre1D(10), mask=mask)
norm[norm < f_limit_blue]=0
hdu = fits.PrimaryHDU(norm)
hdu.writeto('HNORM.fits', clobber=True)
#create the initial detection kernal
ys, xs = norm.shape
xc = int(xs/2.0)
norm[norm>500] = 500
ndata = norm[:,xc]
detect_kern = ndata[5:80]
#these remove light that has bleed at the edges and may need adjusting
norm[:,:20]=0
norm[:,4040:]=0
#detect orders in image
frame = create_orderframe(norm, 53, xc, detect_kern, y_start=8,
y_limit=3920, smooth_length=20)
hdu = fits.PrimaryHDU(frame)
hdu.writeto('HORDER.fits', clobber=True)
return True
function = 'poly'
order = 3
iws = InterIdentify(xarr, fdata, slines[smask], sfluxes[smask], ws, mdiff=mdiff, rstep=rstep,
function=function, order=order, sigma=thresh, niter=niter, wdiff=wdiff,
res=res, dres=dres, dc=dc, ndstep=ndstep, istart=istart,
method=method, smooth=smooth, filename=filename,
subback=subback, textcolor=textcolor, log=log, verbose=True)
return dc_dict, iws
if __name__=='__main__':
arc = CCDData.read(sys.argv[1])
order_frame = CCDData.read(sys.argv[2], unit=u.adu)
n_order = int(sys.argv[3]) #default order to use for initial file
camera_name = arc.header['DETNAM'].lower()
res = 0.5
w_c = None
if camera_name=='hrdet':
arm = 'R'
if arc.header['OBSMODE']=='HIGH RESOLUTION':
xpos = -0.025
target = True
res = 0.1
w_c = mod.models.Polynomial1D(2, c0=0.440318305862, c1=0.000796335104265,c2=-6.59068602173e-07)
parser = argparse.ArgumentParser(description='Process Aries engineering observationss')
parser.add_argument('infile', nargs='*', help='File or files to be processed')
parser.add_argument('--b', dest='bias', help='Master bias file', default=None)
parser.add_argument('--d', dest='dark', help='Master dark file', default=None)
parser.add_argument('--f', dest='flat', help='Master flat file', default=None)
parser.add_argument('-n', dest='cray', default=True, action='store_false', help='Do not cosmic ray clean')
args = parser.parse_args()
infiles = args.infile
print infiles
if args.bias:
mbias = CCDData.read(args.bias, unit = u.adu)
else:
mbias = None
if args.flat:
raise Exception('Flat fielding is not currently implemented')
if args.dark:
raise Exception('Dark correction is not currently implemented')
for infile in infiles:
ccd = CCDData.read(infile, unit = u.adu)
ccd = ccdproc.ccd_process(ccd, oscan='[1117:1181, 1:330]', oscan_median=True,
trim='[17:1116,1:330]', master_bias=mbias,
error=True, gain=1.0 * u.electron/u.adu,
readnoise=5.0 * u.electron)
if args.cray:
def correctData(filename, master_bias, master_flat, filetype):
"""
Correct a science image using the available
master calibrations. Skip a calibration step if the
master frame does not exist.
No reduced file is written in this new scheme.
Instead, the corrected data is passed directly
to the phot() routine, photometry is done as per
the configuration and the photometry is written out
only.
TODO: Finish docstring
"""
print('Reducing {0:s}...'.format(filename))
with fits.open(filename) as fitsfile:
# correct times for science spectra,
# don't bother for arcs
hdr = fitsfile[0].header
if filetype == 'science':
half_exptime = hdr[EXPTIME_KEYWORD]/2.
utstart = hdr[UTSTART_KEYWORD]
dateobs = hdr[DATEOBS_KEYWORD]
ra = hdr[RA_KEYWORD]
dec = hdr[DEC_KEYWORD]
time_start = Time('{}T{}'.format(dateobs, utstart),
scale='utc',
format='isot',
location=OBSERVATORY)
# correct to mid exposure time
jd_mid = time_start + half_exptime*u.second
ltt_bary, ltt_helio = getLightTravelTimes(ra, dec, jd_mid)
time_bary = jd_mid.tdb + ltt_bary
time_helio = jd_mid.utc + ltt_helio
hdr['BJD-MID'] = time_bary.jd
hdr['HJD-MID'] = time_helio.jd
hdr['JD-MID'] = jd_mid.jd
hdr['UT-MID'] = jd_mid.isot
ccd = CCDData.read(filename, unit=u.adu)
if master_bias:
ccd = subtract_bias(ccd, master_bias)
else:
print('No master bias, skipping correction...')
if master_flat:
ccd = flat_correct(ccd, master_flat)
else:
print('No master flat, skipping correction...')
# after calibrating we get np.float64 data
# if there are no calibrations we maintain dtype = np.uint16
# sep weeps
# fix this by doing the following
if isinstance(ccd.data[0][0], np.uint16):
ccd.data = ccd.data.astype(np.float64)
# trim the data
ccd_trimmed = trim_image(ccd[1000:3001, :])
# write out the trimmed file and the updated header
#ccd_trimmed.write(filename, hdr, clobber=True)
trimmed_filename = '{}_t.fits'.format(filename.split('.')[0])
fits.writeto(trimmed_filename, ccd_trimmed.data, hdr)
# remove the old untrimmed data
os.system('rm {}'.format(filename))
def sdiff(
afile,
bfile,
yc=roi['yc'],
dy=roi['dy'],
bg1=roi['bg1'],
bg2=roi['bg2'],
headertxt='5500',
wc=5500,
dw=-0.7,
xsum=1,
save=None,
plot=True,
):
# print '--%s %s--' % (afile, bfile)
accd = CCDData.read(afile)
bccd = CCDData.read(bfile)
y1 = yc - dy
y2 = yc + dy
bg1 = bg1
bg2 = bg2
xbin = xsum
# extract signal
aspec = (accd.data[y1:y2,:] - np.median(accd.data[bg1:bg2,:], axis=0)).sum(axis=0)
bspec = (bccd.data[y1:y2,:] - np.median(bccd.data[bg1:bg2,:], axis=0)).sum(axis=0)
rspec = bspec/aspec
# apply spectral binning
abin = aspec.reshape(-1,xbin)
bbin = bspec.reshape(-1,xbin)
rbin = rspec.reshape(-1,xbin)
# estimate variance from counts per bin
# sigma-clip data in each bin for variance calc
#v = np.apply_along_axis(st.sigmaclip, 1, abin, low=3.0, high=3.0)
#avar = np.array([np.var(x) for x in v[:,0]])
#v = np.apply_along_axis(st.sigmaclip, 1, bbin, low=3.0, high=3.0)
#bvar = np.array([np.var(x) for x in v[:,0]])
# estimate variance from ratio per bin
c = np.apply_along_axis(st.sigmaclip, 1, rbin, low=3.0, high=3.0)
rvar = 2.0/xbin*np.array([np.var(x) for x in c[:,0]])
# perform sigma clipping on bins
c = np.apply_along_axis(st.sigmaclip, 1, abin, low=3.0, high=3.0)
aspec = np.array([np.mean(x) for x in c[:,0]])
c = np.apply_along_axis(st.sigmaclip, 1, bbin, low=3.0, high=3.0)
bspec = np.array([np.mean(x) for x in c[:,0]])
#aspec = abin.mean(axis=1)
#bspec = bbin.mean(axis=1)
rspec = bspec/aspec
# extract variance
# accdv = np.nan_to_num(accd.uncertainty.array)
# bccdv = np.nan_to_num(bccd.uncertainty.array)
# avar = accdv[y1:y2,:].sum(axis=0)
# bvar = bccdv[y1:y2,:].sum(axis=0)
# avar = avar.reshape(-1,xbin).mean(axis=1)
# bvar = bvar.reshape(-1,xbin).mean(axis=1)
xarr = np.arange(len(aspec))
warr = (wc + xbin/2) + dw*xbin*xarr
print 'ave = %f %%' % (rspec.mean()*100.0)
print 'stdev = %f %%' % (rspec.std()*100.0)
print 'range = %f %%' % ((rspec.max() - rspec.min())*100.0)
#print 'S/N = %f / %f = %f' % (aspec.mean(), avar.mean(), aspec.mean()/avar.mean())
if save:
oarr = np.array([warr, aspec, bspec, rspec, rvar]).T
oarr = oarr[oarr[:,0].argsort()]
hdrtxt = "" # "\n%s\t%s\t%s\nwavelength [A]\trefspec [counts]\tcompspec [counts]\n" % (headertxt, afile, bfile)
np.savetxt(save, oarr, fmt="%10e", delimiter="\t", header=hdrtxt)
if plot:
pl.figure()
pl.subplot(311)
pl.plot(warr, aspec)
pl.plot(warr, bspec)
pl.ylabel('Counts', size='x-large')
pl.subplot(312)
pl.plot(warr, bspec-aspec)
pl.ylabel('Diff', size='x-large')
pl.subplot(313)
pl.plot(warr, rspec*100.0)
pl.ylabel('Deviation (%)', size='x-large')
pl.xlabel('Wavelength', size='x-large')
pl.show()
return shift_dict, iws
if __name__=='__main__':
import argparse
parser = argparse.ArgumentParser(description='Re-identify SALT HRS arc observations')
parser.add_argument('infile', help='SALT HRS image')
parser.add_argument('n_order', help='Order for the identification')
parser.add_argument('--o', dest='order', help='Master order file')
parser.add_argument('--sol', dest='soldir', help='Directory containing the solutions', default='./')
parser.add_argument('--t', dest='target', help='Force extraction of "upper or "lower" fiber', default=None)
args = parser.parse_args()
arc = CCDData.read(args.infile)
order_frame = CCDData.read(args.order, unit='electron')
n_order = int(args.n_order)
soldir = args.soldir
shift_dict, ws = pickle.load(open(soldir+'sol_%i.pkl' % n_order))
camera_name = arc.header['DETNAM'].lower()
arm, xpos, target, res, w_c, y1, y2 = mode_setup_information(arc.header)
if args.target:
target = args.target
print arm, xpos, target
target = 'upper'
dc_dict, iws = identify(arc, order_frame, n_order, camera_name, xpos, ws=ws,
target=target, interp=True, w_c=w_c,
