def produce_cosmic_ray_masks(hdulist, inputs, fnm):
mask, clean = detect_cosmics(hdulist[0].data, sigfrac=0.15, sigclip=4, objlim=4, cleantype='idw')
if(inputs['PLOTMASKS']):
plt.rcParams['figure.facecolor']='white'
plt.rcParams['font.size']=18
print 'PLOTTING CR MASKS'
# plots to test out code
# print 'TRIMSEC', hdulist[0].header['TRIMSEC']
# print 'trimsec', trimsec
# print np.sum(mask), np.size(hdulist[0].data[trimsec].ravel())
plt.figure(figsize=(23,6))
plt.suptitle(fnm.split('/')[-1].split('.')[0])
trimsec = utilities.parse_region_keyword(hdulist[0].header['TRIMSEC'])
ax = plt.subplot(131)
#print np.percentile((hdulist[0].data), 99.9)
#plt.imshow(hdulist[0].data[trimsec], interpolation = 'none', cmap='jet', vmin = np.percentile(hdulist[0].data[trimsec], 0.5), vmax = np.percentile(hdulist[0].data[trimsec], 99.5))
plt.imshow(np.log10(hdulist[0].data[trimsec]), interpolation = 'none', cmap='jet', vmin = np.log10(np.percentile(hdulist[0].data[trimsec], 0.5)), vmax = np.log10(np.percentile(hdulist[0].data[trimsec], 99.5)))
plt.ylim(4096,0)
plt.xlim(0, 4096)
plt.colorbar()
plt.subplot(132, sharex=ax, sharey=ax)
#plt.imshow(np.ma.masked_where(mask[trimsec]==1,hdulist[0].data[trimsec]), interpolation = 'none', cmap='jet', vmin = np.percentile(hdulist[0].data[trimsec], 0.5), vmax = np.percentile(hdulist[0].data[trimsec], 99.5))
plt.imshow(np.log10(np.ma.masked_where(mask[trimsec]==1,hdulist[0].data[trimsec])), interpolation = 'none', cmap='jet', vmin = np.log10(np.percentile(hdulist[0].data[trimsec], 0.5)), vmax = np.log10(np.percentile(hdulist[0].data[trimsec], 99.5)))
plt.colorbar()
plt.xlabel('Pixel (%1.2f arcsec/pixel)'%hdulist[0].header['PIXSCALE'])
plt.subplot(133, sharex=ax, sharey=ax)
plt.imshow(mask[trimsec], interpolation = 'none', cmap='gray_r')
plt.colorbar()
#plt.subplot(224, sharex=ax, sharey=ax)
#plt.imshow(clean[trimsec], interpolation = 'none', cmap='jet', vmin = np.percentile(clean, 0.5), vmax = np.percentile(clean, 99.5))
#plt.colorbar()
plt.subplots_adjust(left=0.05, right=0.95)
plt.show()
#plt.savefig(outfnm)
return mask
def cosmic_rays1(self,maskflag=4,sigclip=0.5,sigfrac=0.001,objlim=10.0,gain=1.0,readnoise=0.1,pssl=0.0,niter=4,sepmed=True,cleantype='medmask',cr_nthresh=3.0,satlevel=60000):
start=time.time()
maskflag=maskflag | self.bad_flag
mask,_=astroscrappy.detect_cosmics(self.data,inmask=self.mask,sigclip=sigclip,sigfrac=sigfrac,
objlim=objlim,gain=gain, readnoise=readnoise, satlevel=self.saturate,
pssl=pssl,niter=niter,sepmed=sepmed, cleantype=cleantype,verbose=True)
#fsmode='median', psfmodel='gauss', psffwhm=2.5,
#psfsize=7, psfk=None, psfbeta=4.765, verbose=True)
if cr_nthresh is not None and cr_nthresh > 0:
tmpmask=(self.data-self.sky > cr_nthresh*self.sky_rms) & (self.data < self.saturate)
mask=mask & tmpmask
flags=mask*maskflag
count=mask.sum()
if self.flags is not None:
self.flags = np.bitwise_or(self.flags,flags)
else:
self.flags=flags
if self.mask is not None:
self.mask = self.mask | (flags > 0)
else:
self.mask = flags > 0
if self.weight is not None:
tmpmask=flags>0
self.weight[tmpmask]=0
print('cosmic ray time: ',time.time()-start)
print('cosmic ray pixels: ', count)
return count
def normalize(fitsname, output=None, cr_remove=True, multiextension=True,
masterflat="masterflat.fits", masterbias="masterbias.fits",
cr_options=None):
'''
Flat field, bias subtraction, and cosmic ray removal with astroscrappy.
Parameters:
-----------
fitsname: either string or 2d array, if string then name of fits file,
if 2d array, then the output of arrange_fits
output: string, name of file to write out to (optional)
if None, then just returns the array
cr_remove: boolean, if true, the subtract cosmics
multiextension: boolean, if true, then
masterflat: str, name of fits file with master flat (optional)
if None, then make a master flat from files in flatdir
masterbias: str, name of fits file with master bias (optional)
if None, then make a master bias from files in biasdir
:cr_options: dict, keys should be keywords of detect_cosmics from
astroscrappy, overrides defaults below
Returns:
--------
:normalized_data: 2d numpy array, rows are spectral, columns are spatial
'''
if masterbias is None:
bias = make_masterbias()
else:
bias = fits.getdata(masterbias)
if masterflat is None:
flat = make_masterflat()
else:
flat = fits.getdata(masterflat)
if multiextension:
raw_data = multiextension_to_array(fitsname)
header = fits.open(fitsname)[0].header
header.extend(deimos_cards(raw_data.shape))
else:
raw_data = fits.getdata(fitsname)
header = fits.getheader(fitsname)
if cr_remove:
flat_mask = get_slitmask(flat)
kwargs = {'verbose': True, 'inmask': flat_mask, 'cleantype': 'medmask',
'sigclip': 0.5, 'sigfrac': 0.1, 'niter': 4}
if cr_options is not None:
for key, value in cr_options.items():
kwargs[key] = value
mask, cleaned = detect_cosmics(raw_data, **kwargs)
normed_data = (cleaned - bias) / flat
else:
normed_data = (raw_data - bias) / flat
if output is not None:
fits.writeto(output, data=normed_data, header=header, clobber=True)
return normed_data
def get_cr_mask_astroscrappy(d, gain=2.2, readnoise=10.0,
sigclip=5, sigfrac = 0.3, objlim = 5.0):
c = astroscrappy.detect_cosmics(d, gain=gain, readnoise=readnoise,
sigclip=sigclip, sigfrac=sigfrac,
objlim=objlim,
cleantype='medmask',
psfmodel="gaussx")
return c[0]
def crreject(scifiles):
for f in scifiles:
# run lacosmicx
hdu = pyfits.open('st' + f.replace('.txt', '.fits'))
readnoise = 3.5
# figure out what pssl should be approximately
d = hdu[2].data.copy()
dsort = np.sort(d.ravel())
nd = dsort.shape[0]
# Calculate the difference between the 16th and 84th percentiles to be
# robust against outliers
dsig = (dsort[0.84 * nd] - dsort[0.16 * nd]) / 2.0
pssl = (dsig * dsig - readnoise * readnoise)
mask = d == 0.0
crmask, _cleanarr = detect_cosmics(d, inmask=mask, sigclip=4.0,
objlim=1.0, sigfrac=0.05, gain=1.0,
readnoise=readnoise, pssl=pssl)
tofits(f[:-4] + '.lamask.fits', np.array(crmask, dtype=np.uint8), hdr=hdu['SCI'].header.copy())
def make_masterbias(output="masterbias.fits", biasdir="bias/", remove_cr=True):
'''
Creates a median bias file from fits files found in biasdir.
'''
bias_files = glob.glob(biasdir + "*.fits")
# arbitrarily taking the first exposure as the header info
hdulist = fits.open(bias_files[0])
primary_header = hdulist[0].header
headers = [hdu.header for hdu in hdulist[1:9]]
bias_data = []
# axis 0 is different exposures, axis 1 is different CCDs
# axis 2, 3 are y, x
for i, f in enumerate(bias_files):
bias_data.append(np.array([hdu.data for hdu in fits.open(f)[1:9]]))
bias_data = np.array(bias_data)
# if we have more bias frames, we should cr subtract before medianing biases
bias = np.nanmedian(bias_data, axis=0)
if remove_cr:
masks = []
cr_cleaned = []
for frame in bias:
mask, cleaned = detect_cosmics(frame, verbose=True,
cleantype='medmask',
sigclip=0.2)
masks.append(mask)
cr_cleaned.append(cleaned)
bias = np.array(cr_cleaned)
bias = multiextension_to_array(data_arrays=bias, headers=headers)
if output is not None:
header = primary_header
header.extend(deimos_cards(bias.shape))
hdu = fits.PrimaryHDU(data=bias,
header=header)
hdu.writeto(output, clobber=True)
return bias
def combine(do_cti=False, doreduce=True, doshifts=True):
if do_cti:
os.system('stis_cti --crds_update')
if doreduce:
# Defringing didn't seem to converge because of the low S/N
stistools.ocrreject.ocrreject('oc0102070_flc.fits',
'oc0102070_crc.fits')
iraf.normspflat(inflat='oc0102070_crc.fits',
outflat='oc0102070_nsp.fits',
do_cal='no')
iraf.imcalc(input='oc0102070_nsp.fits',
output='temp_nsp.fits',
equals='if(x .lt. 250) then 1 else im1')
iraf.imcopy('temp_nsp.fits[1][1:250,*]',
'oc0102070_nsp.fits[1][1:250,*]')
#iraf.defringe('oc0102060_flc.fits', 'oc0102070_nsp.fits', 'oc0102060_dfr.fits')
#for each image
for im in ['oc0102050_flc', 'oc0102060_flc']:
outbase = 'blue'
if im[:-4] == 'oc0102060':
outbase = 'red'
#reset the aperture table to the newer file (we maybe should check this)
pyfits.setval(im + '.fits',
'APERTAB',
value='oref$y2r1559to_apt.fits')
pyfits.setval(im + '.fits',
'SPTRCTAB',
value='oref$qa31608go_1dt.fits')
# fixpix any negative values. In principle some of this noise
# could be real, but I have found that is often not the case
hdu = fits.open(im + '.fits')
mask1 = hdu[1].data < -20
mask2 = hdu[4].data < -20
hdu.close()
fits.writeto(outbase + 'mask1.fits',
mask1.astype('i'),
clobber=True)
fits.writeto(outbase + 'mask2.fits',
mask2.astype('i'),
clobber=True)
iraf.unlearn(iraf.fixpix)
iraf.fixpix(im + '[1]', outbase + 'mask1.fits')
iraf.unlearn(iraf.fixpix)
iraf.fixpix(im + '[4]', outbase + 'mask2.fits')
# Subtract off the median value
hdu = fits.open(im + '.fits', mode='update')
hdu[1].data -= np.median(hdu[1].data)
hdu[4].data -= np.median(hdu[4].data)
readnoise1 = 1.4826 * np.median(np.abs(hdu[1].data))
readnoise2 = 1.4826 * np.median(np.abs(hdu[4].data))
# Cosmic ray reject both images using scrappy
# Make sure to treat the noise in a sensible way
crmask1, clean1 = detect_cosmics(hdu[1].data,
readnoise=readnoise1,
sigclip=5,
objlim=5,
sigfrac=0.8,
fsmode='median',
psfmodel='gaussy',
psffwhm=2.,
cleantype='idw')
crmask2, clean2 = detect_cosmics(hdu[4].data,
readnoise=readnoise2,
sigclip=5,
objlim=5,
sigfrac=0.8,
fsmode='median',
psfmodel='gaussy',
psffwhm=2.,
cleantype='idw')
hdu.flush()
hdu.close()
fits.writeto(outbase + '_crmask1.fits',
crmask1.astype('i'),
clobber=True)
fits.writeto(outbase + '_crmask2.fits',
crmask2.astype('i'),
clobber=True)
# Run fixpix on the frames
iraf.unlearn(iraf.fixpix)
iraf.fixpix(im + '[1]', outbase + '_crmask1.fits')
iraf.unlearn(iraf.fixpix)
iraf.fixpix(im + '[4]', outbase + '_crmask2.fits')
if outbase == 'red':
iraf.mkfringeflat('oc0102060_flc.fits',
'oc0102070_nsp.fits',
'oc0102070_frr.fits',
beg_scale=0.6,
end_scale=1.5,
scale_step=0.01,
beg_shift=-3.0,
end_shift=3.0,
shift_step=0.05)
iraf.defringe('oc0102060_flc.fits', 'oc0102070_frr.fits',
'oc0102060_dfr.fits')
#Run x2d on the flt frame
stistools.x2d.x2d(input='oc0102060_dfr.fits',
output=im[:-4] + 'x2d.fits')
else:
stistools.x2d.x2d(input='oc0102050_flc.fits',
output=im[:-4] + 'x2d.fits')
h = pyfits.open(im[:-4] + 'x2d.fits', mode='update')
#Replace all of the bad pixels in the image by -666 based on the DQ array
#save them to a new file
#Throw away bad reference file pixels and saturated pixels. None of the other error codes
#were in the first file so I haven't included them here, but we might want to
d = h[3].data
badpix = logical_and(
bitwise_and(d, 256) == 256,
bitwise_and(d, 512) == 512)
h[1].data[badpix] = -10000
d = h[6].data
badpix = logical_and(
bitwise_and(d, 256) == 256,
bitwise_and(d, 512) == 512)
h[4].data[badpix] = -10000
h.flush()
# Trim the images
for i in range(1, 7):
h[i].data = h[i].data[100:-100, 100:-100]
h[i].header['CRPIX1'] -= 100
h[i].header['CRPIX2'] -= 100
h.flush()
h.close()
# Combine the images
iraf.unlearn(iraf.imcombine)
iraf.imcombine(input=im[:-4] + 'x2d[1],' + im[:-4] + 'x2d[4]',
output=outbase + '_com.fits',
reject='crreject')
hdu = pyfits.open(outbase + '_com.fits')
mask = hdu[0].data == 0.0
hdu.close()
fits.writeto(outbase + '_mask.fits',
mask.astype('i'),
clobber=True)
iraf.unlearn(iraf.fixpix)
iraf.fixpix(outbase + '_com.fits', outbase + '_mask.fits')
iraf.unlearn(iraf.apall)
iraf.apall(input=outbase + '_com',
output='13dh_' + outbase,
review='no',
nsum=-500,
b_order=1,
b_function='legendre',
b_niterate=30,
b_naverage=-21,
nfind=1,
t_order=3,
background='median',
weights='variance',
skybox=100)
iraf.splot(outbase + '[SCI]')
sigclip = 3
elif fitsfile[0].header['HIERARCH ESO SEQ ARM'] == 'VIS':
objlim = 7
sigclip = 3
niter = 4
elif fitsfile[0].header['HIERARCH ESO SEQ ARM'] == 'NIR':
gain = 2.12
ron = 8
frac = 0.0001
objlim = 45
sigclip = 30
niter = 10
# try:
# crmask, clean_arr = astroscrappy.detect_cosmics(fitsfile[0].data, inmask = fitsfile[2].data.astype("bool"))
# except:
crmask, clean_arr = astroscrappy.detect_cosmics(fitsfile[0].data, sigclip=sigclip, sigfrac=frac, objlim=objlim, cleantype='medmask', niter=niter, sepmed=True, verbose=True)
# Replace data array with cleaned image
fitsfile[0].data = clean_arr
# Try to retain info of corrected pixel if extension is present.
try:
fitsfile[2].data[crmask] = 16 #Flag value for removed cosmic ray
except:
print("No bad-pixel extension present. No flag set for corrected pixels")
# Update file
fitsfile.writeto(n[:-5]+"cosmicced.fits", output_verify='fix')
# Moving original file
dirname = objectname+"/backup"
def _rem_cr_array(data, err, index, **kwargs):
for i in range(data.shape[0]):
mask = err[i] == missing
data[i] = asc.detect_cosmics(data[i], inmask=mask, **kwargs)[1]
del mask
def combine(do_cti=False, doreduce=True, doshifts=True):
if do_cti:
os.system('stis_cti --crds_update')
if doreduce:
# Defringing didn't seem to converge because of the low S/N
stistools.ocrreject.ocrreject('oc0102070_flc.fits','oc0102070_crc.fits')
iraf.normspflat(inflat='oc0102070_crc.fits',outflat='oc0102070_nsp.fits', do_cal='no')
iraf.imcalc(input='oc0102070_nsp.fits', output='temp_nsp.fits', equals='if(x .lt. 250) then 1 else im1')
iraf.imcopy('temp_nsp.fits[1][1:250,*]', 'oc0102070_nsp.fits[1][1:250,*]')
#iraf.defringe('oc0102060_flc.fits', 'oc0102070_nsp.fits', 'oc0102060_dfr.fits')
#for each image
for im in ['oc0102050_flc','oc0102060_flc']:
outbase = 'blue'
if im[:-4] == 'oc0102060':
outbase = 'red'
#reset the aperture table to the newer file (we maybe should check this)
pyfits.setval(im +'.fits','APERTAB',value='oref$y2r1559to_apt.fits')
pyfits.setval(im +'.fits', 'SPTRCTAB', value='oref$qa31608go_1dt.fits')
# fixpix any negative values. In principle some of this noise
# could be real, but I have found that is often not the case
hdu = fits.open(im+ '.fits')
mask1 = hdu[1].data < -20
mask2 = hdu[4].data < -20
hdu.close()
fits.writeto(outbase+'mask1.fits', mask1.astype('i'), clobber=True)
fits.writeto(outbase+'mask2.fits', mask2.astype('i'), clobber=True)
iraf.unlearn(iraf.fixpix)
iraf.fixpix(im+'[1]', outbase+'mask1.fits')
iraf.unlearn(iraf.fixpix)
iraf.fixpix(im+'[4]', outbase+'mask2.fits')
# Subtract off the median value
hdu = fits.open(im+ '.fits', mode='update')
hdu[1].data -= np.median(hdu[1].data)
hdu[4].data -= np.median(hdu[4].data)
readnoise1 = 1.4826 * np.median(np.abs(hdu[1].data))
readnoise2 = 1.4826 * np.median(np.abs(hdu[4].data))
# Cosmic ray reject both images using scrappy
# Make sure to treat the noise in a sensible way
crmask1, clean1 = detect_cosmics(hdu[1].data, readnoise=readnoise1,
sigclip=5, objlim=5, sigfrac=0.8,
fsmode='median', psfmodel='gaussy',
psffwhm=2., cleantype='idw')
crmask2, clean2 = detect_cosmics(hdu[4].data, readnoise=readnoise2,
sigclip=5, objlim=5, sigfrac=0.8,
fsmode='median', psfmodel='gaussy',
psffwhm=2., cleantype='idw')
hdu.flush()
hdu.close()
fits.writeto(outbase + '_crmask1.fits', crmask1.astype('i'), clobber=True)
fits.writeto(outbase + '_crmask2.fits', crmask2.astype('i'), clobber=True)
# Run fixpix on the frames
iraf.unlearn(iraf.fixpix)
iraf.fixpix(im+'[1]', outbase+'_crmask1.fits')
iraf.unlearn(iraf.fixpix)
iraf.fixpix(im+'[4]', outbase+'_crmask2.fits')
if outbase=='red':
iraf.mkfringeflat('oc0102060_flc.fits', 'oc0102070_nsp.fits', 'oc0102070_frr.fits',
beg_scale=0.6, end_scale=1.5, scale_step=0.01,
beg_shift=-3.0, end_shift=3.0,shift_step=0.05)
iraf.defringe('oc0102060_flc.fits', 'oc0102070_frr.fits', 'oc0102060_dfr.fits')
#Run x2d on the flt frame
stistools.x2d.x2d(input='oc0102060_dfr.fits',output=im[:-4]+'x2d.fits')
else:
stistools.x2d.x2d(input='oc0102050_flc.fits',output=im[:-4]+'x2d.fits')
h = pyfits.open(im[:-4]+'x2d.fits', mode='update')
#Replace all of the bad pixels in the image by -666 based on the DQ array
#save them to a new file
#Throw away bad reference file pixels and saturated pixels. None of the other error codes
#were in the first file so I haven't included them here, but we might want to
d = h[3].data
badpix = logical_and(bitwise_and(d,256) == 256,bitwise_and(d,512) == 512)
h[1].data[badpix] = -10000
d = h[6].data
badpix = logical_and(bitwise_and(d,256) == 256,bitwise_and(d,512) == 512)
h[4].data[badpix] = -10000
h.flush()
# Trim the images
for i in range(1,7):
h[i].data = h[i].data[100:-100, 100:-100]
h[i].header['CRPIX1'] -= 100
h[i].header['CRPIX2'] -= 100
h.flush()
h.close()
# Combine the images
iraf.unlearn(iraf.imcombine)
iraf.imcombine(input=im[:-4]+'x2d[1],'+im[:-4]+'x2d[4]', output=outbase+'_com.fits',
reject='crreject')
hdu = pyfits.open(outbase +'_com.fits')
mask = hdu[0].data == 0.0
hdu.close()
fits.writeto(outbase+'_mask.fits', mask.astype('i'), clobber=True)
iraf.unlearn(iraf.fixpix)
iraf.fixpix(outbase+'_com.fits', outbase+'_mask.fits')
iraf.unlearn(iraf.apall)
iraf.apall(input=outbase+'_com',output='13dh_'+outbase, review='no',
nsum = -500, b_order = 1,
b_function='legendre',b_niterate=30, b_naverage = -21,
nfind=1,t_order=3,background='median',weights='variance',
skybox=100 )
iraf.splot(outbase+'[SCI]')
def process_file(filename,
night=None,
site=None,
fram=None,
verbose=False,
replace=False,
base='photometry'):
if not posixpath.exists(filename):
return None
if site is None:
# Simple heuristics to derive the site name
for _ in ['auger', 'cta-n', 'cta-s0', 'cta-s1']:
if _ in filename:
site = _
break
# Rough but fast checking of whether the file is already processed
if not replace and posixpath.exists(
posixpath.splitext(
posixpath.join(base, site, '/'.join(filename.split('/')[-4:])))
[0] + '.cat'):
return
header = fits.getheader(filename)
if header['IMAGETYP'] != 'object':
return
ccd = header.get('CCD_NAME')
fname = header.get('FILTER', 'unknown')
time = parse_iso_time(header['DATE-OBS'])
target = header.get('TARGET', -1)
if fname not in ['B', 'V', 'R', 'I', 'z', 'N']:
return
if fname == 'N' and site == 'cta-n':
effective_fname = 'R'
else:
effective_fname = fname
if night is None:
if header.get('LONGITUD') is not None:
night = get_night(time, lon=header['LONGITUD'])
else:
night = get_night(time, site=site)
dirname = '%s/%s/%s/%05d/%s' % (base, site, night, target, ccd)
basename = posixpath.splitext(posixpath.split(filename)[-1])[0]
basename = dirname + '/' + basename
catname = basename + '.cat'
if not replace and posixpath.exists(catname):
return
if verbose:
print(filename, site, night, ccd, fname, effective_fname)
image = fits.getdata(filename).astype(np.double)
if fram is None:
fram = Fram()
# Basic calibration
darkname = fram.find_image('masterdark', header=header, debug=False)
flatname = fram.find_image('masterflat', header=header, debug=False)
if darkname:
dark = fits.getdata(darkname)
else:
dcname = fram.find_image('dcurrent', header=header, debug=False)
biasname = fram.find_image('bias', header=header, debug=False)
if dcname and biasname:
bias = fits.getdata(biasname)
dc = fits.getdata(dcname)
dark = bias + header['EXPOSURE'] * dc
else:
dark = None
if flatname:
flat = fits.getdata(flatname)
else:
flat = None
if dark is None or flat is None:
survey.save_objects(catname, None)
return
image, header = calibrate.calibrate(image, header, dark=dark)
image0 = image.copy()
image *= np.median(flat) / flat
# Basic masking
mask = image > 50000
mask |= dark > np.median(dark) + 10.0 * np.std(dark)
cmask = np.zeros_like(mask)
# WCS + catalogue
wcs = WCS(header)
pixscale = np.hypot(wcs.pixel_scale_matrix[0, 0],
wcs.pixel_scale_matrix[0, 1])
gain = header.get('GAIN', 1.0)
if gain > 100:
gain /= 1000
ra0, dec0, sr0 = survey.get_frame_center(header=header)
if 'WF' in header['CCD_NAME']:
if header['CCD_NAME'] in ['WF6', 'WF7', 'WF8']:
cat = fram.get_stars(
ra0,
dec0,
sr0,
limit=100000,
catalog='gaia',
extra=['g<15', 'good=1 and var=0 and multi_30=0'])
else:
cat = fram.get_stars(
ra0,
dec0,
sr0,
limit=100000,
catalog='gaia',
extra=['g<15', 'good=1 and var=0 and multi_70=0'])
else:
cat = fram.get_stars(ra0,
dec0,
sr0,
catalog='atlas',
extra=[],
limit=100000)
# Cosmic rays
if not 'WF' in header['CCD_NAME']:
obj0 = survey.get_objects_sep(image,
mask=mask,
wcs=wcs,
minnthresh=3,
edge=10,
use_fwhm=True,
sn=10,
verbose=False)
cmask, cimage = astroscrappy.detect_cosmics(image0,
inmask=mask,
gain=gain,
readnoise=10,
psffwhm=np.median(
obj0['fwhm']),
satlevel=50000,
verbose=False)
cimage /= gain
# Object extraction
if ccd == 'C0':
obj = survey.get_objects_sep(image,
mask=mask | cmask,
wcs=wcs,
edge=10,
aper=5,
verbose=False,
sn=5)
else:
obj = survey.get_objects_sextractor(image,
mask=mask | cmask,
wcs=wcs,
gain=gain,
edge=10,
aper=3.0,
minarea=3.0,
r0=0,
sn=3,
verbose=False,
_tmpdir='tmp/',
extra_params=['FLUX_MAX'])
# Match with catalogue
match = Match(width=image.shape[1], height=image.shape[0])
sr = pixscale * np.median(obj['fwhm'])
if not match.match(obj=obj,
cat=cat,
sr=sr,
filter_name=effective_fname,
order=0,
bg_order=None,
color_order=None,
verbose=False) or match.ngoodstars < 10:
# if verbose:
# print(match.ngoodstars, 'good matches, retrying without spatial term')
# if not match.match(obj=obj, cat=cat, sr=sr, filter_name=effective_fname, order=0, bg_order=None, color_order=None, verbose=False) or match.ngoodstars < 10:
if verbose:
print('Matching failed for', filename, ':', match.ngoodstars,
'good matches')
survey.save_objects(catname, None)
return
if verbose:
print(match.ngoodstars, 'good matches, std =', match.std)
# Store results
try:
os.makedirs(dirname)
except:
pass
obj['mag_limit'] = match.mag_limit
obj['color_term'] = match.color_term
obj['filename'] = filename
obj['site'] = site
obj['night'] = night
obj['ccd'] = ccd
obj['filter'] = fname
obj['cat_filter'] = effective_fname
obj['time'] = time
obj['mag_id'] = match.mag_id
obj['good_idx'] = match.good_idx
obj['calib_mag'] = match.mag
obj['calib_magerr'] = match.magerr
obj['std'] = match.std
obj['nstars'] = match.ngoodstars
survey.save_objects(catname, obj, header=header)
def __cosmics(ccddata):
# ccddata: CCDData instance
ccddata.data = detect_cosmics(ccddata.data)[1]
def cosmicrays(infile, sigclip=5.0, sigfrac=0.2, objlim=2.0, niter = 4, \
overwrite=False):
print('\n#############################')
print('Cosmicray removing.')
inhdl = fits.open(infile)
inhdr = inhdl[0].header
scidata = inhdl[0].data
basename = inhdr['FRAMEID']
crname = basename + '.cr.fits'
maskname = basename + '.mask.fits'
if not fi.check_version(inhdl):
inhdl.close()
return crname, maskname, False
inhdl.close()
if os.path.isfile(crname):
if not overwrite:
print('\t Cosmicray-removed frame already exits. ' + crname)
print('\t This procedure is skipped.')
inhdl.close()
return crname, maskname, True
print('Cosmicray removing for ' + str(inhdr['FRAMEID']))
nx = scidata.shape[1]
ny = scidata.shape[0]
# Cosmicray removing
print('\t L.A.Cosmic')
#fitdata = scidata - residualdata
crmask, cleandata = \
astroscrappy.detect_cosmics(scidata, sigclip=sigclip, sigfrac=sigfrac, \
objlim=objlim, gain=1.0, readnoise=4.0, satlevel=np.inf, \
pssl=0.0, niter=niter, sepmed=False, cleantype='medmask', \
fsmode='median', verbose=True)
# Writing output files
cleanhdu = fits.PrimaryHDU(data=cleandata)
#cleanhdu = fits.PrimaryHDU(data=cleandata)
cleanhdl = fits.HDUList([cleanhdu])
cleanhdl[0].header = inhdr
cleanhdl[0].header['LACO_VER'] = (astroscrappy.__version__, \
'Python script version of LACOSMIC')
crmaskint = crmask.astype(np.uint8)
maskhdu = fits.PrimaryHDU(data=crmaskint)
#maskhdu = fits.PrimaryHDU(data=fitdata)
maskhdl = fits.HDUList([maskhdu])
maskhdl[0].header = inhdr
maskhdl[0].header['LACO_VER'] = (astroscrappy.__version__, \
'Python script version of LACOSMIC')
cleanhdl.writeto(crname, overwrite=overwrite)
maskhdl.writeto(maskname, overwrite=overwrite)
cleanhdl.close()
maskhdl.close()
print('\t Cleaned image ' + crname)
print('\t Mask image: ' + maskname)
return crname, maskname, True
def bdf_process(ccd,
output=None,
mbiaspath=None,
mdarkpath=None,
mflatpath=None,
trim_fits_section=None,
calc_err=False,
unit='adu',
gain=None,
gain_key="GAIN",
gain_unit=u.electron / u.adu,
rdnoise=None,
rdnoise_key="RDNOISE",
rdnoise_unit=u.electron,
dark_exposure=None,
data_exposure=None,
exposure_key="EXPTIME",
exposure_unit=u.s,
dark_scale=False,
normalize_exposure=False,
normalize_average=False,
flat_min_value=None,
flat_norm_value=None,
do_crrej=False,
crrej_kwargs=None,
propagate_crmask=False,
verbose_crrej=False,
verbose_bdf=True,
output_verify='fix',
overwrite=True,
dtype="float32",
uncertainty_dtype="float32"):
''' Do bias, dark and flat process.
Parameters
----------
ccd: CCDData
The ccd to be processed.
output : path-like or None, optional.
The path if you want to save the resulting ``ccd`` object.
Default is ``None``.
mbiaspath, mdarkpath, mflatpath : path-like, optional.
The path to master bias, dark, flat FITS files. If ``None``, the
corresponding process is not done.
trim_fits_section: str, optional
Region of ``ccd`` to be trimmed; see
``ccdproc.subtract_overscan`` for details. Default is ``None``.
calc_err : bool, optional.
Whether to calculate the error map based on Poisson and readnoise
error propagation.
unit : `~astropy.units.Unit` or str, optional.
The units of the data.
Default is ``'adu'``.
gain, rdnoise : None, float, optional
The gain and readnoise value. These are all ignored if
``calc_err=False``. If ``calc_err=True``, it automatically seeks
for suitable gain and readnoise value. If ``gain`` or
``readnoise`` is specified, they are interpreted with
``gain_unit`` and ``rdnoise_unit``, respectively. If they are
not specified, this function will seek for the header with
keywords of ``gain_key`` and ``rdnoise_key``, and interprete the
header value in the unit of ``gain_unit`` and ``rdnoise_unit``,
respectively.
gain_key, rdnoise_key : str, optional
See ``gain``, ``rdnoise`` explanation above.
These are all ignored if ``calc_err=False``.
gain_unit, rdnoise_unit : astropy Unit, optional
See ``gain``, ``rdnoise`` explanation above.
These are all ignored if ``calc_err=False``.
dark_exposure, data_exposure : None, float, astropy Quantity, optional
The exposure times of dark and data frame, respectively. They
should both be specified or both ``None``.
These are all ignored if ``mdarkpath=None``.
If both are not specified while ``mdarkpath`` is given, then the
code automatically seeks for header's ``exposure_key``. Then
interprete the value as the quantity with unit
``exposure_unit``.
If ``mdkarpath`` is not ``None``, then these are passed to
``ccdproc.subtract_dark``.
exposure_key : str, optional
The header keyword for exposure time.
Ignored if ``mdarkpath=None``.
exposure_unit : astropy Unit, optional.
The unit of the exposure time.
Ignored if ``mdarkpath=None``.
flat_min_value : float or None, optional
min_value of `ccdproc.flat_correct`.
Minimum value for flat field. The value can either be None and
no minimum value is applied to the flat or specified by a float
which will replace all values in the flat by the min_value.
Default is ``None``.
flat_norm_value : float or None, optional
norm_value of `ccdproc.flat_correct`.
If not ``None``, normalize flat field by this argument rather
than the mean of the image. This allows fixing several different
flat fields to have the same scale. If this value is negative or
0, a ``ValueError``
is raised. Default is ``None``.
crrej_kwargs : dict or None, optional
If ``None`` (default), uses some default values defined in
``~.misc.LACOSMIC_KEYS``. It is always discouraged to use
default except for quick validity-checking, because even the
official L.A. Cosmic codes in different versions (IRAF, IDL,
Python, etc) have different default parameters, i.e., there is
nothing which can be regarded as default.
To see all possible keywords, do
``print(astroscrappy.detect_cosmics.__doc__)``
Also refer to
https://nbviewer.jupyter.org/github/ysbach/AO2019/blob/master/Notebooks/07-Cosmic_Ray_Rejection.ipynb
propagate_crmask : bool, optional
Whether to save (propagate) the mask from CR rejection
(``astroscrappy``) to the CCD's mask.
Default is ``False``.
output_verify : str
Output verification option. Must be one of ``"fix"``,
``"silentfix"``, ``"ignore"``, ``"warn"``, or ``"exception"``.
May also be any combination of ``"fix"`` or ``"silentfix"`` with
``"+ignore"``, ``+warn``, or ``+exception" (e.g.
``"fix+warn"``). See the astropy documentation below:
http://docs.astropy.org/en/stable/io/fits/api/verification.html#verify
dtype : str or `numpy.dtype` or None, optional
Allows user to set dtype. See `numpy.array` ``dtype`` parameter
description. If ``None`` it uses ``np.float64``.
Default is ``None``.
'''
def _add_and_print(s, header, verbose):
header.add_history(s)
if verbose:
print(s)
# Set strings for header history & print (if verbose_bdf)
str_bias = "Bias subtracted using {}"
str_dark = "Dark subtracted using {}"
str_dscale = "Dark scaling {} using {}"
str_flat = "Flat corrected using {}"
str_trim = "Trim by FITS section {}"
str_grd = "From {}, {} = {:.3f} [{}]"
# str_grd.format(user/header_key, gain/rdnoise, val, unit)
str_e0 = ("Readnoise propagated with Poisson noise (using gain above)" +
" of source.")
str_ed = "Poisson noise from subtracted dark was propagated."
str_ef = "Flat uncertainty was propagated."
str_nexp = "Normalized by the exposure time."
str_navg = "Normalized by the average value of the frame."
str_cr = ("Cosmic-Ray rejected by astroscrappy (v {}), " +
"with parameters: {}")
# Initial setting
proc = CCDData(ccd)
hdr_new = proc.header
# Add PROCESS key
try:
_ = hdr_new["PROCESS"]
except KeyError:
hdr_new["PROCESS"] = ("", "The processed history: see comment.")
hdr_new["PROCVER"] = (ccdproc.__version__,
"ccdproc version used for processing.")
hdr_new.add_comment("PROCESS key can be B (bias), D (dark), F (flat), " +
"T (trim), W (WCS astrometry), C(CRrej).")
# Set for BIAS
if mbiaspath is None:
do_bias = False
mbias = CCDData(np.zeros_like(ccd), unit=proc.unit)
else:
do_bias = True
mbias = CCDData.read(mbiaspath, unit=unit)
hdr_new["PROCESS"] += "B"
_add_and_print(str_bias.format(mbiaspath), hdr_new, verbose_bdf)
# Set for DARK
if mdarkpath is None:
do_dark = False
mdark = CCDData(np.zeros_like(ccd), unit=proc.unit)
else:
do_dark = True
mdark = CCDData.read(mdarkpath, unit=unit)
hdr_new["PROCESS"] += "D"
_add_and_print(str_dark.format(mdarkpath), hdr_new, verbose_bdf)
if dark_scale:
_add_and_print(str_dscale.format(dark_scale, exposure_key),
hdr_new, verbose_bdf)
# Set for FLAT
if mflatpath is None:
do_flat = False
mflat = CCDData(np.ones_like(ccd), unit=proc.unit)
else:
do_flat = True
mflat = CCDData.read(mflatpath)
hdr_new["PROCESS"] += "F"
_add_and_print(str_flat.format(mflatpath), hdr_new, verbose_bdf)
# Set gain and rdnoise if at least one of calc_err and do_crrej is True.
if calc_err or do_crrej:
if gain is None:
gain_Q = get_from_header(hdr_new,
gain_key,
unit=gain_unit,
verbose=False,
default=1.)
gain_from = gain_key
else:
if not isinstance(gain, u.Quantity):
gain_Q = gain * gain_unit
else:
gain_Q = gain
gain_from = "User"
_add_and_print(
str_grd.format(gain_from, "gain", gain_Q.value, gain_Q.unit),
hdr_new, verbose_bdf)
if rdnoise is None:
rdnoise_Q = get_from_header(hdr_new,
rdnoise_key,
unit=rdnoise_unit,
verbose=False,
default=1.)
rdnoise_from = rdnoise_key
else:
if not isinstance(rdnoise, u.Quantity):
rdnoise_Q = rdnoise * rdnoise_unit
else:
rdnoise_Q = rdnoise
rdnoise_from = "User"
_add_and_print(
str_grd.format(rdnoise_from, "rdnoise", rdnoise_Q.value,
rdnoise_Q.unit), hdr_new, verbose_bdf)
# Do TRIM
if trim_fits_section is not None:
proc = trim_image(proc, trim_fits_section)
mbias = trim_image(mbias, trim_fits_section)
mdark = trim_image(mdark, trim_fits_section)
mflat = trim_image(mflat, trim_fits_section)
hdr_new["PROCESS"] += "T"
_add_and_print(str_trim.format(trim_fits_section), hdr_new,
verbose_bdf)
# Do BIAS
if do_bias:
proc = subtract_bias(proc, mbias)
# Do DARK
if do_dark:
proc = subtract_dark(proc,
mdark,
dark_exposure=dark_exposure,
data_exposure=data_exposure,
exposure_time=exposure_key,
exposure_unit=exposure_unit,
scale=dark_scale)
# Make UNCERT extension before doing FLAT
# It is better to make_errmap a priori because of mathematical and
# computational convenience. See ``if do_flat:`` clause below.
if calc_err:
err = make_errmap(proc, gain_epadu=gain, subtracted_dark=mdark)
proc.uncertainty = StdDevUncertainty(err)
_add_and_print(str_e0, hdr_new, verbose_bdf)
if do_dark:
_add_and_print(str_ed, hdr_new, verbose_bdf)
# Do FLAT
if do_flat:
# Flat error propagation is done automatically by
# ``ccdproc.flat_correct``if it has the uncertainty attribute.
proc = flat_correct(proc,
mflat,
min_value=flat_min_value,
norm_value=flat_norm_value)
if calc_err and mflat.uncertainty is not None:
_add_and_print(str_ef, hdr_new, verbose_bdf)
# Normalize by the exposure time (e.g., ADU per sec)
if normalize_exposure:
if data_exposure is None:
data_exposure = hdr_new[exposure_key]
proc = proc.divide(data_exposure) # uncertainty will also be..
_add_and_print(str_nexp, hdr_new, verbose_bdf)
# Normalize by the mean value
if normalize_average:
avg = np.mean(proc.data)
proc = proc.divide(avg)
_add_and_print(str_navg, hdr_new, verbose_bdf)
# Do CRREJ
if do_crrej:
import astroscrappy
from astroscrappy import detect_cosmics
from .misc import LACOSMIC_KEYS
if crrej_kwargs is None:
crrej_kwargs = LACOSMIC_KEYS
warn("You are not specifying CR-rejection parameters and blindly" +
" using defaults. It can be dangerous.")
if (("B" in hdr_new["PROCESS"]) + ("D" in hdr_new["PROCESS"]) +
("F" in hdr_new["PROCESS"])) < 2:
warn("L.A. Cosmic should be run AFTER B/D/F process. " +
f"You are running it with {hdr_new['PROCESS']}. " +
"See http://www.astro.yale.edu/dokkum/lacosmic/notes.html")
# remove the fucxing cosmic rays
crmask, cleanarr = detect_cosmics(proc.data,
inmask=proc.mask,
gain=gain_Q.value,
readnoise=rdnoise_Q.value,
**crrej_kwargs,
verbose=verbose_crrej)
# create the new ccd data object
# astroscrappy automatically does the gain correction, so return
# back to avoid confusion.
proc.data = cleanarr / gain_Q.value
if propagate_crmask:
if proc.mask is None:
proc.mask = crmask
else:
proc.mask = proc.mask + crmask
hdr_new["PROCESS"] += "C"
_add_and_print(str_cr.format(astroscrappy.__version__, crrej_kwargs),
hdr_new, verbose_crrej)
proc = CCDData_astype(proc,
dtype=dtype,
uncertainty_dtype=uncertainty_dtype)
proc.header = hdr_new
if output is not None:
if verbose_bdf:
print(f"Writing FITS to {output}... ", end='')
proc.write(output, output_verify=output_verify, overwrite=overwrite)
if verbose_bdf:
print(f"Saved.")
return proc
if (uinst_filt[i].split('/')[0] == 'WFC3'):
ext_list = [1, 3]
for k in np.arange(len(ext_list)):
if (k % 2 == 0):
print("Writing " + 'f' + ufilt[i][1:4] + '_%02d_sci%1d.fits' %
(j + 1, 1 + k // 2) + "...")
sci, hdr = fits.getdata(str(np.array(img)[order][j]),
ext=ext_list[k],
header=True)
# dq = fits.getdata(str(np.array(img)[order][j]), ext=ext_list[k+1], header=False)
# cr = (dq >= ip.cr_thre)
# if (np.sum(cr) >= 1):
# sci[cr] = ip.cr_mask
epadu = hd0['CCDGAIN']
crmask, cleanarr = detect_cosmics(sci,
gain=epadu,
cleantype='medmask')
fits.writeto('f' + ufilt[i][1:4] + '_%02d_sci%1d.fits' %
(j + 1, 1 + k // 2),
cleanarr / epadu,
hdr,
overwrite=True)
f.write(str(np.array(img)[order][j]))
f.write('\t')
f.write('f' + ufilt[i][1:4] + '_%02d_sci%1d.fits' %
(j + 1, 1 + k // 2))
f.write('\n')
else:
continue
f.close()
def remove_cr(data):
'''
Removes high value pixels which are presumed to be cosmic ray hits.
'''
m, imdata = detect_cosmics(data, readnoise=20., gain=1.4, sigclip=5., sigfrac=.5, objlim=6.)
return imdata
def cosmicray_lacosmic(ccd, sigclip=4.5, sigfrac=0.3,
objlim=5.0, gain=1.0, readnoise=6.5,
satlevel=65535.0, pssl=0.0, niter=4,
sepmed=True, cleantype='meanmask', fsmode='median',
psfmodel='gauss', psffwhm=2.5, psfsize=7,
psfk=None, psfbeta=4.765, verbose=False):
r"""
Identify cosmic rays through the lacosmic technique. The lacosmic technique
identifies cosmic rays by identifying pixels based on a variation of the
Laplacian edge detection. The algorithm is an implementation of the
code describe in van Dokkum (2001) [1]_ as implemented by McCully (2014)
[2]_. If you use this algorithm, please cite these two works.
Parameters
----------
ccd : `~ccdproc.CCDData` or `numpy.ndarray`
Data to have cosmic ray cleaned.
sigclip : float, optional
Laplacian-to-noise limit for cosmic ray detection. Lower values will
flag more pixels as cosmic rays. Default: 4.5.
sigfrac : float, optional
Fractional detection limit for neighboring pixels. For cosmic ray
neighbor pixels, a lapacian-to-noise detection limit of
sigfrac * sigclip will be used. Default: 0.3.
objlim : float, optional
Minimum contrast between Laplacian image and the fine structure image.
Increase this value if cores of bright stars are flagged as cosmic
rays. Default: 5.0.
pssl : float, optional
Previously subtracted sky level in ADU. We always need to work in
electrons for cosmic ray detection, so we need to know the sky level
that has been subtracted so we can add it back in. Default: 0.0.
gain : float, optional
Gain of the image (electrons / ADU). We always need to work in
electrons for cosmic ray detection. Default: 1.0
readnoise : float, optional
Read noise of the image (electrons). Used to generate the noise model
of the image. Default: 6.5.
satlevel : float, optional
Saturation level of the image (electrons). This value is used to
detect saturated stars and pixels at or above this level are added to
the mask. Default: 65535.0.
niter : int, optional
Number of iterations of the LA Cosmic algorithm to perform. Default: 4.
sepmed : bool, optional
Use the separable median filter instead of the full median filter.
The separable median is not identical to the full median filter, but
they are approximately the same and the separable median filter is
significantly faster and still detects cosmic rays well. Default: True
cleantype : str, optional
Set which clean algorithm is used:
- ``"median"``: An umasked 5x5 median filter.
- ``"medmask"``: A masked 5x5 median filter.
- ``"meanmask"``: A masked 5x5 mean filter.
- ``"idw"``: A masked 5x5 inverse distance weighted interpolation.
Default: ``"meanmask"``.
fsmode : str, optional
Method to build the fine structure image:
- ``"median"``: Use the median filter in the standard LA Cosmic \
algorithm.
- ``"convolve"``: Convolve the image with the psf kernel to calculate \
the fine structure image.
Default: ``"median"``.
psfmodel : str, optional
Model to use to generate the psf kernel if fsmode == 'convolve' and
psfk is None. The current choices are Gaussian and Moffat profiles:
- ``"gauss"`` and ``"moffat"`` produce circular PSF kernels.
- The ``"gaussx"`` and ``"gaussy"`` produce Gaussian kernels in the x \
and y directions respectively.
Default: ``"gauss"``.
psffwhm : float, optional
Full Width Half Maximum of the PSF to use to generate the kernel.
Default: 2.5.
psfsize : int, optional
Size of the kernel to calculate. Returned kernel will have size
psfsize x psfsize. psfsize should be odd. Default: 7.
psfk : `numpy.ndarray` (with float dtype) or None, optional
PSF kernel array to use for the fine structure image if
``fsmode == 'convolve'``. If None and ``fsmode == 'convolve'``, we
calculate the psf kernel using ``psfmodel``. Default: None.
psfbeta : float, optional
Moffat beta parameter. Only used if ``fsmode=='convolve'`` and
``psfmodel=='moffat'``. Default: 4.765.
verbose : bool, optional
Print to the screen or not. Default: False.
Notes
-----
Implementation of the cosmic ray identification L.A.Cosmic:
http://www.astro.yale.edu/dokkum/lacosmic/
Returns
-------
nccd : `~ccdproc.CCDData` or `numpy.ndarray`
An object of the same type as ccd is returned. If it is a
`~ccdproc.CCDData`, the mask attribute will also be updated with
areas identified with cosmic rays masked.
crmask : `numpy.ndarray`
If an `numpy.ndarray` is provided as ccd, a boolean ndarray with the
cosmic rays identified will also be returned.
References
----------
.. [1] van Dokkum, P; 2001, "Cosmic-Ray Rejection by Laplacian Edge
Detection". The Publications of the Astronomical Society of the
Pacific, Volume 113, Issue 789, pp. 1420-1427.
doi: 10.1086/323894
.. [2] McCully, C., 2014, "Astro-SCRAPPY",
https://github.com/astropy/astroscrappy
Examples
--------
1) Given an numpy.ndarray object, the syntax for running
cosmicrar_lacosmic would be:
>>> newdata, mask = cosmicray_lacosmic(data, sigclip=5) #doctest: +SKIP
where the error is an array that is the same shape as data but
includes the pixel error. This would return a data array, newdata,
with the bad pixels replaced by the local median from a box of 11
pixels; and it would return a mask indicating the bad pixels.
2) Given an `~ccdproc.CCDData` object with an uncertainty frame, the syntax
for running cosmicrar_lacosmic would be:
>>> newccd = cosmicray_lacosmic(ccd, sigclip=5) # doctest: +SKIP
The newccd object will have bad pixels in its data array replace and the
mask of the object will be created if it did not previously exist or be
updated with the detected cosmic rays.
"""
from astroscrappy import detect_cosmics
if isinstance(ccd, np.ndarray):
data = ccd
crmask, cleanarr = detect_cosmics(
data, inmask=None, sigclip=sigclip,
sigfrac=sigfrac, objlim=objlim, gain=gain,
readnoise=readnoise, satlevel=satlevel, pssl=pssl,
niter=niter, sepmed=sepmed, cleantype=cleantype,
fsmode=fsmode, psfmodel=psfmodel, psffwhm=psffwhm,
psfsize=psfsize, psfk=psfk, psfbeta=psfbeta,
verbose=verbose)
return cleanarr, crmask
elif isinstance(ccd, CCDData):
crmask, cleanarr = detect_cosmics(
ccd.data, inmask=ccd.mask,
sigclip=sigclip, sigfrac=sigfrac, objlim=objlim, gain=gain,
readnoise=readnoise, satlevel=satlevel, pssl=pssl,
niter=niter, sepmed=sepmed, cleantype=cleantype,
fsmode=fsmode, psfmodel=psfmodel, psffwhm=psffwhm,
psfsize=psfsize, psfk=psfk, psfbeta=psfbeta, verbose=verbose)
# create the new ccd data object
nccd = ccd.copy()
nccd.data = cleanarr
if nccd.mask is None:
nccd.mask = crmask
else:
nccd.mask = nccd.mask + crmask
return nccd
else:
raise TypeError('ccd is not a CCDData or ndarray object.')
def cosmicray_lacosmic(ccd, sigclip=4.5, sigfrac=0.3,
objlim=5.0, gain=1.0, readnoise=6.5,
satlevel=65536.0, pssl=0.0, niter=4,
sepmed=True, cleantype='meanmask', fsmode='median',
psfmodel='gauss', psffwhm=2.5, psfsize=7,
psfk=None, psfbeta=4.765, verbose=False):
r"""
Identify cosmic rays through the lacosmic technique. The lacosmic technique
identifies cosmic rays by identifying pixels based on a variation of the
Laplacian edge detection. The algorithm is an implementation of the
code describe in van Dokkum (2001) :ref:[1]_ as implemented by McCully (2014)
[2]_. If you use this algorithm, please cite these two works.
Parameters
----------
ccd: `~ccdproc.CCDData` or `~numpy.ndarray`
Data to have cosmic ray cleaned
sigclip : float, optional
Laplacian-to-noise limit for cosmic ray detection. Lower values will
flag more pixels as cosmic rays. Default: 4.5.
sigfrac : float, optional
Fractional detection limit for neighboring pixels. For cosmic ray
neighbor pixels, a lapacian-to-noise detection limit of
sigfrac * sigclip will be used. Default: 0.3.
objlim : float, optional
Minimum contrast between Laplacian image and the fine structure image.
Increase this value if cores of bright stars are flagged as cosmic
rays. Default: 5.0.
pssl : float, optional
Previously subtracted sky level in ADU. We always need to work in
electrons for cosmic ray detection, so we need to know the sky level
that has been subtracted so we can add it back in. Default: 0.0.
gain : float, optional
Gain of the image (electrons / ADU). We always need to work in
electrons for cosmic ray detection. Default: 1.0
readnoise : float, optional
Read noise of the image (electrons). Used to generate the noise model
of the image. Default: 6.5.
satlevel : float, optional
Saturation of level of the image (electrons). This value is used to
detect saturated stars and pixels at or above this level are added to
the mask. Default: 65536.0.
niter : int, optional
Number of iterations of the LA Cosmic algorithm to perform. Default: 4.
sepmed : boolean, optional
Use the separable median filter instead of the full median filter.
The separable median is not identical to the full median filter, but
they are approximately the same and the separable median filter is
significantly faster and still detects cosmic rays well. Default: True
cleantype : {'median', 'medmask', 'meanmask', 'idw'}, optional
Set which clean algorithm is used:\n
'median': An umasked 5x5 median filter\n
'medmask': A masked 5x5 median filter\n
'meanmask': A masked 5x5 mean filter\n
'idw': A masked 5x5 inverse distance weighted interpolation\n
Default: "meanmask".
fsmode : {'median', 'convolve'}, optional
Method to build the fine structure image:\n
'median': Use the median filter in the standard LA Cosmic algorithm
'convolve': Convolve the image with the psf kernel to calculate the
fine structure image.
Default: 'median'.
psfmodel : {'gauss', 'gaussx', 'gaussy', 'moffat'}, optional
Model to use to generate the psf kernel if fsmode == 'convolve' and
psfk is None. The current choices are Gaussian and Moffat profiles.
'gauss' and 'moffat' produce circular PSF kernels. The 'gaussx' and
'gaussy' produce Gaussian kernels in the x and y directions
respectively. Default: "gauss".
psffwhm : float, optional
Full Width Half Maximum of the PSF to use to generate the kernel.
Default: 2.5.
psfsize : int, optional
Size of the kernel to calculate. Returned kernel will have size
psfsize x psfsize. psfsize should be odd. Default: 7.
psfk : float numpy array, optional
PSF kernel array to use for the fine structure image if
fsmode == 'convolve'. If None and fsmode == 'convolve', we calculate
the psf kernel using 'psfmodel'. Default: None.
psfbeta : float, optional
Moffat beta parameter. Only used if fsmode=='convolve' and
psfmodel=='moffat'. Default: 4.765.
verbose : boolean, optional
Print to the screen or not. Default: False.
{log}
Notes
-----
Implementation of the cosmic ray identification L.A.Cosmic:
http://www.astro.yale.edu/dokkum/lacosmic/
Returns
-------
nccd : `~ccdproc.CCDData` or `~numpy.ndarray`
An object of the same type as ccd is returned. If it is a
`~ccdproc.CCDData`, the mask attribute will also be updated with
areas identified with cosmic rays masked.
crmask : `~numpy.ndarray`
If an `~numpy.ndarray` is provided as ccd, a boolean ndarray with the
cosmic rays identified will also be returned.
References
----------
.. [1] van Dokkum, P; 2001, "Cosmic-Ray Rejection by Laplacian Edge
Detection". The Publications of the Astronomical Society of the
Pacific, Volume 113, Issue 789, pp. 1420-1427.
doi: 10.1086/323894
.. [2] McCully, C., 2014, "Astro-SCRAPPY",
https://github.com/astropy/astroscrappy
Examples
--------
1) Given an numpy.ndarray object, the syntax for running
cosmicrar_lacosmic would be:
>>> newdata, mask = cosmicray_lacosmic(data, sigclip=5) #doctest: +SKIP
where the error is an array that is the same shape as data but
includes the pixel error. This would return a data array, newdata,
with the bad pixels replaced by the local median from a box of 11
pixels; and it would return a mask indicating the bad pixels.
2) Given an `~ccdproc.CCDData` object with an uncertainty frame, the syntax
for running cosmicrar_lacosmic would be:
>>> newccd = cosmicray_lacosmic(ccd, sigclip=5) # doctest: +SKIP
The newccd object will have bad pixels in its data array replace and the
mask of the object will be created if it did not previously exist or be
updated with the detected cosmic rays.
"""
from astroscrappy import detect_cosmics
if isinstance(ccd, np.ndarray):
data = ccd
crmask, cleanarr = detect_cosmics(
data, inmask=None, sigclip=sigclip,
sigfrac=sigfrac, objlim=objlim, gain=gain,
readnoise=readnoise, satlevel=satlevel, pssl=pssl,
niter=niter, sepmed=sepmed, cleantype=cleantype,
fsmode=fsmode, psfmodel=psfmodel, psffwhm=psffwhm,
psfsize=psfsize, psfk=psfk, psfbeta=psfbeta,
verbose=verbose)
return cleanarr, crmask
elif isinstance(ccd, CCDData):
crmask, cleanarr = detect_cosmics(
ccd.data, inmask=ccd.mask,
sigclip=sigclip, sigfrac=sigfrac, objlim=objlim, gain=gain,
readnoise=readnoise, satlevel=satlevel, pssl=pssl,
niter=niter, sepmed=sepmed, cleantype=cleantype,
fsmode=fsmode, psfmodel=psfmodel, psffwhm=psffwhm,
psfsize=psfsize, psfk=psfk, psfbeta=psfbeta, verbose=verbose)
# create the new ccd data object
nccd = ccd.copy()
nccd.data = cleanarr
if nccd.mask is None:
nccd.mask = crmask
else:
nccd.mask = nccd.mask + crmask
return nccd
else:
raise TypeError('ccddata is not a CCDData or ndarray object')
infile = args.infile
mccd = CCDData.read(args.mccd, unit='electron')
if os.path.basename(infile).startswith('H'):
ccd = blue_process(infile, masterbias=mccd, oscan_correct=args.oscan)
elif os.path.basename(infile).startswith('R'):
ccd = red_process(infile, masterbias=mccd, oscan_correct=args.oscan)
else:
exit('Are you sure this is an HRS file?')
if args.cray:
from astroscrappy import detect_cosmics
crmask, cleanarr = detect_cosmics(ccd.data, inmask=None, sigclip=4.5, sigfrac=0.3,
objlim=5.0, gain=1.0, readnoise=6.5,
satlevel=65536.0, pssl=0.0, niter=4,
sepmed=True, cleantype='meanmask', fsmode='median',
psfmodel='gauss', psffwhm=2.5, psfsize=7,
psfk=None, psfbeta=4.765, verbose=False)
ccd.data = cleanarr
if ccd.mask == None:
ccd.mask = crmask
else:
ccd.mask = ccd.mask * crmask
if args.flat:
order_frame = CCDData.read(args.order, unit=u.adu)
flat_frame = CCDData.read(args.flat)
ccd=flatfield_science(ccd, flat_frame, order_frame, median_filter_size=None, interp=True)
outfile = 'p'+infile
ccd.write(outfile, clobber=True)
def run_astroscrappy(
filename,
psffwhm,
contrast=5,
cr_threshold=4.5,
niter=4,
inmask=None,
sigfrac=0.3,
readnoise=10,
satlevel=50000,
pssl=0.0,
sepmed=True,
cleantype='medmask',
fsmode='median',
psfmodel='gauss',
psfsize=7,
psfk=None,
psfbeta=4.765,
verbose=False,
outLevel=1
):
"""Run astroscrappy to remove cosmics"""
imagelist = np.atleast_1d(filename)
psffwhm = np.atleast_1d(psffwhm)
for i, ima in enumerate(imagelist):
path, filename_ext = os.path.split(ima)
if path:
folder = path + "/"
else:
folder = ""
filename2 = os.path.splitext(filename_ext)[0]
# Make copy of original image
if outLevel == 2:
cp_p(ima, folder + filename2 + "_CR_notcleaned.fits")
hdulist = fits.open(ima)
hdr = hdulist[0].header
saturate = np.min([satlevel, hdr.get('SATURATE', 50000)])
gain = hdr.get('GAIN', 1)
readnoise = hdr.get('RN', readnoise)
crmask, cleanarr = detect_cosmics(
hdulist[0].data,
inmask=inmask,
sigclip=cr_threshold,
sigfrac=sigfrac,
objlim=contrast,
gain=gain,
readnoise=readnoise,
satlevel=saturate,
pssl=pssl,
niter=niter,
sepmed=sepmed,
cleantype=cleantype,
fsmode=fsmode,
psfmodel=psfmodel,
psffwhm=psffwhm[i],
psfsize=psfsize,
psfk=psfk,
psfbeta=psfbeta,
verbose=verbose)
# Create image cleaned from cosmic rays
hdulist[0].data = cleanarr
hdulist.writeto(ima, overwrite=True)
if outLevel == 2:
# Create mask of cosmic rays
hdulist[0].data = np.asarray(crmask, dtype=np.int16)
hdulist.writeto(
folder +
filename2 +
"_CRmask.fits",
overwrite=True)
from astroscrappy import detect_cosmics
print('\tCosmic-ray removal...')
half_cbox_targ = inputs.cbox_size_targ // 2
inmask = np.zeros_like(image).astype(bool)
inmask[y_targ-half_cbox_targ:y_targ+half_cbox_targ, \
x_targ-half_cbox_targ:x_targ+half_cbox_targ] = True
# detect_cosmics erroneously erases the target because it is kind of
# tiny bright point. Hence, I used the inmask to protect the target region.
crmask, image_cr = detect_cosmics(image, \
inmask=inmask, \
psfmodel='moffat', \
satlevel=np.infty,\
readnoise=inputs.RONOISE,\
# sepmed=False,\
cleantype='medmask',\
# fsmode='median',\
verbose=False)
# print(detect_cosmics.__doc__)
# To reproduce the most similar behavior to the original LA Cosmic
# (written in IRAF), set inmask = None, satlevel = np.inf, sepmed=False,
# cleantype='medmask', and fsmode='median'.
# cleantype='meanmask' (default) seems to make 'holes' in CR-detected region..
# * inmask[y,x] near the target should be set, especially when the observation
# was made in non-sidereal tracking mode. The CR-rejection may misunderstand
# the target as a point-like cosmic-ray.
image_reduc = image_cr.copy()
from astropy.io import fits
import astroscrappy as ascr
import matplotlib.pyplot as plt
import numpy as np
test = fits.open('ESPRE.2019-05-23T00:17:14.397.fits')
data = test[1].data
med = np.median(data)
std = np.std(data)
mask, clean = ascr.detect_cosmics(data)
sel = np.s_[6200:7000, 800:2200]
plt.imshow(data[sel], vmin=med - std, vmax=med + std)
#plt.imshow(clean[sel], vmin=med-std, vmax=med+std)
#plt.imshow(mask[sel])
plt.show()