def remove_cosmics(sci_file, prefix='c'):
path = os.path.dirname(sci_file)
if len(path) > 0:
path += "/"
base = os.path.basename(sci_file)
new_sci_file = path + prefix + base
new_err_file = path + 'e.' + prefix + base
array, header = cosmics.fromfits(sci_file)
e_array, e_header = cosmics.fromfits(path + 'e.' + base)
# split in half so can use individual readnoise
array_l, array_u = np.split(array, 2, 1)
readnoise_u = 1.0
readnoise_l = 1.0
try: # average read noise
readnoise_u = float(header['RDNOISEU'])
readnoise_l = float(header['RDNOISEL'])
except:
pass
# Build the object :
# There are other options, check the manual...
# sigclip --- want sharp edge; noise is around 10, sources pretty much never above 120-140, so 20x on noise is safe
# sigfrac -- pixels neighboring cosmics ... because sigclip is high and cosmics can be hot (10K+) this needs to be small
# objlim -- needs to be small, real sources very faint vs cosmics (this is a contrast between cosmics and source)
cu = cosmics.cosmicsimage(array_u, gain=1.0, readnoise=readnoise_u, sigclip=25.0, sigfrac=0.001, objlim=0.001, satlevel=-1.0)
cl = cosmics.cosmicsimage(array_l, gain=1.0, readnoise=readnoise_l, sigclip=25.0, sigfrac=0.001, objlim=0.001, satlevel=-1.0)
# Run the removal
# note: sometimes get warnings about invalid values encountered. This appears due to NaN in the fits error files (noise)
cu.run(maxiter=4)
cl.run(maxiter=4)
# recombine arrays
# use the mask to make our edits
c = np.where(cl.mask == True)
for x, y in zip(c[0], c[1]):
array[x][y] = 0.0
e_array[x][y] = -1.0
x_off,y_off = array_u.shape
c = np.where(cu.mask == True)
for x, y in zip(c[0], c[1]):
array[x][y+y_off] = 0.0
e_array[x][y+y_off] = -1.0
# Write the cleaned image into a new FITS file, conserving the original header :
header['HISTORY']='removed cosmics (LACosmics)'
e_header['HISTORY'] = 'removed cosmics (LACosmics)'
cosmics.tofits(new_sci_file, array, header)
cosmics.tofits(new_err_file, e_array, e_header)
return
def clean_cosmics(fits_in, fits_clean, fits_mask, gain, readnoise, sigclip,
sigfrac, objlim, maxiter, satlevel, verbose):
t = time.time()
print '\nexecuting clean_cosmics ...'
# Read the FITS :
array, header = cosmics.fromfits(fits_in)
# array is a 2D numpy array
# Build the object :
result = cosmics.cosmicsimage(array,
gain=gain,
readnoise=readnoise,
sigclip=sigclip,
sigfrac=sigfrac,
objlim=objlim,
satlevel=satlevel,
verbose=verbose)
# Run the full artillery :
result.run(maxiter=maxiter)
# Write the cleaned and mask image to FITS files, conserving the original header:
cosmics.tofits(fits_clean, result.cleanarray, header)
cosmics.tofits(fits_mask, result.mask, header)
if timing: print 'wall-time spent in clean_cosmics', time.time() - t
return result.cleanarray
def clean_cosmic(f):
'''
From lacosmic.
'''
import cosmics
out = f.replace('.fits', '_clean.fits')
#If it does already exist, just return the name.
if (os.path.isfile(out)):
return out
#Otherwise, run the cosmic ray rejection based on LA Cosmic.
g = fitsutils.get_par(f, "GAIN")
if (fitsutils.has_par(f, "RDNOISE")):
rn = fitsutils.get_par(f, "RDNOISE")
else:
rn = 20
array, header = cosmics.fromfits(f)
try:
c = cosmics.cosmicsimage(array, gain=g, readnoise=rn, sigclip = 8.0, sigfrac = 0.3, satlevel = 64000.0)
c.run(maxiter = 3)
out = f.replace('.fits', '_clean.fits')
cosmics.tofits(out, c.cleanarray, header)
fitsutils.update_par(out, "CRREJ", 1)
#os.remove(f)
except:
pass
return out
def cosmicray_removal(textlist_cosmic, clip_section, prefix_str='c'):
"""
Corrects for cosmic rays in the OBJECT image after clipping based on the string 'clip_section'
Args:
textlist_cosmic : Text list containing names of FITS file to be corrected for Cosmic rays
clip_section : String list containing the section of the FITS file to be copied
prefix_str : Prefix to distinguish the aligned FITS file from the original FITS file
Returns:
None
"""
imcopy_clip(textlist_clip=textlist_cosmic,
clip_section=clip_section,
prefix_str='')
list_cosmic = text_list_to_python_list(textlist_cosmic)
for file_name in list_cosmic:
input_array, input_header = cs.fromfits(file_name)
input_object = cs.cosmicsimage(input_array,
gain=float(ccd_gain),
readnoise=float(read_noise),
sigclip=15.0,
sigfrac=0.5,
objlim=5.0,
satlevel=int(data_max),
verbose=False)
input_object.run(maxiter=2, verbose=False)
output_filename = prefix_str + file_name
remove_file(output_filename)
cs.tofits(outfilename=output_filename,
pixelarray=input_object.cleanarray,
hdr=input_header)
def clean_cosmicrays( image, params ):
imdat, imhead = cosmics.fromfits(image, verbose=False)
# DEFAULT HEADER PARAMETERS FOR VIRUS-P AS OF 2012
try: gain = imhead['GAIN1']
except: gain = 2.0
try: rdnoise = imhead['RDNOISE1']
except: rdnoise = 2.5
c = cosmics.cosmicsimage( imdat, gain=gain, readnoise=rdnoise, \
sigclip=5., sigfrac=0.3, objlim=5.0 )
c.run( maxiter=4 )
cosmics.tofits(image[:-5]+'c.fits', c.cleanarray, imhead)
cosmics.tofits(image[:-5]+'_mask.fits', c.mask, imhead)
def cosmicsClean(inFile,
outFile,
mask="yes",
Gain=2.2,
Readnoise=10.0,
Sigclip=5.0,
Sigfrac=0.3,
Objlim=5.0):
try:
print(
"MASK=%s, GAIN=%s, READNOISE=%s, SIGCLIP=%s, SIGFRAG=%s, OBJLIM=%s"
% (mask, Gain, Readnoise, Sigclip, Sigfrac, Objlim))
fp, fn = os.path.split(outFile)
fn = fn.split(".")[0]
# Thanks for cosmics.py to Malte Tewes (mtewes (at) astro.uni-bonn.de), http://obswww.unige.ch/people/malte.tewes/cosmics_dot_py/
# Read the FITS :
array, header = cosmics.fromfits(inFile)
# array is a 2D numpy array
# Build the object :
c = cosmics.cosmicsimage(array,
gain=Gain,
readnoise=Readnoise,
sigclip=Sigclip,
sigfrac=Sigfrac,
objlim=Objlim)
# There are other options, check the manual...
# Run the full artillery :
c.run(maxiter=4)
# Write the cleaned image into a new FITS file, conserving the original header :
cosmics.tofits(outFile, c.cleanarray, header)
# If you want the mask, here it is :
if mask == "yes":
print("%s/%s_mask.fit" % (fp, fn))
cosmics.tofits("%s/%s_mask.fit" % (fp, fn), c.mask, header)
# (c.mask is a boolean numpy array, that gets converted here to an integer array)
print("cosmicsClean succeed.")
return True
except:
print("cosmicsClean failed.")
return False
def cleanCosmics(self):
if self.ui.listWidget.count() != 0:
it = 0
self.ui.progressBar.setProperty("value", 0)
odir = QtGui.QFileDialog.getExistingDirectory(
self, 'Select Directory to Save PNG File(s)')
if os.path.exists(odir):
for x in xrange(self.ui.listWidget.count()):
it = it + 1
image = self.ui.listWidget.item(x)
filename = ntpath.basename(image.text())
basename, extension = os.path.splitext(filename)
# Read the FITS :
array, header = cosmics.fromfits(image.text())
# array is a 2D numpy array
# Build the object :
c = cosmics.cosmicsimage(array,
gain=2.2,
readnoise=10.0,
sigclip=5.0,
sigfrac=0.3,
objlim=5.0)
# There are other options, check the manual...
# Run the full artillery :
c.run(maxiter=4)
# Write the cleaned image into a new FITS file, conserving the original header :
cosmics.tofits("%s/%s_clean.fits" % (odir, basename),
c.cleanarray, header)
if self.ui.checkBox.isChecked():
# If you want the mask, here it is :
cosmics.tofits("%s/%s_mask.fits" % (odir, basename),
c.mask, header)
# (c.mask is a boolean numpy array, that gets converted here to an integer array)
self.ui.progressBar.setProperty(
"value",
math.ceil(100 * (float(
float(it) / float(self.ui.listWidget.count())))))
def cosmiczap(filename, outname, sigclip=6.0, maxiter=3, verbose=True):
"""
NAME:
cosmiczap
PURPOSE:
Removes cosmic rays using Laplacian cosmic ray identification written in cosmics.py
INPUTS:
filename - file or list of files to be cosmic ray zapped
outfile - name of output file
OPTIONAL KEYWORDS:
sigclip - sigma to clip
maxiter - maximum number of times to iterate loop
verbose - quiet?
EXAMPLE:
cosmiczap(filename, outname)
DEPENDENCIES:
cosmic.py (described in http://arxiv.org/pdf/1506.07791v3.pdf)
FUTURE IMPROVEMENTS:
Read readnoise from header?
"""
data, head = cosmics.fromfits(filename, verbose=False)
gain = head['GAIN']
c = cosmics.cosmicsimage(data,
gain=gain,
readnoise=18,
sigclip=sigclip,
sigfrac=0.5,
objlim=5.0,
verbose=False)
tot = c.run(maxiter=maxiter, verbose=False)
head['NPZAP'] = (tot, "Num. of pixels zapped by cosmiczap")
date = datetime.datetime.now().isoformat()
head.add_history('Processed by cosmiczap ' + date)
if verbose:
print ' Zapped %d total affected pixels (%.3f%% of total)' % (
tot, tot * 100.0 / np.size(data))
cosmics.tofits(outname, c.cleanarray, head, verbose=False)
def cosmic_reduce(dir, field, band):
"""
cosmic_reduce: read the FITS file and use L.A. Cosmic (http://www.astro.yale.edu/dokkum/lacosmic/)
to remove cosmic rays in the images
INPUT:
- dir: directory input of the combine images ('reduced/')
- field: beginning of the file name (e.g., 'Field027_A_72')
- band: {'g','r','i','z'} band
PARAMETERS for LA Cosmic:
- gain and readnoise are the property from the telescope (PISCO: gain 4 ADU/e, readnoise 3 e -Brian[3/27/17])
- satlevel: identify saturated level for bright stars
- sigclip, sigfrac, objlim
OUTPUT:
- nField..._g.fits: not clean data (original with a mask cut)
- cField..._g.fits: clean version, removed cosmic ray
- mField..._g.fits: masked file to remove cosmic ray
"""
array, header = cosmics.fromfits(
os.path.join(dir, field + '_' + band + '.fits'))
#cutting the circular aperature of the image out to only have good pixels in the center
if band == 'g':
array_c = array[10:-10, 350:2550]
elif band == 'r':
array_c = array[10:-10, 350:2550]
elif band == 'z':
array_c = array[10:-10, 650:2800]
elif band == 'i':
array_c = array[10:-10, 650:2800]
c = cosmics.cosmicsimage(array_c, gain=4.0, readnoise=3.0, sigclip = 2.5, sigfrac = 0.5,\
objlim = 5.0, satlevel=3000.0, verbose=False)
c.run(maxiter=5)
# if not os.path.exists(os.path.join(dir,'cosmics')):
# os.makedirs(os.path.join(dir,'cosmics'))
cosmics.tofits(
os.path.join(dir, 'cosmics', 'n' + field + '_' + band + '.fits'),
array_c, header)
cosmics.tofits(
os.path.join(dir, 'cosmics', 'c' + field + '_' + band + '.fits'),
c.cleanarray, header)
cosmics.tofits(
os.path.join(dir, 'cosmics', 'm' + field + '_' + band + '.fits'),
c.mask, header)
def clean_cosmic(f, name):
'''
From lacosmic.
'''
import cosmics
g = fitsutils.get_par(f, "GAIN")
rn = fitsutils.get_par(f, "RDNOISE")
array, header = cosmics.fromfits(f)
try:
c = cosmics.cosmicsimage(array, gain=g, readnoise=rn, sigclip = 8.0, sigfrac = 0.3, satlevel = 64000.0)
c.run(maxiter = 10)
out = f.replace('.fits', '_clean.fits')
cosmics.tofits(out, c.cleanarray, header)
os.remove(f)
except:
pass
def origRemoval(fitsFile):
"""
Runs the original cosmics.py script on the input FITS file.
Outputs: origClean.fits, origMask.Fits
"""
# Read the FITS :
array, header = origCosmics.fromfits(fitsFile)
# array is a 2D numpy array
# Build the object :
c = origCosmics.cosmicsimage(array, gain=2.2, readnoise=10.0, sigclip = 5.0,
sigfrac = 0.3, objlim = 5.0)
# There are other options, check the manual...
# Run the full artillery :
c.run(maxiter = 4)
# Write the cleaned image into a new FITS file, conserving the original header :
origCosmics.tofits("origClean.fits", c.cleanarray, header)
# If you want the mask, here it is :
origCosmics.tofits("origMask.fits", c.mask, header)
def cosmiczap(filename, outname, sigclip=6.0, maxiter=3, verbose=True):
"""
NAME:
cosmiczap
PURPOSE:
Removes cosmic rays using Laplacian cosmic ray identification written in cosmics.py
INPUTS:
filename - file or list of files to be cosmic ray zapped
outfile - name of output file
OPTIONAL KEYWORDS:
sigclip - sigma to clip
maxiter - maximum number of times to iterate loop
verbose - quiet?
EXAMPLE:
cosmiczap(filename, outname)
DEPENDENCIES:
cosmic.py (described in http://arxiv.org/pdf/1506.07791v3.pdf)
FUTURE IMPROVEMENTS:
Read readnoise from header?
"""
data, head = cosmics.fromfits(filename, verbose=False)
gain = head['GAIN']
c = cosmics.cosmicsimage(data, gain=gain, readnoise=18, sigclip=sigclip,
sigfrac = 0.5, objlim = 5.0, verbose=False)
tot = c.run(maxiter=maxiter, verbose=False)
head['NPZAP'] = (tot, "Num. of pixels zapped by cosmiczap")
date = datetime.datetime.now().isoformat()
head.add_history('Processed by cosmiczap ' + date)
if verbose: print ' Zapped %d total affected pixels (%.3f%% of total)' \
%(tot,tot*100.0/np.size(data))
cosmics.tofits(outname, c.cleanarray, head, verbose=False)
def clean_cosmic(f):
'''
From lacosmic.
'''
import cosmics
out = f.replace('.fits', '_clean.fits')
#If it does already exist, just return the name.
if (os.path.isfile(out)):
return out
#Otherwise, run the cosmic ray rejection based on LA Cosmic.
g = fitsutils.get_par(f, "GAIN")
if (fitsutils.has_par(f, "RDNOISE")):
rn = fitsutils.get_par(f, "RDNOISE")
else:
rn = 20
array, header = cosmics.fromfits(f)
try:
c = cosmics.cosmicsimage(array,
gain=g,
readnoise=rn,
sigclip=8.0,
sigfrac=0.3,
satlevel=64000.0)
c.run(maxiter=3)
out = f.replace('.fits', '_clean.fits')
cosmics.tofits(out, c.cleanarray, header)
fitsutils.update_par(out, "CRREJ", 1)
#os.remove(f)
except:
pass
return out
imgobjdir = os.path.join(objdir, image['imgname'])
os.chdir(imgobjdir)
# We reset everyting :
if os.path.isfile("cosmicsmask.fits"):
os.remove("cosmicsmask.fits")
if os.path.isfile("cosmicslist.pkl"):
os.remove("cosmicslist.pkl")
if os.path.isfile("orig_sig.fits"):
# Then we reset the original sigma image :
if os.path.isfile("sig.fits"):
os.remove("sig.fits")
os.rename("orig_sig.fits", "sig.fits")
# We read array and header of that fits file :
(a, h) = cosmics.fromfits("g.fits", verbose=False)
# We gather some parameters :
pssl = image['skylevel'] # The Previously Subtracted Sky Level
print "PSSL (TM): %.2f" % pssl
gain = image['gain']
readnoise = image['readnoise']
sigclip = cosmicssigclip
sigfrac = 0.3
objlim = 3.0
# Creating the object :
c = cosmics.cosmicsimage(a,
pssl=pssl,
def findcosmics(image):
imgpsfdir = os.path.join(psfdir, image['imgname'])
print "Image %i : %s" % (image["execi"], imgpsfdir)
os.chdir(os.path.join(imgpsfdir, "results"))
pssl = image['skylevel']
gain = image['gain']
#satlevel = image['saturlevel']*gain*maxpixelvaluecoeff
satlevel = -1.0
readnoise = image['readnoise']
print "Gain %.2f, PSSL %.2f, Readnoise %.2f" % (gain, pssl, readnoise)
for i in range(len(psfstars)):
starfilename = "star_%03i.fits" % (i + 1)
sigfilename = "starsig_%03i.fits" % (i + 1)
origsigfilename = "origstarsig_%03i.fits" % (i + 1)
starmaskfilename = "starmask_%03i.fits" % (i + 1)
starcosmicspklfilename = "starcosmics_%03i.pkl" % (i + 1)
# We reset everyting
if os.path.isfile(origsigfilename):
# Then we reset the original sigma image :
if os.path.isfile(sigfilename):
os.remove(sigfilename)
os.rename(origsigfilename, sigfilename)
if os.path.isfile(starmaskfilename):
os.remove(starmaskfilename)
if os.path.isfile(starcosmicspklfilename):
os.remove(starcosmicspklfilename)
# We read array and header of that fits file :
(a, h) = cosmics.fromfits(starfilename, verbose=False)
# Creating the object :
c = cosmics.cosmicsimage(
a,
pssl=pssl,
gain=gain,
readnoise=readnoise,
sigclip=sigclip,
sigfrac=sigfrac,
objlim=objlim,
satlevel=satlevel,
verbose=False
) # I put a correct satlevel instead of -1, to treat VLT images.
#print pssl, gain, readnoise, sigclip, sigfrac, objlim
c.run(maxiter=3)
ncosmics = np.sum(c.mask)
#print ncosmics
#if ncosmics != 0:
# print "--- %i pixels ---" % ncosmics
# We write the mask :
cosmics.tofits(starmaskfilename,
c.getdilatedmask(size=5),
verbose=False)
# And the labels (for later png display) :
cosmicslist = c.labelmask()
writepickle(cosmicslist, starcosmicspklfilename, verbose=False)
# We modify the sigma image, but keep a copy of the original :
os.rename(sigfilename, origsigfilename)
(sigarray, sigheader) = cosmics.fromfits(origsigfilename,
verbose=False)
sigarray[c.getdilatedmask(size=5)] = 1.0e8
cosmics.tofits(sigfilename, sigarray, sigheader, verbose=False)
def reduce(imglist,files_arc, _cosmic, _interactive_extraction,_arc):
import string
import os
import re
import sys
os.environ["PYRAF_BETA_STATUS"] = "1"
try: from astropy.io import fits as pyfits
except: import pyfits
import numpy as np
import util
import instruments
import combine_sides as cs
import cosmics
from pyraf import iraf
dv = util.dvex()
scal = np.pi / 180.
if not _interactive_extraction:
_interactive = False
else:
_interactive = True
if not _arc:
_arc_identify = False
else:
_arc_identify = True
iraf.noao(_doprint=0)
iraf.imred(_doprint=0)
iraf.ccdred(_doprint=0)
iraf.twodspec(_doprint=0)
iraf.longslit(_doprint=0)
iraf.onedspec(_doprint=0)
iraf.specred(_doprint=0)
iraf.disp(inlist='1', reference='1')
toforget = ['ccdproc', 'imcopy', 'specred.apall', 'longslit.identify', 'longslit.reidentify', 'specred.standard',
'longslit.fitcoords', 'onedspec.wspectext']
for t in toforget:
iraf.unlearn(t)
iraf.ccdred.verbose = 'no'
iraf.specred.verbose = 'no'
iraf.ccdproc.darkcor = 'no'
iraf.ccdproc.fixpix = 'no'
iraf.ccdproc.flatcor = 'no'
iraf.ccdproc.zerocor = 'no'
iraf.ccdproc.ccdtype = ''
iraf.longslit.mode = 'h'
iraf.specred.mode = 'h'
iraf.noao.mode = 'h'
iraf.ccdred.instrument = "ccddb$kpno/camera.dat"
list_arc_b = []
list_arc_r = []
for arcs in files_arc:
hdr = util.readhdr(arcs)
if util.readkey3(hdr, 'VERSION') == 'kastb':
list_arc_b.append(arcs)
elif util.readkey3(hdr, 'VERSION') == 'kastr':
list_arc_r.append(arcs)
else:
print util.readkey3(hdr, 'VERSION') + 'not in database'
sys.exit()
asci_files = []
newlist = [[],[]]
print '\n### images to reduce :',imglist
#raise TypeError
for img in imglist:
if 'b' in img:
newlist[0].append(img)
elif 'r' in img:
newlist[1].append(img)
if len(newlist[1]) < 1:
newlist = newlist[:-1]
for imgs in newlist:
hdr = util.readhdr(imgs[0])
if util.readkey3(hdr, 'VERSION') == 'kastb':
inst = instruments.kast_blue
elif util.readkey3(hdr, 'VERSION') == 'kastr':
inst = instruments.kast_red
else:
print util.readkey3(hdr, 'VERSION') + 'not in database'
sys.exit()
iraf.specred.dispaxi = inst.get('dispaxis')
iraf.longslit.dispaxi = inst.get('dispaxis')
_gain = inst.get('gain')
_ron = inst.get('read_noise')
iraf.specred.apall.readnoi = _ron
iraf.specred.apall.gain = _gain
_object0 = util.readkey3(hdr, 'OBJECT')
_date0 = util.readkey3(hdr, 'DATE-OBS')
_biassec0 = inst.get('biassec')
_trimsec0 = inst.get('trimsec')
_object0 = re.sub(' ', '', _object0)
_object0 = re.sub('/', '_', _object0)
nameout0 = str(_object0) + '_' + inst.get('name') + '_' + str(_date0)
nameout0 = util.name_duplicate(imgs[0], nameout0, '')
timg = nameout0
print '\n### now processing :',timg,' for -> ',inst.get('name')
if len(imgs) > 1:
img_str = ''
for i in imgs:
img_str = img_str + i + ','
iraf.imcombine(img_str, output=timg)
else:
img = imgs[0]
if os.path.isfile(timg):
os.system('rm -rf ' + timg)
iraf.imcopy(img, output=timg)
zero_file = inst.get('archive_zero_file')
os.system('cp ' + zero_file + ' .')
zero_file = string.split(zero_file, '/')[-1]
flat_file = inst.get('archive_flat_file')
os.system('cp ' + flat_file + ' .')
flat_file = string.split(flat_file, '/')[-1]
iraf.ccdproc(timg, output='', overscan='yes', trim='yes', zerocor="no", flatcor="no", readaxi='line',
trimsec=str(_trimsec0),biassec=str(_biassec0), Stdout=1)
iraf.ccdproc(timg, output='', overscan='no', trim='no', zerocor="yes", flatcor="no", readaxi='line',
zero=zero_file,order=3, Stdout=1)
iraf.ccdproc(timg, output='', overscan='no', trim='no', zerocor="no", flatcor="yes", readaxi='line',
flat=flat_file, Stdout=1)
img = timg
#raw_input("Press Enter to continue...")
print '\n### starting cosmic removal'
if _cosmic:
array, header = cosmics.fromfits(img)
c = cosmics.cosmicsimage(array, gain=inst.get('gain'), readnoise=inst.get('read_noise'), sigclip = 4.5, sigfrac = 0.5, objlim = 1.0)
c.run(maxiter = 4)
cosmics.tofits('cosmic_' + img, c.cleanarray, header)
print '\n### cosmic removal finished'
img='cosmic_' + img
if inst.get('name') == 'kast_blue':
arcfile = list_arc_b[0]
elif inst.get('name') == 'kast_red':
arcfile = list_arc_r[0]
if not arcfile.endswith(".fits"):
arcfile=arcfile+'.fits'
if os.path.isfile(arcfile):
util.delete('t' + arcfile)
iraf.ccdproc(arcfile, output= 't' + arcfile, overscan='yes', trim='yes', zerocor="no", flatcor="no",
readaxi='line', trimsec=str(_trimsec0), biassec=str(_biassec0), Stdout=1)
arcfile = 't' + arcfile
else:
print '\n### warning no arcfile \n exit '
sys.exit()
if not os.path.isdir('database/'):
os.mkdir('database/')
if _arc_identify:
arc_ex=re.sub('.fits', '.ms.fits', arcfile)
print '\n### arcfile : ',arcfile
print '\n### arcfile extraction : ',arc_ex
iraf.specred.apall(arcfile, output='', line = 'INDEF', nsum=10, interactive='no', extract='yes',find='yes', nfind=1 ,format='multispec', trace='no',back='no',recen='no')
iraf.longslit.identify(images=arc_ex, section=inst.get('section'),coordli=inst.get('line_list'),function = 'spline3',order=3, mode='h')
else:
arcref = inst.get('archive_arc_extracted')
arcrefid = inst.get('archive_arc_extracted_id')
os.system('cp ' + arcref + ' .')
arcref = string.split(arcref, '/')[-1]
os.system('cp ' + arcrefid + ' ./database')
arc_ex=re.sub('.fits', '.ms.fits', arcfile)
print '\n### arcfile : ',arcfile
print '\n### arcfile extraction : ',arc_ex
print '\n### arc referenece : ',arcref
iraf.specred.apall(arcfile, output=arc_ex, line = 'INDEF', nsum=10, interactive='no', extract='yes',find='yes', nfind=1 ,format='multispec', trace='no',back='no',recen='no')
iraf.longslit.reidentify(referenc=arcref, images=arc_ex, interac='NO', section=inst.get('section'),
coordli=inst.get('line_list'), shift='INDEF', search='INDEF',
mode='h', verbose='YES', step=0,nsum=5, nlost=2, cradius=10, refit='yes',overrid='yes',newaps='no')
#print '\n### checking sky lines '
#_skyfile = inst.get('sky_file')
#shift = util.skyfrom2d(img, _skyfile,'True')
#print '\n### I found a shift of : ',shift
print '\n### extraction using apall'
result = []
hdr_image = util.readhdr(img)
_type=util.readkey3(hdr_image, 'object')
if _type.startswith("arc") or _type.startswith("dflat") or _type.startswith("Dflat") or _type.startswith("Dbias") or _type.startswith("Bias"):
print '\n### warning problem \n exit '
sys.exit()
else:
imgex = util.extractspectrum(
img, dv, inst, _interactive, 'obj')
print '\n### applying wavelength solution'
iraf.disp(inlist=imgex, reference=arc_ex)
sensfile = inst.get('archive_sens')
os.system('cp ' + sensfile + ' .')
sensfile = string.split(sensfile, '/')[-1]
if sensfile:
print '\n### sensitivity function : ',sensfile
imgf = re.sub('.fits', '_f.fits', img)
_extinction = inst.get('extinction_file')
_observatory = inst.get('observatory')
_exptime = util.readkey3(hdr, 'EXPTIME')
_airmass = util.readkey3(hdr, 'AIRMASS')
util.delete(imgf)
dimgex='d'+imgex
iraf.specred.calibrate(input=dimgex, output=imgf, sensiti=sensfile, extinct='yes',
extinction=_extinction,flux='yes', ignorea='yes', airmass=_airmass, exptime=_exptime,
fnu='no')
imgout = imgf
imgasci = re.sub('.fits', '.asci', imgout)
errasci = re.sub('.fits', '_err.asci', imgout)
util.delete(imgasci)
iraf.onedspec.wspectext(imgout + '[*,1,1]', imgasci, header='no')
iraf.onedspec.wspectext(imgout + '[*,1,4]', errasci, header='no')
spec = np.transpose(np.genfromtxt(imgasci))
err = np.transpose(np.genfromtxt(errasci))
util.delete(errasci)
final = np.transpose([spec[0], spec[1], err[1]])
np.savetxt(imgasci, final)
result = result + [imgout, imgasci]
result = result + [imgex] + [timg]
asci_files.append(imgasci)
if not os.path.isdir(_object0 + '/'):
os.mkdir(_object0 + '/')
for img in result:
os.system('mv ' + img + ' ' + _object0 + '/')
else:
for img in result:
os.system('mv ' + img + ' ' + _object0 + '/')
if not _arc_identify:
util.delete(arcref)
util.delete(sensfile)
util.delete(zero_file)
util.delete(flat_file)
util.delete(arc_ex)
util.delete(arcfile)
util.delete('logfile')
util.delete(dimgex)
util.delete('cosmic_*')
print '\n### now i will merge ...'
if len(asci_files) > 1:
final = cs.combine_blue_red(asci_files[0], asci_files[1], _object0)
print '\n### final result in folder ',_object0,' is ',_object0+'_merged.asci'
return result
"""
Given an image file (.fits format) this script will remove cosmic rays from image which will contaminate the data
The number of iterations the cleanup process will occur is user specified as wll as the various parameters of the clen up process such as gain, noise, and standard deviation of noise etc...
"""
import cosmics
"Import the image file"
file_path = "/Users/edenmolina/Documents/Astro/Coadds/2013/1206_10_1.fits"
name = "J1206"
image_file, hdr = cosmics.fromfits(file_path)
iterations = 10
"Create lacosmics object"
c = cosmics.cosmicsimage(image_file,
gain=7,
readnoise=.4,
sigclip=2,
sigfrac=.5,
objlim=4)
"Clean the image and export"
for i in range(iterations):
print "Iteration: %s" % i
c.lacosmiciteration()
c.clean()
cosmics.tofits('/Users/edenmolina/Desktop/Masks/%s_clean.fits' % name,
c.cleanarray, hdr)
cosmics.tofits("/Users/edenmolina/Desktop/Masks/%s_mask.fits" % name, c.mask,
hdr)
low_reject=0.,
high_reject=0.,
niterate=1,
grow=0.,
graphics="stdgraph",
cursor="")
iraf.imstat('nmflat.fits')
os.system("ls J105829*b.fits GD248*b.fits > flatf.in")
os.system("sed s/b.fits/bf.fits/g flatf.in > flatf.out")
iraf.imarith('@flatf.in', '/', 'nmflat.fits', '@flatf.out')
print("***********************************************************")
print("Flat correction done!")
# Removal of cosmic rays from the images
array, header = cosmics.fromfits(
"J105829_250_3500_4_Grism_7_2015_06_17_yf170018bf.fits")
c = cosmics.cosmicsimage(array,
gain=1.22,
readnoise=4.8,
sigclip=4.5,
sigfrac=0.3,
objlim=4.5)
c.run(maxiter=4)
cosmics.tofits("J105829_250_3500_4_Grism_7_2015_06_17_yf170018bfc.fits",
c.cleanarray, header)
array, header = cosmics.fromfits(
"GD248_standard_250_3500_4_Grism_7_yf170031bf.fits")
c = cosmics.cosmicsimage(array,
gain=1.22,
readnoise=4.8,
def esi_cosmic(date):
#Get edge masks from file
orders_mask = pickle.load(
open(str(date) + '/Calibs/orders_mask_' + str(date) + '.p', 'rb'))
background_mask = pickle.load(
open(str(date) + '/Calibs/background_mask_' + str(date) + '.p', 'rb'))
#Bias
bias = pyfits.getdata(str(date) + '/Calibs/bias_' + str(date) + '.fits')
#Normalized Flat
flat = pyfits.getdata(
str(date) + '/Calibs/norm_flat_' + str(date) + '.fits')
#READ LOG
im1 = open(str(date) + '/Logs/esi_info_' + str(date) + '.dat', 'r')
data1 = im1.readlines()
im1.close()
filename = []
dateobs = []
objname = []
imgtype = []
ra = []
dec = []
exptime = []
usable = []
for line in data1:
p = line.split()
filename.append(p[0])
dateobs.append(p[1])
objname.append(p[2])
imgtype.append(p[3])
ra.append(p[4])
dec.append(p[5])
exptime.append(p[6])
usable.append(p[7])
#Rewrite in a more convenient array with format array[line][element]
alldata = []
for line in range(len(usable)):
alldata.append([
filename[line], dateobs[line], objname[line], imgtype[line],
ra[line], dec[line], exptime[line], usable[line]
])
#Find good files:
good = []
for line in range(len(alldata)):
if "yes" in alldata[line][7]:
good.append(alldata[line])
#Find list of objects:
names = []
for line in range(len(alldata)):
if ("Object" in alldata[line][3]
and float(alldata[line][6]) > 600) or "*" in alldata[line][2]:
names.append(alldata[line][2])
objects = np.array(list(set(names)))
objects.sort() #ascending order, modify in place
#Find list of lines
names = []
for line in range(len(alldata)):
if "Line" in alldata[line][3]:
names.append(alldata[line][2])
lines = np.array(list(set(names)))
lines.sort() #ascending order, modify in place
#Find list of stars:
names = []
for line in range(len(alldata)):
if "*" in alldata[line][2]:
names.append(alldata[line][2])
stars = np.array(list(set(names)))
stars.sort() #ascending order, modify in place
#Make directory to hold decosmicified files
if not os.path.exists(str(date) + '/Calibs/cosmicless/'):
os.makedirs(str(date) + '/Calibs/cosmicless/')
#Remove cosmics from each lamp
for obj_id in lines:
print "reducing " + str(obj_id) + '...'
#Find files related to this object
aobj_id = []
for line in range(len(good)):
if str(obj_id) in good[line][2]:
aobj_id.append(good[line])
print str(len(aobj_id)), "files for " + str(obj_id)
#Write path to each objects files:
obj_locs = [
str(date) + '/Raw/' + str(aobj_id[line][0])
for line in range(len(aobj_id))
]
#Read in and de-cosmify
for line in range(len(obj_locs)):
array, header = cosmics.fromfits(obj_locs[line])
#array = array - bias #backwards for some reason
c = cosmics.cosmicsimage(array,
gain=1.29,
readnoise=2.2,
sigclip=6,
objlim=5.0,
sigfrac=0.7,
satlevel=1e4)
c.run(
maxiter=3
) # can increase up to 4 to improve precision, but takes longer
cosmics.tofits(
str(date) + '/Calibs/cosmicless/' + str(obj_id) + '_' +
str(line + 1) + 'decos.fits', c.cleanarray, header)
#now zip it (to save space)
f = pyfits.getdata(
str(date) + '/Calibs/cosmicless/' + str(obj_id) + '_' +
str(line + 1) + 'decos.fits')
hdu = pyfits.CompImageHDU(f)
hdu.writeto(str(date) + '/Calibs/cosmicless/' + str(obj_id) + '_' +
str(line + 1) + 'decos.fits',
clobber=True)
#Remove cosmics from each object
for obj_id in objects:
print "reducing " + str(obj_id) + '...'
#Find files related to this object
aobj_id = []
for line in range(len(good)):
if str(obj_id) in good[line][2]:
aobj_id.append(good[line])
print str(len(aobj_id)), "files for " + str(obj_id)
#Write path to each objects files:
obj_locs = [
str(date) + '/Raw/' + str(aobj_id[line][0])
for line in range(len(aobj_id))
]
#Read in and de-cosmify
for line in range(len(obj_locs)):
array, header = cosmics.fromfits(obj_locs[line])
#array = array - bias #backwards for some reason
c = cosmics.cosmicsimage(array,
gain=1.29,
readnoise=2.2,
sigclip=3.8,
objlim=3.0,
sigfrac=0.7,
satlevel=-1)
c.run(
maxiter=3
) # can increase up to 4 to improve precision, but takes longer
cosmics.tofits(
str(date) + '/Calibs/cosmicless/' + str(obj_id) + '_' +
str(line + 1) + 'decos.fits', c.cleanarray, header)
#now zip it (to save space)
f = pyfits.getdata(
str(date) + '/Calibs/cosmicless/' + str(obj_id) + '_' +
str(line + 1) + 'decos.fits')
hdu = pyfits.CompImageHDU(f)
hdu.writeto(str(date) + '/Calibs/cosmicless/' + str(obj_id) + '_' +
str(line + 1) + 'decos.fits',
clobber=True)
#Remove cosmics from each star
for obj_id in stars:
print "reducing " + str(obj_id) + '...'
#Find files related to this object
aobj_id = []
for line in range(len(good)):
if str(obj_id) == good[line][2]:
aobj_id.append(good[line])
print str(len(aobj_id)), "files for " + str(obj_id)
#Write path to each objects files:
obj_locs = [
str(date) + '/Raw/' + str(aobj_id[line][0])
for line in range(len(aobj_id))
]
#Read in and de-cosmify
for line in range(len(obj_locs)):
array, header = cosmics.fromfits(obj_locs[line])
#array = array - bias #backwards for some reason
c = cosmics.cosmicsimage(array,
gain=1.29,
readnoise=2.2,
sigclip=10,
objlim=7.0,
sigfrac=0.7,
satlevel=15000)
c.run(
maxiter=3
) # can increase up to 4 to improve precision, but takes longer
cosmics.tofits(
str(date) + '/Calibs/cosmicless/' + str(obj_id) + '_' +
str(line + 1) + 'decos.fits', c.cleanarray, header)
#now zip it (to save space)
f = pyfits.getdata(
str(date) + '/Calibs/cosmicless/' + str(obj_id) + '_' +
str(line + 1) + 'decos.fits')
hdu = pyfits.CompImageHDU(f)
hdu.writeto(str(date) + '/Calibs/cosmicless/' + str(obj_id) + '_' +
str(line + 1) + 'decos.fits',
clobber=True)
def reduce(imglist, files_arc, files_flat, _cosmic, _interactive_extraction,
_arc):
import string
import os
import re
import sys
import pdb
os.environ["PYRAF_BETA_STATUS"] = "1"
try:
from astropy.io import fits as pyfits
except:
import pyfits
import numpy as np
import util
import instruments
import combine_sides as cs
import cosmics
from pyraf import iraf
dv = util.dvex()
scal = np.pi / 180.
if not _interactive_extraction:
_interactive = False
else:
_interactive = True
if not _arc:
_arc_identify = False
else:
_arc_identify = True
iraf.noao(_doprint=0)
iraf.imred(_doprint=0)
iraf.ccdred(_doprint=0)
iraf.twodspec(_doprint=0)
iraf.longslit(_doprint=0)
iraf.onedspec(_doprint=0)
iraf.specred(_doprint=0)
iraf.disp(inlist='1', reference='1')
toforget = [
'ccdproc', 'imcopy', 'specred.apall', 'longslit.identify',
'longslit.reidentify', 'specred.standard', 'longslit.fitcoords',
'onedspec.wspectext'
]
for t in toforget:
iraf.unlearn(t)
iraf.ccdred.verbose = 'no'
iraf.specred.verbose = 'no'
iraf.ccdproc.darkcor = 'no'
iraf.ccdproc.fixpix = 'no'
iraf.ccdproc.flatcor = 'no'
iraf.ccdproc.zerocor = 'no'
iraf.ccdproc.ccdtype = ''
iraf.longslit.mode = 'h'
iraf.specred.mode = 'h'
iraf.noao.mode = 'h'
iraf.ccdred.instrument = "ccddb$kpno/camera.dat"
list_arc_b = []
list_arc_r = []
for arcs in files_arc:
hdr = util.readhdr(arcs)
br, inst = instruments.blue_or_red(arcs)
if br == 'blue':
list_arc_b.append(arcs)
elif br == 'red':
list_arc_r.append(arcs)
else:
errStr = '{} '.format(str(util.readkey3(hdr, 'VERSION')))
errStr += 'not in database'
print(errStr)
sys.exit()
asci_files = []
newlist = [[], []]
print('\n### images to reduce :', imglist)
#raise TypeError
for img in imglist:
if 'b' in img:
newlist[0].append(img)
elif 'r' in img:
newlist[1].append(img)
if len(newlist[1]) < 1:
newlist = newlist[:-1]
elif len(newlist[0]) < 1:
newlist = newlist[1:]
else:
sides = raw_input("Reduce which side? ([both]/b/r): ")
if sides == 'b':
newlist = newlist[:-1]
elif sides == 'r':
newlist = newlist[1:]
for imgs in newlist:
hdr = util.readhdr(imgs[0])
br, inst = instruments.blue_or_red(imgs[0])
if br == 'blue':
flat_file = '../RESP_blue'
elif br == 'red':
flat_file = '../RESP_red'
else:
errStr = 'Not in intrument list'
print(errStr)
sys.exit()
iraf.specred.dispaxi = inst.get('dispaxis')
iraf.longslit.dispaxi = inst.get('dispaxis')
_gain = inst.get('gain')
_ron = inst.get('read_noise')
iraf.specred.apall.readnoi = _ron
iraf.specred.apall.gain = _gain
_object0 = util.readkey3(hdr, 'OBJECT')
_date0 = util.readkey3(hdr, 'DATE-OBS')
_object0 = re.sub(' ', '', _object0)
_object0 = re.sub('/', '_', _object0)
nameout0 = str(_object0) + '_' + inst.get('name') + '_' + str(_date0)
nameout0 = util.name_duplicate(imgs[0], nameout0, '')
timg = nameout0
print('\n### now processing :', timg, ' for -> ', inst.get('name'))
if len(imgs) > 1:
img_str = ''
for i in imgs:
img_str = img_str + i + ','
iraf.imcombine(img_str, output=timg)
else:
img = imgs[0]
if os.path.isfile(timg):
os.system('rm -rf ' + timg)
iraf.imcopy(img, output=timg)
# should just do this by hand
iraf.ccdproc(timg,
output='',
overscan='no',
trim='no',
zerocor="no",
flatcor="yes",
readaxi='line',
flat=flat_file,
Stdout=1)
img = timg
#raw_input("Press Enter to continue...")
if _cosmic:
print('\n### starting cosmic removal')
array, header = cosmics.fromfits(img)
c = cosmics.cosmicsimage(array,
gain=inst.get('gain'),
readnoise=inst.get('read_noise'),
sigclip=5,
sigfrac=0.5,
objlim=2.0)
c.run(maxiter=5)
cosmics.tofits('cosmic_' + img, c.cleanarray, header)
img = 'cosmic_' + img
print('\n### cosmic removal finished')
else:
print(
'\n### No cosmic removal, saving normalized image for inspection???'
)
if inst.get('arm') == 'blue' and len(list_arc_b) > 0:
arcfile = list_arc_b[0]
elif inst.get('arm') == 'red' and len(list_arc_r) > 0:
arcfile = list_arc_r[0]
else:
arcfile = None
if arcfile is not None and not arcfile.endswith(".fits"):
arcfile = arcfile + '.fits'
if not os.path.isdir('database/'):
os.mkdir('database/')
if _arc_identify:
os.system('cp ' + arcfile + ' .')
arcfile = string.split(arcfile, '/')[-1]
arc_ex = re.sub('.fits', '.ms.fits', arcfile)
arcref = inst.get('archive_arc_extracted')
arcref_img = string.split(arcref, '/')[-1]
arcref_img = arcref_img.replace('.ms.fits', '')
arcrefid = inst.get('archive_arc_extracted_id')
os.system('cp ' + arcref + ' .')
arcref = string.split(arcref, '/')[-1]
os.system('cp ' + arcrefid + ' ./database')
aperture = inst.get('archive_arc_aperture')
os.system('cp ' + aperture + ' ./database')
print('\n### arcfile : ', arcfile)
print('\n### arcfile extraction : ', arc_ex)
print('\n### arc reference : ', arcref)
# read for some meta data to get the row right
tmpHDU = pyfits.open(arcfile)
header = tmpHDU[0].header
try:
spatialBin = int(header['binning'].split(',')[0])
except KeyError:
spatialBin = 1
apLine = 700 // spatialBin
iraf.specred.apall(arcfile,
output=arc_ex,
ref=arcref_img,
line=apLine,
nsum=10,
interactive='no',
extract='yes',
find='yes',
nfind=1,
format='multispec',
trace='no',
back='no',
recen='no')
iraf.longslit.reidentify(referenc=arcref,
images=arc_ex,
interac='NO',
section=inst.get('section'),
coordli=inst.get('line_list'),
shift='INDEF',
search='INDEF',
mode='h',
verbose='YES',
step=0,
nsum=5,
nlost=2,
cradius=10,
refit='yes',
overrid='yes',
newaps='no')
print('\n### extraction using apall')
result = []
hdr_image = util.readhdr(img)
_type = util.readkey3(hdr_image, 'object')
if (_type.startswith("arc") or _type.startswith("dflat")
or _type.startswith("Dflat") or _type.startswith("Dbias")
or _type.startswith("Bias")):
print('\n### warning problem \n exit ')
sys.exit()
else:
imgex = util.extractspectrum(img, dv, inst, _interactive, 'obj')
print('\n### applying wavelength solution')
print(arc_ex)
iraf.disp(inlist=imgex, reference=arc_ex)
result = result + [imgex] + [timg]
# asci_files.append(imgasci)
if not os.path.isdir(_object0 + '_ex/'):
os.mkdir(_object0 + '_ex/')
if not _arc_identify:
util.delete(arcref)
else:
util.delete(arcfile)
util.delete(arc_ex)
util.delete(img)
util.delete(imgex)
util.delete(arcref)
util.delete('logfile')
#if _cosmic:
#util.delete(img[7:])
#util.delete("cosmic_*")
os.system('mv ' + 'd' + imgex + ' ' + _object0 + '_ex/')
use_sens = raw_input('Use archival flux calibration? [y]/n ')
if use_sens != 'no':
sensfile = inst.get('archive_sens')
os.system('cp ' + sensfile + ' ' + _object0 + '_ex/')
bstarfile = inst.get('archive_bstar')
os.system('cp ' + bstarfile + ' ' + _object0 + '_ex/')
return result
else:
images = db.select(imgdb, ['flagali', 'gogogo', 'treatme'],
['==1', True, True], ['recno', 'imgname', 'stddev'],
sortFields=['imgname'],
returnType='dict')
print "I will treat %i images." % len(images)
proquest(askquestions)
for n, image in enumerate(images):
print n + 1, "/", len(images), ":", image['imgname']
aliimg = os.path.join(alidir, image['imgname'] + "_ali.fits")
(a, h) = cosmics.fromfits(aliimg, verbose=False)
zeroa = a <= (0.01 * image["stddev"])
cols = zeroa.all(axis=0)
lins = zeroa.all(axis=1)
colsshape = a[:, cols].shape
a[:, cols] = (np.random.randn(colsshape[0] * colsshape[1]) *
image["stddev"]).reshape(colsshape)
linsshape = a[lins, :].shape
a[lins, :] = (np.random.randn(linsshape[0] * linsshape[1]) *
image["stddev"]).reshape(linsshape)
cosmics.tofits(aliimg, a, verbose=False)
def cr_clean(fits_fn):
det = pyfits.getval(fits_fn, 'DETECTOR')
if det == 'IR':
print('CR-cleaning IR FLT file: %s' % fits_fn)
readnoise = 20.0 # for WFC3/IR
#sigclip = 5.0
sigclip = 5.0
sigfrac = 0.3 # small means crs are grown more 0.3 is good!
#objlim = 5.0
objlim = 12.0
#maxiter = 4
maxiter = 3
exptime = pyfits.getval(fits_fn, 'EXPTIME')
gain = exptime
(image, header) = cosmics.fromfits(fits_fn)
c = cosmics.cosmicsimage(image, gain=gain, readnoise=readnoise, sigclip=sigclip, sigfrac=sigfrac, objlim=objlim)
c.run(maxiter=maxiter)
clean_fn = fits_fn.replace('.fits', '_crclean.fits')
mask_fn = fits_fn.replace('.fits', '_crmask.fits')
fo = pyfits.open(fits_fn)
fo[1].data = c.cleanarray
fo.writeto(clean_fn)
fo.close()
#cosmics.tofits(clean_fn, c.cleanarray, header)
cosmics.tofits(mask_fn, c.mask, header)
if det == 'UVIS':
print('CR-cleaning UVIS FLT file: %s' % fits_fn)
readnoise = 3.105 # for WFC3/UVIS
# good
#sigclip = 5.0
#sigfrac = 0.1 # small means crs are grown more 0.3 is good!
#objlim = 5.0
# some residuals
#sigclip = 4.5
#sigfrac = 0.1 # small means crs are grown more 0.3 is good!
#objlim = 4.0
sigclip = 4.5
sigfrac = 0.1 # small means crs are grown more 0.3 is good!
objlim = 2.0
maxiter = 4
exptime = pyfits.getval(fits_fn, 'EXPTIME')
gain = 1.0
ff = pyfits.open(fits_fn)
header = ff[0].header
image = ff[1].data
#(image, header) = cosmics.fromfits(fits_fn)
c = cosmics.cosmicsimage(image, gain=gain, readnoise=readnoise, sigclip=sigclip, sigfrac=sigfrac, objlim=objlim)
c.run(maxiter=maxiter)
image2 = ff[4].data
d = cosmics.cosmicsimage(image2, gain=gain, readnoise=readnoise, sigclip=sigclip, sigfrac=sigfrac, objlim=objlim)
d.run(maxiter=maxiter)
clean_fn = fits_fn.replace('.fits', '_crclean.fits')
mask_fn = fits_fn.replace('.fits', '_crmask.fits')
fo = pyfits.open(fits_fn)
fo[1].data = c.cleanarray
fo[4].data = d.cleanarray
fo.writeto(clean_fn)
fo.close()
#cosmics.tofits(clean_fn, c.cleanarray, header)
cosmics.tofits(mask_fn, c.mask, header)
def esi_cosmic(date):
#Get edge masks from file
orders_mask = pickle.load(open(str(date)+'/Calibs/orders_mask_'+str(date)+'.p', 'rb'))
background_mask = pickle.load(open(str(date)+'/Calibs/background_mask_'+str(date)+'.p', 'rb'))
#Bias
bias = pyfits.getdata(str(date)+'/Calibs/bias_'+str(date)+'.fits')
#Normalized Flat
flat = pyfits.getdata(str(date)+'/Calibs/norm_flat_'+str(date)+'.fits')
#READ LOG
im1 = open(str(date)+'/Logs/esi_info_'+str(date)+'.dat','r')
data1 = im1.readlines()
im1.close()
filename = []
dateobs = []
objname = []
imgtype = []
ra = []
dec = []
exptime = []
usable = []
for line in data1:
p = line.split()
filename.append(p[0])
dateobs.append(p[1])
objname.append(p[2])
imgtype.append(p[3])
ra.append(p[4])
dec.append(p[5])
exptime.append(p[6])
usable.append(p[7])
#Rewrite in a more convenient array with format array[line][element]
alldata = []
for line in range(len(usable)):
alldata.append([filename[line],dateobs[line],objname[line],imgtype[line],
ra[line],dec[line],exptime[line],usable[line]])
#Find good files:
good = []
for line in range(len(alldata)):
if "yes" in alldata[line][ 7]:
good.append(alldata[line])
#Find list of objects:
names = []
for line in range(len(alldata)):
if ("Object" in alldata[line][3] and float(alldata[line][6]) > 600) or "*" in alldata[line][2]:
names.append(alldata[line][2])
objects = np.array(list(set(names)))
objects.sort() #ascending order, modify in place
#Find list of lines
names = []
for line in range(len(alldata)):
if "Line" in alldata[line][3]:
names.append(alldata[line][2])
lines = np.array(list(set(names)))
lines.sort() #ascending order, modify in place
#Find list of stars:
names = []
for line in range(len(alldata)):
if "*" in alldata[line][2]:
names.append(alldata[line][2])
stars = np.array(list(set(names)))
stars.sort() #ascending order, modify in place
#Make directory to hold decosmicified files
if not os.path.exists(str(date)+'/Calibs/cosmicless/'):
os.makedirs(str(date)+'/Calibs/cosmicless/')
#Remove cosmics from each lamp
for obj_id in lines:
print "reducing " + str(obj_id) + '...'
#Find files related to this object
aobj_id = []
for line in range(len(good)):
if str(obj_id) in good[line][2]:
aobj_id.append(good[line])
print str(len(aobj_id)), "files for "+ str(obj_id)
#Write path to each objects files:
obj_locs = [str(date)+'/Raw/' + str(aobj_id[line][0]) for line in range(len(aobj_id))]
#Read in and de-cosmify
for line in range(len(obj_locs)):
array, header = cosmics.fromfits(obj_locs[line])
#array = array - bias #backwards for some reason
c = cosmics.cosmicsimage(array, gain = 1.29, readnoise = 2.2, sigclip = 6, objlim = 5.0, sigfrac = 0.7, satlevel = 1e4)
c.run(maxiter = 3) # can increase up to 4 to improve precision, but takes longer
cosmics.tofits(str(date)+'/Calibs/cosmicless/' + str(obj_id)+'_'+str(line+1)+'decos.fits', c.cleanarray, header)
#now zip it (to save space)
f = pyfits.getdata(str(date)+'/Calibs/cosmicless/' + str(obj_id)+'_'+str(line+1)+'decos.fits')
hdu = pyfits.CompImageHDU(f)
hdu.writeto(str(date)+'/Calibs/cosmicless/' + str(obj_id)+'_'+str(line+1)+'decos.fits', clobber=True)
#Remove cosmics from each object
for obj_id in objects:
print "reducing " + str(obj_id) + '...'
#Find files related to this object
aobj_id = []
for line in range(len(good)):
if str(obj_id) in good[line][2]:
aobj_id.append(good[line])
print str(len(aobj_id)), "files for "+ str(obj_id)
#Write path to each objects files:
obj_locs = [str(date)+'/Raw/' + str(aobj_id[line][0]) for line in range(len(aobj_id))]
#Read in and de-cosmify
for line in range(len(obj_locs)):
array, header = cosmics.fromfits(obj_locs[line])
#array = array - bias #backwards for some reason
c = cosmics.cosmicsimage(array, gain = 1.29, readnoise = 2.2, sigclip = 3.8, objlim = 3.0, sigfrac = 0.7, satlevel = -1)
c.run(maxiter = 3) # can increase up to 4 to improve precision, but takes longer
cosmics.tofits(str(date)+'/Calibs/cosmicless/' + str(obj_id)+'_'+str(line+1)+'decos.fits', c.cleanarray, header)
#now zip it (to save space)
f = pyfits.getdata(str(date)+'/Calibs/cosmicless/' + str(obj_id)+'_'+str(line+1)+'decos.fits')
hdu = pyfits.CompImageHDU(f)
hdu.writeto(str(date)+'/Calibs/cosmicless/' + str(obj_id)+'_'+str(line+1)+'decos.fits', clobber=True)
#Remove cosmics from each star
for obj_id in stars:
print "reducing " + str(obj_id) + '...'
#Find files related to this object
aobj_id = []
for line in range(len(good)):
if str(obj_id) == good[line][2]:
aobj_id.append(good[line])
print str(len(aobj_id)), "files for "+ str(obj_id)
#Write path to each objects files:
obj_locs = [str(date)+'/Raw/' + str(aobj_id[line][0]) for line in range(len(aobj_id))]
#Read in and de-cosmify
for line in range(len(obj_locs)):
array, header = cosmics.fromfits(obj_locs[line])
#array = array - bias #backwards for some reason
c = cosmics.cosmicsimage(array, gain = 1.29, readnoise = 2.2, sigclip = 10, objlim = 7.0, sigfrac = 0.7, satlevel = 15000)
c.run(maxiter = 3) # can increase up to 4 to improve precision, but takes longer
cosmics.tofits(str(date)+'/Calibs/cosmicless/' + str(obj_id)+'_'+str(line+1)+'decos.fits', c.cleanarray, header)
#now zip it (to save space)
f = pyfits.getdata(str(date)+'/Calibs/cosmicless/' + str(obj_id)+'_'+str(line+1)+'decos.fits')
hdu = pyfits.CompImageHDU(f)
hdu.writeto(str(date)+'/Calibs/cosmicless/' + str(obj_id)+'_'+str(line+1)+'decos.fits', clobber=True)
# The following two lines are only needed as cosmic.py is not in this directory nor in the python path.
# They would not be required if you copy cosmics.py in this directory.
import sys
sys.path.append("../.") # The directory that contains cosmic.py
import cosmics
# Read the FITS :
array, header = cosmics.fromfits("input.fits")
# array is a 2D numpy array
# Build the object :
c = cosmics.cosmicsimage(array, gain=2.2, readnoise=10.0, sigclip = 5.0, sigfrac = 0.3, objlim = 5.0)
# There are other options, check the manual...
# Run the full artillery :
c.run(maxiter = 4)
# Write the cleaned image into a new FITS file, conserving the original header :
cosmics.tofits("clean.fits", c.cleanarray, header)
# If you want the mask, here it is :
cosmics.tofits("mask.fits", c.mask, header)
# (c.mask is a boolean numpy array, that gets converted here to an integer array)
def run_cosmics(filename):
'''
The main controller.
'''
output = filename.split('_')[0] + "_cr_" + filename.split('_')[1]
# Assert the input file exists
error = filename + ' input for run_cosmics in '
error += 'run_cosmics.py does not exist.'
assert os.access(filename, os.F_OK), error
# Define the output name and delete if exists.
query = os.access(output, os.F_OK)
if query == True:
os.remove(output)
# Find the number of extentions.
header = pyfits.open(filename)
header_count = len(header)
header.close()
for ext in range(0, header_count):
print filename + '[' + str(ext) + ']'
if ext in [0, 5]:
hdu = pyfits.open(filename)
array = hdu[ext].data
header = hdu[ext].header
pyfits.append(
output,
array,
header)
hdu.close()
else:
# Read the FITS :
array, header = cosmics.fromfits(filename, hdu = ext)
# array is a 2D numpy array
# Build the object
# WFPC2 PC Chip Settings
if ext == 1:
c = cosmics.cosmicsimage(
array,
pssl = 0.0,
gain = 1.0,
readnoise = 5.0,
sigclip = 3.0,
sigfrac = 0.01,
objlim = 4.0,
satlevel = 4095.0,
verbose = True)
# Build the object
# WFPC2 WF Chip Settings
elif ext in [2, 3, 4]:
c = cosmics.cosmicsimage(
array,
pssl = 0.0,
gain=1.0,
readnoise=5.0,
sigclip = 2.5,
sigfrac = 0.001,
objlim = 5.0,
satlevel = 4095.0,
verbose = True)
# Run the full artillery
c.run(maxiter = 7)
# Write the cleaned image into a new FITS file
# conserving the original header :
#cosmics.tofits(output, c.cleanarray, header)
pyfits.append(
output,
c.cleanarray.transpose(),
header,
ext = 'SCI')
# If you want the mask, here it is :
# cosmics.tofits("mask.fits", c.mask, header)
# (c.mask is a boolean numpy array, that gets
# converted here to an integer array)
assert os.path.isfile(output) == True, 'run_cosmics did not create: ' + output
# Create a symbolic link to the original c1m files that matches the
# Output filename.
make_c1m_link(os.path.abspath(filename))
def whirc_reduce(obj_id):
import pyfits
import numpy as np
import matplotlib.pyplot as plt
import cosmics
import astropy.coordinates as coord
import astropy.units as u
#READ IN DATA AND OBSERVING LOG
#Read in the bias file to be subtracted from flats
bias = pyfits.getdata("Calibs/bias.fits")
bias = bias[0:2048,0:2048]
#Read in the dark file to be subtracted from flats
dark = pyfits.getdata("Calibs/dark.fits")
dark = dark[0:2048,0:2048]
#Read in flats
J_flat = pyfits.getdata('Calibs/New_J_Flat.fits')
K_flat = pyfits.getdata('Calibs/New_K_Flat.fits')
#Read in data from "whirc_info.dat" (the observation log)
im1 = open('whirc_info.dat','r')
data1 = im1.readlines()
im1.close()
filename = []
dateobs = []
objname = []
imgtype = []
ra = []
dec = []
exptime = []
usable = []
rotangle = []
raoffset = []
decoffset = []
for line in data1:
p = line.split()
filename.append(p[0])
dateobs.append(p[1])
objname.append(p[2])
imgtype.append(p[3])
ra.append(p[4])
dec.append(p[5])
exptime.append(p[6])
usable.append(p[7])
rotangle.append(p[8])
raoffset.append(p[9])
decoffset.append(p[10])
#Rewrite in a more convenient array with format array[line][element]
alldata = []
for line in range(len(usable)):
alldata.append([filename[line],dateobs[line],objname[line],imgtype[line],ra[line],dec[line],exptime[line],usable[line],rotangle[line],raoffset[line],decoffset[line]])
#Find junk files and good files:
junk = []
good = []
for line in range(len(alldata)):
if "no" in alldata[line][7]:
junk.append(alldata[line])
if "yes" in alldata[line][7]:
good.append(alldata[line])
#Find files related to obj_id in good files
aobj_id = []
for line in range(len(good)):
if str(obj_id) in good[line][2]:
aobj_id.append(good[line])
print str(len(aobj_id)),"files for "+str(obj_id)
#Find files through J_filter
aobj_idJ = []
for line in range(len(aobj_id)):
if "J" in aobj_id[line][2]:
aobj_idJ.append(aobj_id[line])
print str(len(aobj_idJ)), "J files for "+str(obj_id)
#Find files through K_filter
aobj_idK = []
for line in range(len(aobj_id)):
if "K" in aobj_id[line][2]:
aobj_idK.append(aobj_id[line])
print str(len(aobj_idK)), "K files for "+str(obj_id)
#Find J files from Night 1
aobj_idJN1 = []
for line in range(len(aobj_idJ)):
if "N1" in aobj_idJ[line][0]:
aobj_idJN1.append(aobj_idJ[line])
print str(len(aobj_idJN1)), "J files in N1"
#Fink K files from Night 1
aobj_idKN1 = []
for line in range(len(aobj_idK)):
if "N1" in aobj_idK[line][0]:
aobj_idKN1.append(aobj_idK[line])
print str(len(aobj_idKN1)), "K files in N1"
#Find J files from Night 2
aobj_idJN2 = []
for line in range(len(aobj_idJ)):
if "N2" in aobj_idJ[line][0]:
aobj_idJN2.append(aobj_idJ[line])
print str(len(aobj_idJN2)), "J files in N2"
#Find K files from Night 2
aobj_idKN2 = []
for line in range(len(aobj_idK)):
if "N2" in aobj_idK[line][0]:
aobj_idKN2.append(aobj_idK[line])
print str(len(aobj_idKN2)), "K files in N2"
#Find J files from Night 3
aobj_idJN3 = []
for line in range(len(aobj_idJ)):
if "N3" in aobj_idJ[line][0]:
aobj_idJN3.append(aobj_idJ[line])
print str(len(aobj_idJN3)), "J files in N3"
#Find K files from Night 3
aobj_idKN3 = []
for line in range(len(aobj_idK)):
if "N3" in aobj_idK[line][0]:
aobj_idKN3.append(aobj_idK[line])
print str(len(aobj_idKN3)), "K files in N3"
#WRITE PATH TO J_FILES_N1
Jobj_idN1_locs = []
for line in range(len(aobj_idJN1)):
Jobj_idN1_locs.append("Raw/"+str(aobj_idJN1[line][0]))
#WRITE PATH TO K_FILES_N1
Kobj_idN1_locs = []
for line in range(len(aobj_idKN1)):
Kobj_idN1_locs.append("Raw/"+str(aobj_idKN1[line][0]))
#WRITE PATH OF J_FILES_N2
Jobj_idN2_locs = []
for line in range(len(aobj_idJN2)):
Jobj_idN2_locs.append("Raw/"+str(aobj_idJN2[line][0]))
#WRITE PATH OF K_FILES_N2
Kobj_idN2_locs = []
for line in range(len(aobj_idKN2)):
Kobj_idN2_locs.append("Raw/"+str(aobj_idKN2[line][0]))
#WRITE PATH OF J_FILES_N3
Jobj_idN3_locs = []
for line in range(len(aobj_idJN3)):
Jobj_idN3_locs.append("Raw/"+str(aobj_idJN3[line][0]))
#WRITE PATH OF K_FILES_N3
Kobj_idN3_locs = []
for line in range(len(aobj_idKN3)):
Kobj_idN3_locs.append("Raw/"+str(aobj_idKN3[line][0]))
#READ IN SKY FILES
#READ IN SCIENCE EXPOSURES
#J_filter, Night 1
print "reading in JN1..."
if len(aobj_idJN1) > 0:
skyJN1 = pyfits.getdata('Calibs/sky/sky_'+str(obj_id)+'J_N1.fits')
for line in range(len(aobj_idJN1)):
sci = pyfits.getdata(Jobj_idN1_locs[line])[0:2048,0:2048]
reduced_sci = (sci - bias - dark - skyJN1)/J_flat
file = pyfits.PrimaryHDU(reduced_sci)
file.writeto('Calibs/reduced/'+str(obj_id)+'_'+'JN1_'+str(line + 1)+'.fits', clobber = True)
array, header = cosmics.fromfits('Calibs/reduced/'+str(obj_id)+'_'+'JN1_'+str(line + 1)+'.fits')
c = cosmics.cosmicsimage(array)
c.run(maxiter = 1) # can increase up to 4 to improve precision, but takes longer
cosmics.tofits('Calibs/reduced/'+str(obj_id)+'_'+'JN1_cos'+str(line + 1)+'.fits', c.cleanarray, header)
#K_filter, Night 1
print "reading in KN1..."
if len(aobj_idKN1) > 0:
skyKN1 = pyfits.getdata('Calibs/sky/sky_'+str(obj_id)+'K_N1.fits')
for line in range(len(aobj_idKN1)):
sci = pyfits.getdata(Kobj_idN1_locs[line])[0:2048,0:2048]
reduced_sci = (sci - bias - dark - skyKN1)/K_flat
file = pyfits.PrimaryHDU(reduced_sci)
file.writeto('Calibs/reduced/'+str(obj_id)+'_'+'KN1_'+str(line + 1)+'.fits', clobber = True)
array, header = cosmics.fromfits('Calibs/reduced/'+str(obj_id)+'_'+'KN1_'+str(line + 1)+'.fits')
c = cosmics.cosmicsimage(array)
c.run(maxiter = 1) # can increase up to 4 to improve precision, but takes longer
cosmics.tofits('Calibs/reduced/'+str(obj_id)+'_'+'KN1_cos'+str(line + 1)+'.fits', c.cleanarray, header)
#J_filter, Night 2
print "reading in JN2..."
if len(aobj_idJN2) > 0:
skyJN2 = pyfits.getdata('Calibs/sky/sky_'+str(obj_id)+'J_N2.fits')
for line in range(len(aobj_idJN2)):
sci = pyfits.getdata(Jobj_idN2_locs[line])[0:2048,0:2048]
reduced_sci = (sci - bias - dark - skyJN2)/J_flat
file = pyfits.PrimaryHDU(reduced_sci)
file.writeto('Calibs/reduced/'+str(obj_id)+'_'+'JN2_'+str(line + 1)+'.fits', clobber = True)
array, header = cosmics.fromfits('Calibs/reduced/'+str(obj_id)+'_'+'JN2_'+str(line + 1)+'.fits')
c = cosmics.cosmicsimage(array)
c.run(maxiter = 1) # can increase up to 4 to improve precision, but takes longer
cosmics.tofits('Calibs/reduced/'+str(obj_id)+'_'+'JN2_cos'+str(line + 1)+'.fits', c.cleanarray, header)
#K_filter, Night 2
print "reading in KN2..."
if len(aobj_idKN2) > 0:
skyKN2 = pyfits.getdata('Calibs/sky/sky_'+str(obj_id)+'K_N2.fits')
for line in range(len(aobj_idKN2)):
sci = pyfits.getdata(Kobj_idN2_locs[line])[0:2048,0:2048]
reduced_sci = (sci - bias - dark - skyKN2)/K_flat
file = pyfits.PrimaryHDU(reduced_sci)
file.writeto('Calibs/reduced/'+str(obj_id)+'_'+'KN2_'+str(line + 1)+'.fits', clobber = True)
array, header = cosmics.fromfits('Calibs/reduced/'+str(obj_id)+'_'+'KN2_'+str(line + 1)+'.fits')
c = cosmics.cosmicsimage(array)
c.run(maxiter = 1) # can increase up to 4 to improve precision, but takes longer
cosmics.tofits('Calibs/reduced/'+str(obj_id)+'_'+'KN2_cos'+str(line + 1)+'.fits', c.cleanarray, header)
#J_filter, Night 3
print "reading in JN3..."
if len(aobj_idJN3) > 0:
skyJN3 = pyfits.getdata('Calibs/sky/sky_'+str(obj_id)+'J_N3.fits')
for line in range(len(aobj_idJN3)):
sci = pyfits.getdata(Jobj_idN3_locs[line])[0:2048,0:2048]
reduced_sci = (sci - bias - dark - skyJN3)/J_flat
file = pyfits.PrimaryHDU(reduced_sci)
file.writeto('Calibs/reduced/'+str(obj_id)+'_'+'JN3_'+str(line + 1)+'.fits', clobber = True)
array, header = cosmics.fromfits('Calibs/reduced/'+str(obj_id)+'_'+'JN3_'+str(line + 1)+'.fits')
c = cosmics.cosmicsimage(array)
c.run(maxiter = 1) # can increase up to 4 to improve precision, but takes longer
cosmics.tofits('Calibs/reduced/'+str(obj_id)+'_'+'JN3_cos'+str(line + 1)+'.fits', c.cleanarray, header)
if len(aobj_idKN3) > 0:
skyKN3 = pyfits.getdata('Calibs/sky/sky_'+str(obj_id)+'K_N3.fits')
for line in range(len(aobj_idKN3)):
sci = pyfits.getdata(Kobj_idN3_locs[line])[0:2048,0:2048]
reduced_sci = (sci - bias - dark - skyKN3)/K_flat
file = pyfits.PrimaryHDU(reduced_sci)
file.writeto('Calibs/reduced/'+str(obj_id)+'_'+'KN3_'+str(line + 1)+'.fits', clobber = True)
array, header = cosmics.fromfits('Calibs/reduced/'+str(obj_id)+'_'+'KN3_'+str(line + 1)+'.fits')
c = cosmics.cosmicsimage(array)
c.run(maxiter = 1) # can increase up to 4 to improve precision, but takes longer
cosmics.tofits('Calibs/reduced/'+str(obj_id)+'_'+'KN3_cos'+str(line + 1)+'.fits', c.cleanarray, header)
#!/usr/bin/python
from dbauth import *
import cosmics
import os
file_name = 'a.fits'
# some comments on params are here http://obswww.unige.ch/~tewes/cosmics_dot_py/
pssl = 0.0
ccd_gain = 2.687
ccd_readnoise = 8.14
sigclip = 5.0
sigfrac = 0.2
objlim = 3.0
satlevel = 30000
maxiter = 10
verbose = True
(array, header) = cosmics.fromfits(file_name, verbose=verbose)
c = cosmics.cosmicsimage(array, pssl=pssl, gain=ccd_gain, readnoise=ccd_readnoise,
sigclip=sigclip, sigfrac=sigfrac, objlim=objlim,
satlevel=satlevel, verbose=verbose)
c.run(maxiter=maxiter, verbose=verbose)
#cleaned_file_name = "a_crcl.fits"
#cosmics.tofits(cleaned_file_name, c.cleanarray, header, verbose=verbose)
#c.make_plots('/disk/Work/Cosmic_rays/Code/CosmicRayExtractor/RP','a_crcl.png')
def clean_image(infile, true_rot=True, crop=True, wcs=None):
basefile = os.path.basename(infile)
cleanfile = os.path.join(DATA_DIR, basefile.replace('.fits','_clean.fits'))
maskfile = os.path.join(MASK_DIR, basefile.replace('.fits','_mask.fits'))
# Read the FITS :
array, header = cosmics.fromfits(infile)
# array is a 2D numpy array
if wcs is None:
try:
clean_image.wcs
except AttributeError:
#clean_image.wcs = WCS(header) # use the wcs coord of the first input
clean_image.wcs = header # use the wcs coord of the first input
# crop array to be a uniform shape (2048x2048 square)
# check the rotation angle on the sky
wcs = WCS(clean_image.wcs)
rotangle = header['ROTSKYPA']
# perform WCS transform
if true_rot: # use PIL to rotate the image
hdu = fits.open(infile)[0]
array = reproject_interp(hdu, clean_image.wcs)[0]
print "reprojected to", WCS(clean_image.wcs)
#im = Image.fromarray(array)
#if header['CCDRDOUT'] == 'DUMMYBOTTOMRIGHT':
# rotangle = rotangle+180
#rot_im = im.rotate(rotangle, expand=True,
# resample=INTERPOLATION)
#array = np.array(rot_im)
else: # use numpy to rotate to nearest 90 degrees
# find the number of requred rotations
rotangle = int(-rotangle/10+0.5)
k = (rotangle/9)
# make the number positive if necessary
k = (k+4) % 4
# and finally rotate the array
array = np.rot90(array, k=k, axes=(0,1))
if crop: # Helps ISIS interpolate more accurately
array = array[:2048,:2048]
# get parameters
gain = header['GAIN']
readnoise = header['READNOIS']
sigclip = 5.0
sigfrac = 0.3
objlim = 5.0
maxiter=4
# Build the object :
c = cosmics.cosmicsimage(array, gain=gain,
readnoise=readnoise, sigclip=sigclip,
sigfrac=sigfrac, objlim=objlim)
# There are other options, check the manual...
# Run the full artillery :
c.run(maxiter=maxiter)
header.update(wcs.to_header())
# update our header to describe cleaning
header['IC_PREF'] = 'IC CP'
header.comments['IC_PREF'] = 'IC: Image Correct Params, CP: Cosmics Params'
header['IC_DATE'] = date.today().isoformat()
header.comments['IC_DATE'] = 'Date corrections were made'
header['IC_VER'] = __version__
header.comments['IC_VER'] = 'Correction script version number'
header['IC_SCRPT'] = os.path.basename(__file__)
header.comments['IC_SCRPT'] = 'Name of script used for corrections'
header['IC_ATHR'] = __author__
header.comments['IC_ATHR'] = 'Correction script author'
header['IC_MOD'] = __date__
header.comments['IC_MOD'] = 'Date of last modification to correction script'
header['IC_MNTNR'] = __maintainer__
header.comments['IC_MNTNR'] = 'Correction script maintainer'
header['IC_EMAIL'] = __email__
header.comments['IC_EMAIL'] = 'Email of current correction script maintainer'
#header['IC_ROT'] = 'PIL.Image.rotate' if true_rot else 'numpy.rot90'
header['IC_ROT'] = 'WCS' if true_rot else 'numpy.rot90'
header.comments['IC_ROT'] = 'Rotation, PIL: exact, numpy: nearest 90 deg'
header['IC_CROP'] = crop
header.comments['IC_CROP'] = 'If the Image was cropped to 2048x2048'
header['IC_INTRP'] = 'BICUBIC' if true_rot else 'None'
header.comments['IC_INTRP'] = 'PIL rotation interpolation method'
header['CP_COSMC'] = "L.A.Cosmic"
header.comments['CP_COSMC'] = 'Package used for cosmic ray removal'
header['CP_SGCLP'] = sigclip
header.comments['CP_SGCLP'] = 'sigclip kwarg for cosmic ray removal'
header['CP_SGFRC'] = sigfrac
header.comments['CP_SGFRC'] = 'sigfrac kwarg for cosmic ray removal'
header['CP_OBJLM'] = objlim
header.comments['CP_OBJLM'] = 'objlim kwarg for cosmic ray removal'
header['CP_GAIN'] = gain
header.comments['CP_GAIN'] = 'gain kwarg for cosmic ray removal'
header['CP_RDNSE'] = readnoise
header.comments['CP_RDNSE'] = 'readnoise kwarg for cosmic ray removal'
header['CP_MITR'] = maxiter
header.comments['CP_MITR'] = 'maxiter kwarg for cosmic ray removal'
# Write the cleaned image into a new FITS file, conserving the original header :
cosmics.tofits(cleanfile, c.cleanarray, hdr=header, cols={'MASK':c.mask})
def reduce_data(filelist_new, filelist_masterflats, filelist_masterthars):
if len(filelist_new) == 0:
print("Please, choose science files to reduce first!")
elif len(filelist_masterflats) == 0:
print("Please, choose masterflat files first!")
elif len(filelist_masterthars) == 0:
print("Please, choose Thorium-Argon reference files first!")
else:
print("Starting reduction process...")
extracted_science_files = []
current_directory = os.getcwd()
years = [
'{:04d}'.format(year) for year in range(2005,
date.today().year + 1)
]
month = ['{:02d}'.format(mon) for mon in range(1, 13)]
for i in range(len(filelist_new)):
trim = trim_remove_bias(filelist_new[i])
chosen_masterflat = None
dummyI = trim.replace(".trim_will_be_removed.fits", ".dummyI.fits")
dummyII = trim.replace(".trim_will_be_removed.fits",
".dummyII.fits")
if os.path.exists(dummyI):
os.remove(dummyI)
if os.path.exists(dummyII):
os.remove(dummyII)
for y in years:
for mon in month:
date_check_1 = y + "_" + mon
date_check_2 = y + "-" + mon
if ((date_check_1 in trim)
or (date_check_2 in trim)) == True:
for masterflat in filelist_masterflats:
if ((date_check_1 in masterflat)
or (date_check_2 in masterflat)) == True:
chosen_masterflat = masterflat
if chosen_masterflat is None:
print("Masterflat File not found!!")
continue
iraf.imarith(trim, "/", chosen_masterflat, dummyI)
iraf.apscatter(dummyI,
output=dummyII,
interactive="No",
apertures="1-51",
references="find_orders.fits")
clean = trim.replace(".trim_will_be_removed.fits", ".clean.fits")
mask = trim.replace(".trim_will_be_removed.fits", ".mask.fits")
mean_image = iraf.imstat(dummyII,
Stdout=1,
fields="mean",
format="no")
print(mean_image)
if float(mean_image[0]) > 5000:
array, header = cosmics.fromfits(dummyII)
sigclip = 50.0
objlim = 50.0
c = cosmics.cosmicsimage(array,
gain=0.368,
readnoise=3.7,
sigclip=sigclip,
sigfrac=10.0,
objlim=objlim)
c.run(maxiter=0)
cosmics.tofits(clean, c.cleanarray, header)
cosmics.tofits(mask, c.mask, header)
elif 5000 > float(mean_image[0]) > 1000:
array, header = cosmics.fromfits(dummyII)
sigclip = 20.0
objlim = 20.0
c = cosmics.cosmicsimage(array,
gain=0.368,
readnoise=3.7,
sigclip=sigclip,
sigfrac=4.0,
objlim=objlim)
c.run(maxiter=2)
cosmics.tofits(clean, c.cleanarray, header)
cosmics.tofits(mask, c.mask, header)
else:
array, header = cosmics.fromfits(dummyII)
sigclip = 5.0
objlim = 5.0
c = cosmics.cosmicsimage(array,
gain=0.368,
readnoise=3.7,
sigclip=sigclip,
sigfrac=1.8,
objlim=objlim)
c.run(maxiter=2)
cosmics.tofits(clean, c.cleanarray, header)
cosmics.tofits(mask, c.mask, header)
extract = trim.replace(".trim_will_be_removed.fits",
".extracted.fits")
extracted_science_files.append(extract)
iraf.apall(clean,
output=extract,
references="find_orders",
profiles="find_orders",
apertures="1-51")
# iraf.apall(clean, output=extract, references= "find_orders", profiles= "find_orders", apertures = "3-50" )
# iraf.apall(clean, output=extract, references= "find_orders", profiles= "find_orders", apertures = "3-50" )
os.remove(dummyI)
os.remove(dummyII)
os.remove(trim)
write_reftable(filelist_new, filelist_masterthars)
thar_directory = os.path.dirname(filelist_masterthars[0])
if thar_directory == os.path.dirname(filelist_new[0]):
pass
else:
remove_file(thar_directory, "science.lis")
remove_file(thar_directory, "reftable.dat")
shutil.move("science.lis", thar_directory)
shutil.move("reftable.dat", thar_directory)
os.chdir(thar_directory)
iraf.refspectra("@science.lis",
references="reftable.dat",
ignoreaps="Yes",
sort="",
group="",
override="yes",
confirm="no",
assign="yes")
iraf.dispcor("@science.lis", output="@science.lis")
if thar_directory == os.path.dirname(filelist_new[0]):
pass
else:
remove_file(current_directory, "science.lis")
remove_file(current_directory, "reftable.dat")
shutil.move("science.lis", current_directory)
shutil.move("reftable.dat", current_directory)
os.chdir(current_directory)
normalize_and_merge(extracted_science_files)
print("Success! Reduction process complete.")
