def imred(rawdir, prodir, cleanup=True):
print rawdir
print prodir
#get the name of the files
infile_list=glob.glob(rawdir+'*.fits')
infiles=','.join(['%s' % x for x in infile_list])
#get the current date for the files
obsdate=os.path.basename(infile_list[0])[1:9]
print obsdate
#set up some files that will be needed
logfile='imred'+obsdate+'.log'
flatimage='FLAT%s.fits' % (obsdate)
dbfile='spec%s.db' % obsdate
#create the observation log
obs_dict=obslog(infile_list)
#prepare the data
saltprepare(infiles, '', 'p', createvar=False, badpixelimage='', clobber=True, logfile=logfile, verbose=True)
#bias subtract the data
saltbias('pP*fits', '', 'b', subover=True, trim=True, subbias=False, masterbias='',
median=False, function='polynomial', order=5, rej_lo=3.0, rej_hi=5.0,
niter=10, plotover=False, turbo=False,
clobber=True, logfile=logfile, verbose=True)
#gain correct the data
saltgain('bpP*fits', '', 'g', usedb=False, mult=True, clobber=True, logfile=logfile, verbose=True)
#cross talk correct the data
saltxtalk('gbpP*fits', '', 'x', xtalkfile = "", usedb=False, clobber=True, logfile=logfile, verbose=True)
#cosmic ray clean the data
#only clean the object data
for i in range(len(infile_list)):
if obs_dict['CCDTYPE'][i].count('OBJECT') and obs_dict['INSTRUME'][i].count('RSS'):
img='xgbp'+os.path.basename(infile_list[i])
saltcrclean(img, img, '', crtype='edge', thresh=5, mbox=11, bthresh=5.0,
flux_ratio=0.2, bbox=25, gain=1.0, rdnoise=5.0, fthresh=5.0, bfactor=2,
gbox=3, maxiter=5, multithread=True, clobber=True, logfile=logfile, verbose=True)
#flat field correct the data
flat_imgs=''
for i in range(len(infile_list)):
if obs_dict['CCDTYPE'][i].count('FLAT'):
if flat_imgs: flat_imgs += ','
flat_imgs += 'xgbp'+os.path.basename(infile_list[i])
if len(flat_imgs)!=0:
saltcombine(flat_imgs,flatimage, method='median', reject=None, mask=False, \
weight=True, blank=0, scale='average', statsec='[200:300, 600:800]', lthresh=3, \
hthresh=3, clobber=True, logfile=logfile, verbose=True)
#saltillum(flatimage, flatimage, '', mbox=11, clobber=True, logfile=logfile, verbose=True)
saltflat('xgbpP*fits', '', 'f', flatimage, minflat=500, clobber=True, logfile=logfile, verbose=True)
else:
flats=None
imfiles=glob.glob('xgbpP*fits')
for f in imfiles:
shutil.copy(f, 'f'+f)
#mosaic the data
geomfile=iraf.osfn("pysalt$data/rss/RSSgeom.dat")
saltmosaic('fxgbpP*fits', '', 'm', geomfile, interp='linear', cleanup=True, geotran=True, clobber=True, logfile=logfile, verbose=True)
#clean up the images
if cleanup:
for f in glob.glob('p*fits'): os.remove(f)
for f in glob.glob('bp*fits'): os.remove(f)
for f in glob.glob('gbp*fits'): os.remove(f)
for f in glob.glob('xgbp*fits'): os.remove(f)
for f in glob.glob('fxgbp*fits'): os.remove(f)
def imred(infile_list, prodir, bpmfile=None, gaindb = None, cleanup=True):
#get the name of the files
infiles=','.join(['%s' % x for x in infile_list])
#get the current date for the files
obsdate=os.path.basename(infile_list[0])[1:9]
print obsdate
#set up some files that will be needed
logfile='im'+obsdate+'.log'
flatimage='FLAT%s.fits' % (obsdate)
dbfile='spec%s.db' % obsdate
#create the observation log
obs_dict=obslog(infile_list)
with logging(logfile, debug) as log:
log.message('Pysalt Version: '+pysalt.verno, with_header=False)
#prepare the data
saltprepare(infiles, '', 'p', createvar=False, badpixelimage='', clobber=True, logfile=logfile, verbose=True)
for img in infile_list:
hdu = pyfits.open('p'+os.path.basename(img), 'update')
# for backwards compatibility
if not hdu[1].header.has_key('XTALK'):
hdu[1].header.update('XTALK',1474)
hdu[2].header.update('XTALK',1474)
hdu[3].header.update('XTALK',1166)
hdu[4].header.update('XTALK',1111)
hdu[5].header.update('XTALK',1377)
hdu[6].header.update('XTALK',1377)
hdu.close()
#bias subtract the data
saltbias('pP*fits', '', 'b', subover=True, trim=True, subbias=False, masterbias='',
median=False, function='polynomial', order=5, rej_lo=3.0, rej_hi=5.0,
niter=10, plotover=False, turbo=False,
clobber=True, logfile=logfile, verbose=True)
add_variance('bpP*fits', bpmfile)
#gain correct the data
usedb = False
if gaindb: usedb = True
saltgain('bpP*fits', '', 'g', gaindb=gaindb, usedb=usedb, mult=True, clobber=True, logfile=logfile, verbose=True)
#cross talk correct the data
saltxtalk('gbpP*fits', '', 'x', xtalkfile = "", usedb=False, clobber=True, logfile=logfile, verbose=True)
#cosmic ray clean the data
#only clean the object data
for i in range(len(infile_list)):
if (obs_dict['CCDTYPE'][i].count('OBJECT') \
and obs_dict['LAMPID'][i].count('NONE') \
and obs_dict['INSTRUME'][i].count('RSS')):
img='xgbp'+os.path.basename(infile_list[i])
saltcrclean(img, img, '', crtype='edge', thresh=5, mbox=11, bthresh=5.0,
flux_ratio=0.2, bbox=25, gain=1.0, rdnoise=5.0, fthresh=5.0, bfactor=2,
gbox=3, maxiter=5, multithread=True, clobber=True, logfile=logfile, verbose=True)
#mosaic the data
#khn: attempt to use most recent previous geometry to obsdate.
#NOTE: mosaicing does not do this correctly
#geomdb = open(datadir+"RSSgeom.dat",'r')
#for geomline in geomdb:
# if geomline[0]=='#': continue
# if (int(obsdate) > int(geomline.split(' ')[0].replace('-',''))): break
#geomfile = "RSSgeom_obsdate.dat"
#open(geomfile,'w').write(geomline)
geomfile=iraf.osfn("pysalt$data/rss/RSSgeom.dat")
try:
saltmosaic('xgbpP*fits', '', 'm', geomfile, interp='linear', cleanup=True, geotran=True, clobber=True, logfile=logfile, verbose=True)
except:
saltmosaic('xgbpP*fits', '', 'm', geomfile, interp='linear', cleanup=True, geotran=True, clobber=True, logfile=logfile, verbose=True)
#khn: fix mosaiced VAR and BPM extensions
#khn: fix mosaiced bpm missing some of gap
for img in infile_list:
filename = 'mxgbp'+os.path.basename(img)
hdu = pyfits.open(filename, 'update')
hdu[2].header.update('EXTNAME','VAR')
hdu[3].header.update('EXTNAME','BPM')
bpm_rc = (hdu[3].data>0).astype('uint8')
zeroscicol = hdu['SCI'].data.sum(axis=0) == 0
bpmgapcol = bpm_rc.mean(axis=0) == 1
addbpmcol = zeroscicol & ~bpmgapcol
addbpmcol[np.argmax(addbpmcol)-4:np.argmax(addbpmcol)] = True # allow for chip tilt
bpm_rc[:,addbpmcol] = 1
hdu[3].data = bpm_rc
hdu.writeto(filename,clobber=True)
#clean up the images
if cleanup:
for f in glob.glob('p*fits'): os.remove(f)
for f in glob.glob('bp*fits'): os.remove(f)
for f in glob.glob('gbp*fits'): os.remove(f)
for f in glob.glob('xgbp*fits'): os.remove(f)
def science_red(rawdir, prodir, imreduce=True, specreduce=True, bpmfile=None, calfile=None, lampfile='Ar', automethod='Matchlines', skysection=[800,1000], cleanup=True):
print rawdir
print prodir
#get the name of the files
infile_list=glob.glob(rawdir+'P*.fits')
infiles=','.join(['%s' % x for x in infile_list])
#get the current date for the files
obsdate=os.path.basename(infile_list[0])[1:9]
print obsdate
#set up some files that will be needed
logfile='spec'+obsdate+'.log'
flatimage='FLAT%s.fits' % (obsdate)
dbfile='spec%s.db' % obsdate
#create the observation log
obs_dict=obslog(infile_list)
if imreduce:
#prepare the data
saltprepare(infiles, '', 'p', createvar=False, badpixelimage='', clobber=True, logfile=logfile, verbose=True)
#bias subtract the data
saltbias('pP*fits', '', 'b', subover=True, trim=True, subbias=False, masterbias='',
median=False, function='polynomial', order=5, rej_lo=3.0, rej_hi=5.0,
niter=10, plotover=False, turbo=False,
clobber=True, logfile=logfile, verbose=True)
add_variance('bpP*fits', bpmfile)
#gain correct the data
saltgain('bpP*fits', '', 'g', usedb=False, mult=True, clobber=True, logfile=logfile, verbose=True)
#cross talk correct the data
saltxtalk('gbpP*fits', '', 'x', xtalkfile = "", usedb=False, clobber=True, logfile=logfile, verbose=True)
#flat field correct the data
flat_imgs=''
for i in range(len(infile_list)):
if obs_dict['CCDTYPE'][i].count('FLAT'):
if flat_imgs: flat_imgs += ','
flat_imgs += 'xgbp'+os.path.basename(infile_list[i])
if 0: #len(flat_imgs)!=0:
saltcombine(flat_imgs,flatimage, method='median', reject=None, mask=False, \
weight=False, blank=0, scale=None, statsec='[200:300, 600:800]', lthresh=3, \
hthresh=3, clobber=True, logfile=logfile, verbose=True)
saltillum(flatimage, flatimage, '', mbox=11, clobber=True, logfile=logfile, verbose=True)
saltflat('xgbpP*fits', '', 'f', flatimage, minflat=0.8, allext=False, clobber=True, logfile=logfile, verbose=True)
else:
flats=None
imfiles=glob.glob('xgbpP*fits')
for f in imfiles:
shutil.copy(f, 'f'+f)
#cosmic ray clean the data
#only clean the object data
for i in range(len(infile_list)):
if obs_dict['CCDTYPE'][i].count('OBJECT') and obs_dict['INSTRUME'][i].count('RSS'):
img='fxgbp'+os.path.basename(infile_list[i])
saltcrclean(img, img, '', crtype='edge', thresh=5, mbox=11, bthresh=5.0,
flux_ratio=0.2, bbox=25, gain=1.0, rdnoise=5.0, fthresh=5.0, bfactor=2,
gbox=3, maxiter=5, multithread=True, clobber=True, logfile=logfile, verbose=True)
#mosaic the data
geomfile=iraf.osfn("pysalt$data/rss/RSSgeom.dat")
saltmosaic('fxgbpP*fits', '', 'm', geomfile, interp='linear', cleanup=True, geotran=True, clobber=True, logfile=logfile, verbose=True)
#clean up the images
if cleanup:
for f in glob.glob('p*fits'): os.remove(f)
for f in glob.glob('bp*fits'): os.remove(f)
for f in glob.glob('gbp*fits'): os.remove(f)
for f in glob.glob('xgbp*fits'): os.remove(f)
for f in glob.glob('fxgbp*fits'): os.remove(f)
#set up the name of the images
if specreduce:
for i in range(len(infile_list)):
if obs_dict['OBJECT'][i].upper().strip()=='ARC':
lamp=obs_dict['LAMPID'][i].strip().replace(' ', '')
arcimage='mfxgbp'+os.path.basename(infile_list[i])
#lampfile=iraf.osfn("pysalt$data/linelists/%s.salt" % lamp)
specidentify(arcimage, lampfile, dbfile, guesstype='rss',
guessfile='', automethod=automethod, function='legendre', order=3,
rstep=100, rstart='middlerow', mdiff=50, thresh=2, niter=5, smooth=3,
inter=True, clobber=True, logfile=logfile, verbose=True)
specrectify(arcimage, outimages='', outpref='x', solfile=dbfile, caltype='line',
function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None,
blank=0.0, clobber=True, logfile=logfile, verbose=True)
objimages=''
for i in range(len(infile_list)):
if obs_dict['CCDTYPE'][i].count('OBJECT') and obs_dict['INSTRUME'][i].count('RSS'):
if objimages: objimages += ','
objimages+='mfxgbp'+os.path.basename(infile_list[i])
if specreduce:
#run specidentify on the arc files
specrectify(objimages, outimages='', outpref='x', solfile=dbfile, caltype='line',
function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None,
blank=0.0, clobber=True, logfile=logfile, verbose=True)
return
def imred(rawdir, prodir, cleanup=True):
print rawdir
print prodir
#get the name of the files
infile_list = glob.glob(rawdir + '*.fits')
infiles = ','.join(['%s' % x for x in infile_list])
#get the current date for the files
obsdate = os.path.basename(infile_list[0])[1:9]
print obsdate
#set up some files that will be needed
logfile = 'imred' + obsdate + '.log'
flatimage = 'FLAT%s.fits' % (obsdate)
dbfile = 'spec%s.db' % obsdate
#create the observation log
obs_dict = obslog(infile_list)
#prepare the data
saltprepare(infiles,
'',
'p',
createvar=False,
badpixelimage='',
clobber=True,
logfile=logfile,
verbose=True)
#bias subtract the data
saltbias('pP*fits',
'',
'b',
subover=True,
trim=True,
subbias=False,
masterbias='',
median=False,
function='polynomial',
order=5,
rej_lo=3.0,
rej_hi=5.0,
niter=10,
plotover=False,
turbo=False,
clobber=True,
logfile=logfile,
verbose=True)
#gain correct the data
saltgain('bpP*fits',
'',
'g',
usedb=False,
mult=True,
clobber=True,
logfile=logfile,
verbose=True)
#cross talk correct the data
saltxtalk('gbpP*fits',
'',
'x',
xtalkfile="",
usedb=False,
clobber=True,
logfile=logfile,
verbose=True)
#cosmic ray clean the data
#only clean the object data
for i in range(len(infile_list)):
if obs_dict['CCDTYPE'][i].count(
'OBJECT') and obs_dict['INSTRUME'][i].count('RSS'):
img = 'xgbp' + os.path.basename(infile_list[i])
saltcrclean(img,
img,
'',
crtype='edge',
thresh=5,
mbox=11,
bthresh=5.0,
flux_ratio=0.2,
bbox=25,
gain=1.0,
rdnoise=5.0,
fthresh=5.0,
bfactor=2,
gbox=3,
maxiter=5,
multithread=True,
clobber=True,
logfile=logfile,
verbose=True)
#flat field correct the data
flat_imgs = ''
for i in range(len(infile_list)):
if obs_dict['CCDTYPE'][i].count('FLAT'):
if flat_imgs: flat_imgs += ','
flat_imgs += 'xgbp' + os.path.basename(infile_list[i])
if len(flat_imgs) != 0:
saltcombine(flat_imgs,flatimage, method='median', reject=None, mask=False, \
weight=True, blank=0, scale='average', statsec='[200:300, 600:800]', lthresh=3, \
hthresh=3, clobber=True, logfile=logfile, verbose=True)
#saltillum(flatimage, flatimage, '', mbox=11, clobber=True, logfile=logfile, verbose=True)
saltflat('xgbpP*fits',
'',
'f',
flatimage,
minflat=500,
clobber=True,
logfile=logfile,
verbose=True)
else:
flats = None
imfiles = glob.glob('xgbpP*fits')
for f in imfiles:
shutil.copy(f, 'f' + f)
#mosaic the data
geomfile = iraf.osfn("pysalt$data/rss/RSSgeom.dat")
saltmosaic('fxgbpP*fits',
'',
'm',
geomfile,
interp='linear',
cleanup=True,
geotran=True,
clobber=True,
logfile=logfile,
verbose=True)
#clean up the images
if cleanup:
for f in glob.glob('p*fits'):
os.remove(f)
for f in glob.glob('bp*fits'):
os.remove(f)
for f in glob.glob('gbp*fits'):
os.remove(f)
for f in glob.glob('xgbp*fits'):
os.remove(f)
for f in glob.glob('fxgbp*fits'):
os.remove(f)
def specred(rawdir, prodir, imreduce=True, specreduce=True, calfile=None, lamp='Ar', automethod='Matchlines', skysection=[800,1000], cleanup=True):
print rawdir
print prodir
#get the name of the files
infile_list=glob.glob(rawdir+'*.fits')
infiles=','.join(['%s' % x for x in infile_list])
#get the current date for the files
obsdate=os.path.basename(infile_list[0])[1:9]
print obsdate
#set up some files that will be needed
logfile='spec'+obsdate+'.log'
flatimage='FLAT%s.fits' % (obsdate)
dbfile='spec%s.db' % obsdate
#create the observation log
obs_dict=obslog(infile_list)
if imreduce:
#prepare the data
saltprepare(infiles, '', 'p', createvar=False, badpixelimage='', clobber=True, logfile=logfile, verbose=True)
#bias subtract the data
saltbias('pP*fits', '', 'b', subover=True, trim=True, subbias=False, masterbias='',
median=False, function='polynomial', order=5, rej_lo=3.0, rej_hi=5.0,
niter=10, plotover=False, turbo=False,
clobber=True, logfile=logfile, verbose=True)
#gain correct the data
saltgain('bpP*fits', '', 'g', usedb=False, mult=True, clobber=True, logfile=logfile, verbose=True)
#cross talk correct the data
saltxtalk('gbpP*fits', '', 'x', xtalkfile = "", usedb=False, clobber=True, logfile=logfile, verbose=True)
#cosmic ray clean the data
#only clean the object data
for i in range(len(infile_list)):
if obs_dict['CCDTYPE'][i].count('OBJECT') and obs_dict['INSTRUME'][i].count('RSS'):
img='xgbp'+os.path.basename(infile_list[i])
saltcrclean(img, img, '', crtype='edge', thresh=5, mbox=11, bthresh=5.0,
flux_ratio=0.2, bbox=25, gain=1.0, rdnoise=5.0, fthresh=5.0, bfactor=2,
gbox=3, maxiter=5, multithread=True, clobber=True, logfile=logfile, verbose=True)
#flat field correct the data
flat_imgs=''
for i in range(len(infile_list)):
if obs_dict['CCDTYPE'][i].count('FLAT'):
if flat_imgs: flat_imgs += ','
flat_imgs += 'xgbp'+os.path.basename(infile_list[i])
if len(flat_imgs)!=0:
saltcombine(flat_imgs,flatimage, method='median', reject=None, mask=False, \
weight=True, blank=0, scale='average', statsec='[200:300, 600:800]', lthresh=3, \
hthresh=3, clobber=True, logfile=logfile, verbose=True)
saltillum(flatimage, flatimage, '', mbox=11, clobber=True, logfile=logfile, verbose=True)
saltflat('xgbpP*fits', '', 'f', flatimage, minflat=500, clobber=True, logfile=logfile, verbose=True)
else:
flats=None
imfiles=glob.glob('cxgbpP*fits')
for f in imfiles:
shutil.copy(f, 'f'+f)
#mosaic the data
geomfile=iraf.osfn("pysalt$data/rss/RSSgeom.dat")
saltmosaic('fxgbpP*fits', '', 'm', geomfile, interp='linear', cleanup=True, geotran=True, clobber=True, logfile=logfile, verbose=True)
#clean up the images
if cleanup:
for f in glob.glob('p*fits'): os.remove(f)
for f in glob.glob('bp*fits'): os.remove(f)
for f in glob.glob('gbp*fits'): os.remove(f)
for f in glob.glob('xgbp*fits'): os.remove(f)
for f in glob.glob('fxgbp*fits'): os.remove(f)
#set up the name of the images
if specreduce:
for i in range(len(infile_list)):
if obs_dict['OBJECT'][i].upper().strip()=='ARC':
lamp=obs_dict['LAMPID'][i].strip().replace(' ', '')
arcimage='mfxgbp'+os.path.basename(infile_list[i])
lampfile=iraf.osfn("pysalt$data/linelists/%s.txt" % lamp)
specidentify(arcimage, lampfile, dbfile, guesstype='rss',
guessfile='', automethod=automethod, function='legendre', order=5,
rstep=100, rstart='middlerow', mdiff=10, thresh=3, niter=5,
inter=True, clobber=True, logfile=logfile, verbose=True)
specrectify(arcimage, outimages='', outpref='x', solfile=dbfile, caltype='line',
function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None,
blank=0.0, clobber=True, logfile=logfile, verbose=True)
objimages=''
for i in range(len(infile_list)):
if obs_dict['CCDTYPE'][i].count('OBJECT') and obs_dict['INSTRUME'][i].count('RSS'):
if objimages: objimages += ','
objimages+='mfxgbp'+os.path.basename(infile_list[i])
if specreduce:
#run specidentify on the arc files
specrectify(objimages, outimages='', outpref='x', solfile=dbfile, caltype='line',
function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None,
blank=0.0, clobber=True, logfile=logfile, verbose=True)
#create the spectra text files for all of our objects
spec_list=[]
for img in objimages.split(','):
spec_list.extend(createspectra('x'+img, obsdate, smooth=False, skysection=skysection, clobber=True))
print spec_list
#determine the spectrophotometric standard
extfile=iraf.osfn('pysalt$data/site/suth_extinct.dat')
for spec, am, et, pc in spec_list:
if pc=='CAL_SPST':
stdstar=spec.split('.')[0]
print stdstar, am, et
stdfile=iraf.osfn('pysalt$data/standards/spectroscopic/m%s.dat' % stdstar.lower().replace('-', '_'))
print stdfile
ofile=spec.replace('txt', 'sens')
calfile=ofile #assumes only one observations of a SP standard
specsens(spec, ofile, stdfile, extfile, airmass=am, exptime=et,
stdzp=3.68e-20, function='polynomial', order=3, thresh=3, niter=5,
clobber=True, logfile='salt.log',verbose=True)
for spec, am, et, pc in spec_list:
if pc!='CAL_SPST':
ofile=spec.replace('txt', 'spec')
speccal(spec, ofile, calfile, extfile, airmass=am, exptime=et,
clobber=True, logfile='salt.log',verbose=True)
#clean up the spectra for bad pixels
cleanspectra(ofile)
