def saltcombine(images,outimage, method='average', reject=None, mask=True, \
weight=True, blank=0, scale=None, statsec=None, lthresh=3, \
hthresh=3, clobber=False, logfile='salt.log',verbose=True):
with logging(logfile,debug) as log:
# Check the input images
infiles = saltio.argunpack ('Input',images)
#set reject as None
reject=saltio.checkfornone(reject)
if reject is not None: reject=reject.lower()
#set scale
scale=saltio.checkfornone(scale)
if scale is not None: scale=scale.lower()
#set statsec
statsec=saltio.checkfornone(statsec)
statsec=saltio.getSection(statsec, iraf_format=True)
#Fast combine the images
outstruct=imcombine(infiles, method=method, reject=reject, mask=mask, \
weight=weight, blank=blank, scale=scale, \
statsec=statsec, lthresh=lthresh, hthresh=hthresh)
# housekeeping keywords
fname, hist=history(level=1, wrap=False)
saltkey.housekeeping(outstruct[0],'SCOMBINE', 'File Combined by SALTCOMBINE', hist)
# write FITS file
saltio.writefits(outstruct, outimage)
def saltcombine(images,outimage, method='average', reject=None, mask=True, \
weight=True, blank=0, scale=None, statsec=None, lthresh=3, \
hthresh=3, clobber=False, logfile='salt.log',verbose=True):
with logging(logfile, debug) as log:
# Check the input images
infiles = saltio.argunpack('Input', images)
#set reject as None
reject = saltio.checkfornone(reject)
if reject is not None: reject = reject.lower()
#set scale
scale = saltio.checkfornone(scale)
if scale is not None: scale = scale.lower()
#set statsec
statsec = saltio.checkfornone(statsec)
statsec = saltio.getSection(statsec, iraf_format=True)
#Fast combine the images
outstruct=imcombine(infiles, method=method, reject=reject, mask=mask, \
weight=weight, blank=blank, scale=scale, \
statsec=statsec, lthresh=lthresh, hthresh=hthresh)
# housekeeping keywords
fname, hist = history(level=1, wrap=False)
saltkey.housekeeping(outstruct[0], 'SCOMBINE',
'File Combined by SALTCOMBINE', hist)
# write FITS file
saltio.writefits(outstruct, outimage)
def specsens(specfile, outfile, stdfile, extfile, airmass=None, exptime=None,
stdzp=3.68e-20, function='polynomial', order=3, thresh=3, niter=5,
clobber=True, logfile='salt.log',verbose=True):
with logging(logfile,debug) as log:
#read in the specfile and create a spectrum object
obs_spectra=st.readspectrum(specfile, error=True, ftype='ascii')
#read in the std file and convert from magnitudes to fnu
#then convert it to fwave (ergs/s/cm2/A)
std_spectra=st.readspectrum(stdfile, error=False, ftype='ascii')
std_spectra.flux=Spectrum.magtoflux(std_spectra.flux, stdzp)
std_spectra.flux=Spectrum.fnutofwave(std_spectra.wavelength, std_spectra.flux)
#read in the extinction file (leave in magnitudes)
ext_spectra=st.readspectrum(extfile, error=False, ftype='ascii')
#determine the airmass if not specified
if saltio.checkfornone(airmass) is None:
message='Airmass was not supplied'
raise SALTSpecError(message)
#determine the exptime if not specified
if saltio.checkfornone(airmass) is None:
message='Exposure Time was not supplied'
raise SALTSpecError(message)
#calculate the calibrated spectra
log.message('Calculating the calibration curve for %s' % specfile)
cal_spectra=sensfunc(obs_spectra, std_spectra, ext_spectra, airmass, exptime)
#fit the spectra--first take a first cut of the spectra
#using the median absolute deviation to throw away bad points
cmed=np.median(cal_spectra.flux)
cmad=saltstat.mad(cal_spectra.flux)
mask=(abs(cal_spectra.flux-cmed)<thresh*cmad)
mask=(cal_spectra.flux>0)
#now fit the data
fit=interfit(cal_spectra.wavelength[mask], cal_spectra.flux[mask], function=function, order=order, thresh=thresh, niter=niter)
fit.interfit()
#print 'plotting...'
#figure()
#plot(cal_spectra.wavelength, cal_spectra.flux)
#plot(obs_spectra.wavelength, obs_spectra.flux*cal_spectra.flux.mean()/obs_spectra.flux.mean())
#plot(std_spectra.wavelength, std_spectra.flux*cal_spectra.flux.mean()/std_spectra.flux.mean())
#plot(cal_spectra.wavelength, fit(cal_spectra.wavelength))
#show()
#write the spectra out
cal_spectra.flux=fit(cal_spectra.wavelength)
st.writespectrum(cal_spectra, outfile, ftype='ascii')
def quickspec(profile, lampid=None, findobj=False, objsection=None, skysection=None, clobber=False, logfile='saltclean.log', verbose=True):
"""From mosaicked data, produce wavelength calibrated files"""
struct=pyfits.open(profile)
xlen=len(struct[1].data[0])
ylen=len(struct[1].data)
print xlen,ylen
badpixelimage=None
caltype='rss'
function='polynomial'
order=3
skysub=True
if saltsafeio.checkfornone(lampid): skysub=False
if objsection is None:
y1=0.5*ylen-0.05*ylen
y2=0.5*ylen+0.05*ylen
objsection='[%i:%i]' % (y1,y2)
else:
y1=int(objsection[1:].split(':')[0])
y2=int(objsection[:-1].split(':')[1])
if skysection is None:
skysection='%i:%i' % (y2+0.1*ylen,y2+0.2*ylen)
thresh=3.0
logfile='saltspec.log'
specreduce(images=profile,badpixelimage=badpixelimage,caltype=caltype,
function=function, order=order, skysub=skysub, skysection=skysection,
findobj=findobj, objsection=objsection, thresh=thresh,
clobber=clobber, logfile=logfile, verbose=verbose)
return y1,y2
def saltprepare(images,outimages,outpref,createvar=False, badpixelimage=None, clobber=True,logfile='salt.log',verbose=True):
with logging(logfile,debug) as log:
# Check the input images
infiles = saltio.argunpack ('Input',images)
# create list of output files
outfiles=saltio.listparse('Outfile', outimages, outpref,infiles,'')
#verify that the input and output lists are the same length
saltio.comparelists(infiles,outfiles,'Input','output')
# open the badpixel image
if saltio.checkfornone(badpixelimage) is None:
badpixelstruct=None
else:
try:
badpixelstruct = saltio.openfits(badpixelimage)
except saltio.SaltIOError,e:
msg='badpixel image must be specificied\n %s' % e
raise SaltError(msg)
# open each raw image file
for img, oimg, in zip(infiles, outfiles):
#open the fits file
struct=saltio.openfits(img)
#if createvar, throw a warning if it is using slotmode
if saltkey.fastmode(saltkey.get('DETMODE', struct[0])) and createvar:
msg='Creating variance frames for slotmode data in %s' % img
log.warning(msg)
# identify instrument
instrume,keyprep,keygain,keybias,keyxtalk,keyslot = saltkey.instrumid(struct)
# has file been prepared already?
try:
key = struct[0].header[keyprep]
message = 'ERROR -- SALTPREPARE: File ' + infile
message += ' has already been prepared'
raise SaltError(message)
except:
pass
# prepare file
struct=prepare(struct,createvar=createvar, badpixelstruct=badpixelstruct)
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],keyprep, 'File prepared for IRAF', hist)
# write FITS file
saltio.writefits(struct,oimg, clobber=clobber)
saltio.closefits(struct)
message = 'SALTPREPARE -- %s => %s' % (img, oimg)
log.message(message, with_header=False)
def checkforpropid(image,propids):
"""subroutine to check to see if the propid keyword exists
returns status
"""
#open up the image
struct=pyfits.open(image, mode='update')
if struct:
#get proposal code
propid=saltkey.get('PROPID', struct[0])
#check to see if it is none
if saltio.checkfornone(propid) is None or propid is 'None':
message='\nNo value in PROPID keyword for %s' % image
raise SaltError(message)
#check to see if it is an eng or cal proposal
if propid.count('ENG_') or propid.count('CAL_'):
return
#clean up junk ones
if propid in ['BIAS','COMMON','JUNK','TEST','UNKNOWN']:
return
#check to see if it is in the sdb
if propid not in propids:
message='\n%s for PROPID keyword for %s is invalid' % (propid, image)
raise SaltError(message)
def saltprepare(images,outimages,outpref,createvar=False, badpixelimage=None, clobber=True,logfile='salt.log',verbose=True):
with logging(logfile,debug) as log:
# Check the input images
infiles = saltio.argunpack ('Input',images)
# create list of output files
outfiles=saltio.listparse('Outfile', outimages, outpref,infiles,'')
#verify that the input and output lists are the same length
saltio.comparelists(infiles,outfiles,'Input','output')
# open the badpixel image
if saltio.checkfornone(badpixelimage) is None:
badpixelstruct=None
else:
try:
badpixelstruct = saltio.openfits(badpixelimage)
except saltio.SaltIOError,e:
msg='badpixel image must be specificied\n %s' % e
raise SaltError(msg)
# open each raw image file
for img, oimg, in zip(infiles, outfiles):
#open the fits file
struct=saltio.openfits(img)
#if createvar, throw a warning if it is using slotmode
if saltkey.fastmode(saltkey.get('DETMODE', struct[0])) and createvar:
msg='Creating variance frames for slotmode data in %s' % img
log.warning(msg)
# identify instrument
instrume,keyprep,keygain,keybias,keyxtalk,keyslot = saltkey.instrumid(struct)
# has file been prepared already?
try:
key = struct[0].header[keyprep]
message = 'ERROR -- SALTPREPARE: File ' + infile
message += ' has already been prepared'
raise SaltError(message)
except:
pass
# prepare file
struct=prepare(struct,createvar=createvar, badpixelstruct=badpixelstruct)
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],keyprep, 'File prepared for IRAF', hist)
# write FITS file
saltio.writefits(struct,oimg, clobber=clobber)
saltio.closefits(struct)
message = 'SALTPREPARE -- %s => %s' % (img, oimg)
log.message(message, with_header=False)
def specselfid(images, outimages, outpref, refimage=None, ystart='middlerow',
rstep=3, clobber=False, logfile='salt.log', verbose=True):
with logging(logfile, debug) as log:
# set up the variables
infiles = []
outfiles = []
# Check the input images
infiles = saltio.argunpack('Input', images)
# create list of output files
outfiles = saltio.listparse(
'Outimages',
outimages,
outpref,
infiles,
'')
# set up defaults
if saltio.checkfornone(refimage) is not None:
rhdu = saltio.openfits(refimage)
else:
refimage = None
# read in rectify each image
for img, oimg, in zip(infiles, outfiles):
hdu = saltio.openfits(img)
log.message(
'Performing self-identification and rectification on %s' %
img)
for i in range(1, len(hdu)):
if hdu[i].name == 'SCI':
if refimage is None:
sdata = hdu[i].data
else:
sdata = rhdu[i].data
hdu[i].data = selfid(
hdu[i].data,
sdata,
ystart=ystart,
rstep=rstep)
if saltkey.found('VAREXT', hdu[i]):
varext = saltkey.get('VAREXT', hdu[i])
hdu[varext].data = selfid(
hdu[varext].data,
sdata,
ystart=ystart,
rstep=rstep)
if saltkey.found('BPMEXT', hdu[i]):
bpmext = saltkey.get('BPMEXT', hdu[i])
hdu[bpmext].data = selfid(
hdu[bpmext].data,
sdata,
ystart=ystart,
rstep=rstep)
# write out the oimg
saltio.writefits(hdu, oimg, clobber=clobber)
def specselfid(images,
outimages,
outpref,
refimage=None,
ystart='middlerow',
rstep=3,
clobber=False,
logfile='salt.log',
verbose=True):
with logging(logfile, debug) as log:
# set up the variables
infiles = []
outfiles = []
# Check the input images
infiles = saltio.argunpack('Input', images)
# create list of output files
outfiles = saltio.listparse('Outimages', outimages, outpref, infiles,
'')
# set up defaults
if saltio.checkfornone(refimage) is not None:
rhdu = saltio.openfits(refimage)
else:
refimage = None
# read in rectify each image
for img, oimg, in zip(infiles, outfiles):
hdu = saltio.openfits(img)
log.message(
'Performing self-identification and rectification on %s' % img)
for i in range(1, len(hdu)):
if hdu[i].name == 'SCI':
if refimage is None:
sdata = hdu[i].data
else:
sdata = rhdu[i].data
hdu[i].data = selfid(hdu[i].data,
sdata,
ystart=ystart,
rstep=rstep)
if saltkey.found('VAREXT', hdu[i]):
varext = saltkey.get('VAREXT', hdu[i])
hdu[varext].data = selfid(hdu[varext].data,
sdata,
ystart=ystart,
rstep=rstep)
if saltkey.found('BPMEXT', hdu[i]):
bpmext = saltkey.get('BPMEXT', hdu[i])
hdu[bpmext].data = selfid(hdu[bpmext].data,
sdata,
ystart=ystart,
rstep=rstep)
# write out the oimg
saltio.writefits(hdu, oimg, clobber=clobber)
def specextract(images,
outfile,
method='normal',
section=None,
thresh=3.0,
minsize=3.0,
outformat='ascii',
ext=1,
convert=True,
clobber=True,
logfile='salt.log',
verbose=True):
with logging(logfile, debug) as log:
# Check the input images
infiles = saltio.argunpack('Input', images)
# create list of output files
outfiles = saltio.argunpack('outfile', outfile)
if method is 'weighted':
msg = 'This mode is not supported yet'
raise SALTSpecError(msg)
section = saltio.checkfornone(section)
if section is not None:
sections = saltio.getSection(section, iraf_format=False)
section = []
for i in range(0, len(sections), 2):
section.append((sections[i], sections[i + 1]))
# Identify the lines in each file
for img, ofile in zip(infiles, outfiles):
log.message('\nExtracting spectrum in image %s to %s' %
(img, ofile),
with_header=False,
with_stdout=verbose)
# open the images
hdu = saltio.openfits(img)
ap_list = extract(hdu,
ext=ext,
method=method,
section=section,
minsize=minsize,
thresh=thresh,
convert=convert)
# write the spectra out
if ap_list:
write_extract(ofile.strip(),
ap_list,
outformat=outformat,
clobber=clobber)
log.message('', with_header=False, with_stdout=verbose)
def speccal(specfile, outfile, calfile, extfile, airmass=None, exptime=None,
clobber=True, logfile='salt.log', verbose=True):
with logging(logfile, debug) as log:
# read in the specfile and create a spectrum object
obs_spectra = st.readspectrum(specfile, error=True, ftype='ascii')
# read in the std file and convert from magnitudes to fnu
# then convert it to fwave (ergs/s/cm2/A)
cal_spectra = st.readspectrum(calfile, error=False, ftype='ascii')
# read in the extinction file (leave in magnitudes)
ext_spectra = st.readspectrum(extfile, error=False, ftype='ascii')
# determine the airmass if not specified
if saltio.checkfornone(airmass) is None:
message = 'Airmass was not supplied'
raise SALTSpecError(message)
# determine the exptime if not specified
if saltio.checkfornone(airmass) is None:
message = 'Exposure Time was not supplied'
raise SALTSpecError(message)
# calculate the calibrated spectra
log.message('Calculating the calibration curve for %s' % specfile)
error = False
try:
if obs_spectra.var is not None:
error = True
except:
error = False
flux_spectra = calfunc(
obs_spectra,
cal_spectra,
ext_spectra,
airmass,
exptime,
error)
# write the spectra out
st.writespectrum(flux_spectra, outfile, ftype='ascii', error=error)
def specextract(
images,
outfile,
method="normal",
section=None,
thresh=3.0,
minsize=3.0,
outformat="ascii",
ext=1,
convert=True,
clobber=True,
logfile="salt.log",
verbose=True,
):
with logging(logfile, debug) as log:
# Check the input images
infiles = saltio.argunpack("Input", images)
# create list of output files
outfiles = saltio.argunpack("outfile", outfile)
if method is "weighted":
msg = "This mode is not supported yet"
raise SALTSpecError(msg)
section = saltio.checkfornone(section)
if section is not None:
sections = saltio.getSection(section, iraf_format=False)
section = []
for i in range(0, len(sections), 2):
section.append((sections[i], sections[i + 1]))
# Identify the lines in each file
for img, ofile in zip(infiles, outfiles):
log.message(
"\nExtracting spectrum in image %s to %s" % (img, ofile), with_header=False, with_stdout=verbose
)
# open the images
hdu = saltio.openfits(img)
ap_list = extract(
hdu, ext=ext, method=method, section=section, minsize=minsize, thresh=thresh, convert=convert
)
# write the spectra out
if ap_list:
write_extract(ofile.strip(), ap_list, outformat=outformat, clobber=clobber)
log.message("", with_header=False, with_stdout=verbose)
def speccal(
specfile, outfile, calfile, extfile, airmass=None, exptime=None, clobber=True, logfile="salt.log", verbose=True
):
with logging(logfile, debug) as log:
# read in the specfile and create a spectrum object
obs_spectra = st.readspectrum(specfile, error=True, ftype="ascii")
# read in the std file and convert from magnitudes to fnu
# then convert it to fwave (ergs/s/cm2/A)
cal_spectra = st.readspectrum(calfile, error=False, ftype="ascii")
# read in the extinction file (leave in magnitudes)
ext_spectra = st.readspectrum(extfile, error=False, ftype="ascii")
# determine the airmass if not specified
if saltio.checkfornone(airmass) is None:
message = "Airmass was not supplied"
raise SALTSpecError(message)
# determine the exptime if not specified
if saltio.checkfornone(airmass) is None:
message = "Exposure Time was not supplied"
raise SALTSpecError(message)
# calculate the calibrated spectra
log.message("Calculating the calibration curve for %s" % specfile)
error = False
try:
if obs_spectra.var is not None:
error = True
except:
error = False
flux_spectra = calfunc(obs_spectra, cal_spectra, ext_spectra, airmass, exptime, error)
# write the spectra out
st.writespectrum(flux_spectra, outfile, ftype="ascii", error=error)
def saltfpcalring(images,outfile, waves, method=None, thresh=5, minsize=10, niter=3, conv=0.05,
axc=None, ayc=None,
clobber=False,logfile='salt.log',verbose=True):
"""Fits rings in Fabry-Perot ring images"""
with logging(logfile,debug) as log:
# Check the input images
infiles = saltio.argunpack ('Input',images)
# if the outfile exists and not clobber, fail
saltio.overwrite(outfile,clobber)
#open the output file
fout = saltio.openascii(outfile, 'w')
#make sure that the list of waves is convertable to a numpy array
#convert to floats in case the wave is given as a string
if isinstance(waves, str):
waves=[float(x) for x in waves.split(',')]
try:
waves=np.array(waves)
except:
raise SaltError('%s is not convertable to a numpy array' % waves)
#check the value of method
method=saltio.checkfornone(method)
#setup the output file
fout.write('#Comment\n')
fout.write('# radius err xc yc z ut wave dn file\n')
# open each image and detect the ring
for img,w in zip(infiles, waves):
#open the image
hdu=saltio.openfits(img)
#measure the ring in each file
xc, yc, radius, err, z, ut=make_calring(hdu, method=method, thresh=thresh, niter=niter, conv=conv, minsize=minsize, axc=axc, ayc=ayc)
#output the results
outstr=' %7.2f %6.2f %7.2f %7.2f %6.2f %7.4f %8.3f 0 %s\n' % (radius, err, xc, yc, z, ut, w, img)
fout.write(outstr)
log.message(outstr.strip(), with_stdout=verbose, with_header=False)
fout.close()
def specprepare(images,outimages,outpref,badpixelimage='', \
clobber=True,logfile='salt.log',verbose=True):
with logging(logfile,debug) as log:
# Check the input images
infiles = saltio.argunpack ('Input',images)
# create list of output files
outfiles=saltio.listparse('Outfile', outimages, outpref,infiles,'')
#verify that the input and output lists are the same length
saltio.comparelists(infiles,outfiles,'Input','output')
# open the badpixel image
if saltio.checkfornone(badpixelimage) is None:
badpixelstruct=None
else:
try:
badpixelstruct = saltio.openfits(badpixelimage)
except saltio.SaltIOError,e:
msg='badpixel image must be specificied\n %s' % e
raise SALTSpecError(msg)
# open each raw image file
for img, oimg, in zip(infiles, outfiles):
#open the fits file
struct=saltio.openfits(img)
# prepare file
struct=prepare(struct,badpixelstruct)
# write FITS file
saltio.writefits(struct,oimg, clobber=clobber)
saltio.closefits(struct)
message = 'SPECPREPARE -- %s => %s' % (img, oimg)
log.message(message)
def specprepare(images, outimages, outpref, badpixelimage='',
clobber=True, logfile='salt.log', verbose=True):
with logging(logfile, debug) as log:
# Check the input images
infiles = saltio.argunpack('Input', images)
# create list of output files
outfiles = saltio.listparse('Outfile', outimages, outpref, infiles, '')
# verify that the input and output lists are the same length
saltio.comparelists(infiles, outfiles, 'Input', 'output')
# open the badpixel image
if saltio.checkfornone(badpixelimage) is None:
badpixelstruct = None
else:
try:
badpixelstruct = saltio.openfits(badpixelimage)
except saltio.SaltIOError as e:
msg = 'badpixel image must be specificied\n %s' % e
raise SALTSpecError(msg)
# open each raw image file
for img, oimg, in zip(infiles, outfiles):
# open the fits file
struct = saltio.openfits(img)
# prepare file
struct = prepare(struct, badpixelstruct)
# write FITS file
saltio.writefits(struct, oimg, clobber=clobber)
saltio.closefits(struct)
message = 'SPECPREPARE -- %s => %s' % (img, oimg)
log.message(message)
raise SALTMySQLError(message)
#get the obsstats
#ObsStatsId=getobsstatsid(db, date)
try:
filesize = os.path.getsize(PipelineFileNameString) / 1024.0
except OSError:
filesize = os.path.getsize(FileNameString) / 1024.0
except:
filesize = 0
#get the block id
try:
BlockString = ImageHeader['BLOCKID'].strip()
BlockString = saltio.checkfornone(BlockString)
try:
if int(BlockString) == 0: BlockString = None
except:
pass
except KeyError:
BlockString = ''
#get the block id
try:
BlockVisitString = ImageHeader['BVISITID'].strip()
BlockVisitString = saltio.checkfornone(BlockVisitString)
try:
if int(BlockVisitString) == 0: BlockVisitString = None
except:
pass
def create_insert(db, ImageHeader, FileNameString, PipelineFileNameString):
#set up all the image header variables
ExposureTimeString=createstring(ImageHeader, 'EXPTIME', '%5.3f')
TelEpochString=createstring(ImageHeader, 'EQUINOX', '%6.2f', '2000.0')
TargetNameString=createstring(ImageHeader, 'OBJECT', '%s')
InstrueNameString=createstring(ImageHeader, 'INSTRUME', '%s')
ObsModeString=createstring(ImageHeader, 'OBSMODE', '%s')
DetModeString=createstring(ImageHeader, 'DETMODE', '%s')
InstrueNameString=createstring(ImageHeader, 'INSTRUME', '%s')
ProposalCodeString=createstring(ImageHeader, 'PROPID', '%s', default='UNKNOWN')
ObservatString=createstring(ImageHeader, 'OBSERVAT', '%s')
ProposalString=createstring(ImageHeader, 'PROPOSAL', '%s')
#get the ProposalCode_Id
if not ProposalCodeString.strip():
ProposalCodeString='NONE'
ProposalCodeId=getpropcodeid(db, ProposalCodeString)
#set the pipeline strings that are a little more difficult to set
try:
UTStartString=ImageHeader['DATE-OBS']+' '+ImageHeader['TIME-OBS']
except:
UTStartString='1000-01-01 00:00:00'
if not UTStartString.strip():
UTStartString='1000-01-01 00:00:00'
try:
TelRA=15.0*sex2dec(ImageHeader['TELRA'])
TelDec=sex2dec(ImageHeader['TELDEC'])
TelRAString='%10.7f' % TelRA
TelDecString='%10.7f' % TelDec
except Exception as e:
UTStartString='1000-01-01 00:00:00'
TelRAString='0.00000'
TelDecString='100.0000'
try:
PipelineStartString=datatimeobs2DateTime(ImageHeader['SAL-TLM'])
except Exception as inst:
PipelineStartString='1000-01-01 00:00:00'
#set the number of exposures
nexposures=1
try:
if ImageHeader['DETMODE'].count('SLOT'):
nexposures=ImageHeader['NEXTEND']
#this is hardwired but should be changed to use NAMPS
if ImageHeader['INSTRUME'] is 'RSS':
nexposures=nexposures/6
else:
nexposures=nexposures/4
except:
pass
if not PipelineFileNameString:
PipelineFileNameString=FileNameString
try:
name=os.path.basename(FileNameString).split('.')[0]
date = '%s-%s-%s' % (name[1:5], name[5:7], name[7:9])
except Exception as e:
message='Could not determine the date for %s because %s' \
% (FileNameString, e)
raise SALTMySQLError(message)
#get the obsstats
#ObsStatsId=getobsstatsid(db, date)
try:
filesize=os.path.getsize(PipelineFileNameString)/1024.0
except OSError:
filesize=os.path.getsize(FileNameString)/1024.0
except:
filesize=0
#get the block id
try:
BlockString=ImageHeader['BLOCKID'].strip()
BlockString=saltio.checkfornone(BlockString)
try:
if int(BlockString)==0: BlockString=None
except:
pass
except KeyError:
BlockString=''
#get the block id
try:
BlockVisitString=ImageHeader['BVISITID'].strip()
BlockVisitString=saltio.checkfornone(BlockVisitString)
try:
if int(BlockVisitString)==0: BlockVisitString=None
except:
pass
except KeyError:
BlockVisitString=''
#create the insertion command for the data
try:
insert_command='StepStats_Id=1,'
insert_command += "UTStart='"+UTStartString+"',"
insert_command += "ProposalCode_Id=%i," % ProposalCodeId
insert_command += "Target_Name='"+TargetNameString+"',"
insert_command += "ExposureTime='"+ExposureTimeString+"',"
insert_command += "TelRA='"+TelRAString+"',"
insert_command += "TelDec='"+TelDecString+"',"
insert_command += "TelEpoch='"+TelEpochString+"',"
insert_command += "FileName='"+FileNameString+"',"
insert_command += "PipelineDate='"+PipelineStartString+"',"
insert_command += "PipelineFileName='"+PipelineFileNameString+"',"
insert_command += "INSTRUME='"+InstrueNameString+"',"
insert_command += "OBSMODE='"+ObsModeString+"',"
insert_command += "DETMODE='"+DetModeString+"',"
insert_command += "NExposures='%i',"%nexposures
insert_command += "FileSize='%i'"%filesize
if BlockString: insert_command += ",Block_Id='%s'"%BlockString
if BlockVisitString: insert_command += ",BlockVisit_Id='%s'"%BlockVisitString
except Exception as e:
message='Could not create insert command because %s' % e
raise SALTMySQLError(message)
return insert_command
def hrsclean(images, outpath, obslogfile=None, subover=True, trim=True, masbias=None,
subbias=True, median=False, function='polynomial', order=5, rej_lo=3, rej_hi=3,
niter=5, interp='linear', clobber=False, logfile='salt.log',verbose=True):
"""Convert MEF HRS data into a single image. If variance frames and BPMs, then
convert them to the same format as well. Returns an MEF image but that is
combined into a single frame
"""
with logging(logfile,debug) as log:
# Check the input images
infiles = saltio.argunpack ('Input',images)
# create list of output files
outpath=saltio.abspath(outpath)
if saltio.checkfornone(obslogfile) is None:
raise SaltError('Obslog file is required')
# Delete the obslog file if it already exists
if (os.path.isfile(obslogfile) and clobber) or not os.path.isfile(obslogfile):
if os.path.isfile(obslogfile): saltio.delete(obslogfile)
#read in the obsveration log or create it
headerDict=obslog(infiles, log)
obsstruct=createobslogfits(headerDict)
saltio.writefits(obsstruct, obslogfile)
else:
obsstruct=saltio.openfits(obslogfile)
#create the list of bias frames and process them
filename=obsstruct.data.field('FILENAME')
detmode=obsstruct.data.field('DETMODE')
ccdtype=obsstruct.data.field('OBJECT')
biaslist=filename[ccdtype=='Bias']
masterbias_dict={}
if log: log.message('Processing Bias Frames')
for img in infiles:
if os.path.basename(img) in biaslist:
#open the image
struct=pyfits.open(img)
bimg=outpath+'bgph'+os.path.basename(img)
#print the message
if log:
message='Processing Zero frame %s' % img
log.message(message, with_stdout=verbose, with_header=False)
#process the image
struct=clean(struct, createvar=False, badpixelstruct=None, mult=True,
subover=subover, trim=trim, subbias=False, imstack=False,
bstruct=None, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],'HPREPARE', 'Images have been prepared', hist)
saltkey.new('HGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
#saltkey.new('HXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('HBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])
# write FITS file
saltio.writefits(struct,bimg, clobber=clobber)
saltio.closefits(struct)
#add files to the master bias list
masterbias_dict=compareimages(struct, bimg, masterbias_dict, keylist=hrsbiasheader_list)
#create the master bias frame
for i in list(masterbias_dict.keys()):
bkeys=masterbias_dict[i][0]
blist=masterbias_dict[i][1:]
mbiasname=outpath+createmasterbiasname(blist, bkeys, x1=5, x2=13)
bfiles=','.join(blist)
saltcombine(bfiles, mbiasname, method='median', reject='sigclip', mask=False,
weight=False, blank=0, scale=None, statsec=None, lthresh=3, \
hthresh=3, clobber=False, logfile=logfile,verbose=verbose)
#apply full reductions to the science data
for img in infiles:
nimg=os.path.basename(img)
if not nimg in biaslist:
#open the image
struct=pyfits.open(img)
simg=outpath+'mbgph'+os.path.basename(img)
#print the message
if log:
message='Processing science frame %s' % img
log.message(message, with_stdout=verbose)
#get master bias frame
masterbias=get_masterbias(struct, masterbias_dict, keylist=hrsbiasheader_list)
if masterbias:
subbias=True
bstruct=saltio.openfits(masterbias)
else:
subbias=False
bstruct=None
#process the image
struct=clean(struct, createvar=False, badpixelstruct=None, mult=True,
subover=subover, trim=trim, subbias=subbias, imstack=True,
bstruct=bstruct, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],'HPREPARE', 'Images have been prepared', hist)
saltkey.new('HGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
#saltkey.new('HXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('HBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])
# write FITS file
saltio.writefits(struct,simg, clobber=clobber)
saltio.closefits(struct)
return
def saltquery(selection, logic, startdate, enddate, outfile=None, sdbhost='sdb.saao', sdbname='sdb', \
sdbuser='', password='', clobber=False, logfile='saltlog.log', verbose=True):
"""Query the salt database for FITS files
"""
with logging(logfile,debug) as log:
#check the outfiles
if not saltio.checkfornone(outfile):
outfile=None
#check that the output file can be deleted
if outfile:
saltio.overwrite(outfile, clobber)
#open outfile
if outfile:
fout=saltio.openascii(outfile, 'w')
#connect to the database
sdb=saltmysql.connectdb(sdbhost,sdbname,sdbuser,password)
#Create a list of the selection and then unpack it
selection_list = saltio.argunpack('Selection', selection)
selection=','.join(selection_list)
#write out the header for the outfile
outstr='#'+' '.join(['%s' % x for x in selection_list])
if outfile:
fout.write(outstr+'\n')
else:
print outstr
#set up the table
rsstable=''' FileData
left join FitsHeaderImage using (FileData_Id)
inner join FitsHeaderRss using (FileData_Id)
'''
#set up the table
scamtable=''' FileData
left join FitsHeaderImage using (FileData_Id)
inner join FitsHeaderSalticam using (FileData_Id)
'''
#set up the logic
logic=makelogic(logic, startdate, enddate)
for tab in [rsstable, scamtable]:
msg='''
Mysql querying data is:
SELECT %s
FROM %s
WHERE %s
''' % (selection, tab, logic)
log.message(msg, with_stdout=verbose)
record=saltmysql.select(sdb, selection, tab, logic)
print record
for r in record:
outstr=' '.join(['%s' % x for x in r])
if outfile:
fout.write(outstr+'\n')
else:
print outstr
#close outfile
if outfile: fout.close()
def quickap(profile, lampid=None, findobj=False, objsection=None, skysection=None, clobber=False, logfile='saltclean.log', verbose=True):
profile = os.path.basename(profile)
outfile = profile.replace('fits', 'txt')
#open the file
struct=pyfits.open(profile)
data=struct[1].data
xlen=len(struct[1].data[0])
ylen=len(struct[1].data)
#determine the resolution element
dres = 1.0*calc_resolution(struct)
#correct for the response function
response = create_response(data, spatial_axis=1, order=3, conv=1e-2, niter=10)
data = data / response
if objsection is None:
y1=0.5*ylen-0.05*ylen
y2=0.5*ylen+0.05*ylen
objsection='[%i:%i]' % (y1,y2)
else:
y1=int(objsection[1:].split(':')[0])
y2=int(objsection[:-1].split(':')[1])
#extract object
minsize=5
thresh=5
ap_spec=extract(struct, method='normal', section=[(y1,y2)], minsize=minsize, thresh=thresh, convert=True)[0]
#if it is a lamp, do not sky subtract it
if saltsafeio.checkfornone(lampid):
write_extract(outfile, [ap_spec], outformat='ascii', clobber=clobber)
return y1,y2
if skysection is None:
skysection='%i:%i' % (y2+0.1*ylen,y2+0.2*ylen)
#sky spectrum
sy1, sy2 = skysection.split(':')
sy1=int(sy1)
sy2=int(sy2)
sk_spec=extract(struct, method='normal', section=[(sy1,sy2)], minsize=minsize, thresh=thresh, convert=True)[0]
sk_spec.ldata - sk_spec.ldata
w1=ap_spec.wave.min()
w2=ap_spec.wave.max()
nsect=10
dw=(w2-w1)/nsect
for w in np.arange(w1,w2,dw):
mask=(ap_spec.wave>w)*(ap_spec.wave<w+dw)
sk_spec=quickcross(ap_spec, sk_spec, mask=mask, dx=0.02, nstep=500)
ap_spec.ldata[mask]=ap_spec.ldata[mask]-float(y2-y1)/(sy2-sy1)*sk_spec.ldata[mask]
#set up masks for sky subtraction
amask = (ap_spec.ldata==0)
smask = (sk_spec.ldata==0)
#find offset between two spectra
d1 = ap_spec.ldata
d2 = sk_spec.ldata*float(y2-y1)/(sy2-sy1)
y=d1[d2>0]/d2[d2>0]
y=np.median(y)
#subtract the skyspectrum
clean_spectra(ap_spec, dres=2*dres, grow=dres)
#pl.plot(ap_spec.wave, ap_spec.ldata, ls='', marker='o', ms=2)
#pl.show()
#write it out and return
write_extract(outfile, [ap_spec], outformat='ascii', clobber=clobber)
#clean up the data
if clobber:
print 'Replacing pixels'
os.remove(profile)
struct[1].data[data==0]=np.median(data)
struct.writeto(profile)
return y1,y2
def saltcharge(
obsdate,
outfile,
sdbhost="sdb.saao",
sdbname="sdb",
sdbuser="",
password="",
clobber=False,
logfile="saltlog.log",
verbose=True,
):
"""Calculate the charged time for proposals observed during a night
"""
with logging(logfile, debug) as log:
# check the outfiles
if not saltio.checkfornone(outfile):
outfile = None
# check that the output file can be deleted
if outfile:
saltio.overwrite(outfile, clobber)
# connect the database
sdb = saltmysql.connectdb(sdbhost, sdbname, sdbuser, password)
# get all the proposals observed during the night
select_state = "distinct Proposal_Code"
table_state = "FileData Join ProposalCode using (ProposalCode_Id)"
logic_state = "FileName like '%" + obsdate + "%'"
records = saltmysql.select(sdb, select_state, table_state, logic_state)
pids = []
pid_time = {}
for r in records:
pids.append(r[0])
pid_time[r[0]] = 0
if len(pids) == 0:
message = "There are no proposal to charge time for %s" % obsdate
log.message(message)
return
# get the nightinfo_id
night_id = saltmysql.getnightinfoid(sdb, obsdate)
print night_id
# get a list of all the images taken during the night
select_state = "FileName, Proposal_Code, Target_Name, ExposureTime, UTSTART, INSTRUME, OBSMODE, DETMODE, CCDTYPE, NExposures"
table_state = "FileData Join ProposalCode using (ProposalCode_Id) join FitsHeaderImage using (FileData_Id)"
logic_state = "FileName like '%" + obsdate + "%'"
img_list = saltmysql.select(sdb, select_state, table_state, logic_state)
# get all the blocks visited
select_state = "Block_Id, Accepted, Proposal_Code"
table_state = "Block join BlockVisit using (Block_Id) join Proposal using (Proposal_Id) join ProposalCode using (ProposalCode_Id)"
logic_state = "NightInfo_Id=%i" % night_id
block_list = saltmysql.select(sdb, select_state, table_state, logic_state)
print block_list
# get the start and end of twilight
nightstart = saltmysql.select(sdb, "EveningTwilightEnd", "NightInfo", "NightInfo_Id=%i" % night_id)[0][0]
nightend = saltmysql.select(sdb, "MorningTwilightStart", "NightInfo", "NightInfo_Id=%i" % night_id)[0][0]
print nightstart, nightend, (nightend - nightstart).seconds
print
# download the SOMMI log from the night
try:
sommi_log = saltmysql.select(sdb, "SONightLog", "NightLogs", "NightInfo_Id=%i" % night_id)[0][0]
except Exception, e:
msg = "Unable to read in SOMMI log for %s" % obsdate
raise SaltError(msg)
# add to account for error in SOMMI log
if int(obsdate) < 20111027:
sommi_log = sommi_log.replace("Object name", "\nObject name")
# parse the sommi log
point_list = parseforpointings(sommi_log)
# now find all blocks observed during a night
for point in point_list:
# proposal for that block
pid = point[0].strip()
# start time for that block
starttime = point[1]
# find the end time for that block
endtime = findnexttime(point[1], point_list, nightend)
# find the start time of the first object to be observed
# for this block
startimage = findfirstimage(pid, starttime, img_list)
lastimage = findlastimage(pid, endtime, img_list)
# check to see if the end time for the last file observed for
# this block
if startimage is not None:
if startimage > endtime:
startimage = None
# determine the acquisition time for the block
if startimage is not None:
acqdelta = (startimage - starttime).seconds
else:
acqdelta = -1
# find the shutter open time
shuttertime = calculate_shutteropen(img_list, starttime, endtime)
# if the shutter and time of the last image is substantially different from the
# end of the block, then take the last image as the end of the block
# *TODO* Change to use expected block time
st = starttime + datetime.timedelta(0, shuttertime + acqdelta)
if (endtime - st).seconds > 900:
if lastimage is None:
endtime = st
elif (lastimage - st).seconds > 3600:
endtime = st
else:
endtime = lastimage
# account for the time for that block
tblock = (endtime - starttime).seconds
if acqdelta > -1:
# find the associated block
block_id, blockvisit_id, accepted = getblockid(sdb, night_id, pid, img_list, starttime, endtime)
if accepted and pid.count("-3-"):
# charged time for that block
try:
pid_time[pid] += tblock
except KeyError:
pid_time[pid] = tblock
print block_id, blockvisit_id, pid, starttime, tblock, acqdelta, shuttertime, shuttertime / tblock, block_id, accepted
# update the charge time
slewdelta = 0
scidelta = tblock - acqdelta
update_cmd = "TotalSlewTime=%i, TotalAcquisitionTime=%i, TotalScienceTime=%i" % (
slewdelta,
acqdelta,
scidelta,
)
table_cmd = "BlockVisit"
logic_cmd = "BlockVisit_Id=%i" % blockvisit_id
saltmysql.update(sdb, update_cmd, table_cmd, logic_cmd)
return
# update the charged time information
# TODO: Check to see if the block was approved
ptime_tot = 0
stime_tot = 0
for k in pid_time:
ptime = pid_time[k]
if ptime > 0:
obsratio = stime / ptime
else:
obsratio = 0
print "%25s %5i %5i %3.2f" % (k, ptime, stime, obsratio)
if k.count("-3-"):
ptime_tot += ptime
stime_tot += stime
# print out the total night statistics
tdelta = nightend - nightstart
print tdelta.seconds, ptime_tot, stime_tot
def hrsclean(images,
outpath,
obslogfile=None,
subover=True,
trim=True,
masbias=None,
subbias=True,
median=False,
function='polynomial',
order=5,
rej_lo=3,
rej_hi=3,
niter=5,
interp='linear',
clobber=False,
logfile='salt.log',
verbose=True):
"""Convert MEF HRS data into a single image. If variance frames and BPMs, then
convert them to the same format as well. Returns an MEF image but that is
combined into a single frame
"""
with logging(logfile, debug) as log:
# Check the input images
infiles = saltio.argunpack('Input', images)
# create list of output files
outpath = saltio.abspath(outpath)
if saltio.checkfornone(obslogfile) is None:
raise SaltError('Obslog file is required')
# Delete the obslog file if it already exists
if (os.path.isfile(obslogfile)
and clobber) or not os.path.isfile(obslogfile):
if os.path.isfile(obslogfile): saltio.delete(obslogfile)
#read in the obsveration log or create it
headerDict = obslog(infiles, log)
obsstruct = createobslogfits(headerDict)
saltio.writefits(obsstruct, obslogfile)
else:
obsstruct = saltio.openfits(obslogfile)
#create the list of bias frames and process them
filename = obsstruct.data.field('FILENAME')
detmode = obsstruct.data.field('DETMODE')
ccdtype = obsstruct.data.field('OBJECT')
biaslist = filename[ccdtype == 'Bias']
masterbias_dict = {}
if log: log.message('Processing Bias Frames')
for img in infiles:
if os.path.basename(img) in biaslist:
#open the image
struct = pyfits.open(img)
bimg = outpath + 'bgph' + os.path.basename(img)
#print the message
if log:
message = 'Processing Zero frame %s' % img
log.message(message,
with_stdout=verbose,
with_header=False)
#process the image
struct = clean(struct,
createvar=False,
badpixelstruct=None,
mult=True,
subover=subover,
trim=trim,
subbias=False,
imstack=False,
bstruct=None,
median=median,
function=function,
order=order,
rej_lo=rej_lo,
rej_hi=rej_hi,
niter=niter,
log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist = history(
level=1,
wrap=False,
exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0], 'HPREPARE',
'Images have been prepared', hist)
saltkey.new('HGAIN', time.asctime(time.localtime()),
'Images have been gain corrected', struct[0])
#saltkey.new('HXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('HBIAS', time.asctime(time.localtime()),
'Images have been de-biased', struct[0])
# write FITS file
saltio.writefits(struct, bimg, clobber=clobber)
saltio.closefits(struct)
#add files to the master bias list
masterbias_dict = compareimages(struct,
bimg,
masterbias_dict,
keylist=hrsbiasheader_list)
#create the master bias frame
for i in masterbias_dict.keys():
bkeys = masterbias_dict[i][0]
blist = masterbias_dict[i][1:]
mbiasname = outpath + createmasterbiasname(
blist, bkeys, x1=5, x2=13)
bfiles = ','.join(blist)
saltcombine(bfiles, mbiasname, method='median', reject='sigclip', mask=False,
weight=False, blank=0, scale=None, statsec=None, lthresh=3, \
hthresh=3, clobber=False, logfile=logfile,verbose=verbose)
#apply full reductions to the science data
for img in infiles:
nimg = os.path.basename(img)
if not nimg in biaslist:
#open the image
struct = pyfits.open(img)
simg = outpath + 'mbgph' + os.path.basename(img)
#print the message
if log:
message = 'Processing science frame %s' % img
log.message(message, with_stdout=verbose)
#get master bias frame
masterbias = get_masterbias(struct,
masterbias_dict,
keylist=hrsbiasheader_list)
if masterbias:
subbias = True
bstruct = saltio.openfits(masterbias)
else:
subbias = False
bstruct = None
#process the image
struct = clean(struct,
createvar=False,
badpixelstruct=None,
mult=True,
subover=subover,
trim=trim,
subbias=subbias,
imstack=True,
bstruct=bstruct,
median=median,
function=function,
order=order,
rej_lo=rej_lo,
rej_hi=rej_hi,
niter=niter,
log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist = history(
level=1,
wrap=False,
exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0], 'HPREPARE',
'Images have been prepared', hist)
saltkey.new('HGAIN', time.asctime(time.localtime()),
'Images have been gain corrected', struct[0])
#saltkey.new('HXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('HBIAS', time.asctime(time.localtime()),
'Images have been de-biased', struct[0])
# write FITS file
saltio.writefits(struct, simg, clobber=clobber)
saltio.closefits(struct)
return
def saltfpringfind(images,
method=None,
section=None,
thresh=5,
minsize=10,
niter=5,
conv=0.05,
displayimage=True,
clobber=False,
logfile='salt.log',
verbose=True):
with logging(logfile, debug) as log:
# Check the input images
infiles = saltio.argunpack('Input', images)
#check the method
method = saltio.checkfornone(method)
# read in the section
section = saltio.checkfornone(section)
if section is None:
pass
else:
section = saltio.getSection(section)
# open each raw image file
for img in infiles:
#open the fits file
struct = saltio.openfits(img)
data = struct[0].data
#determine the background value for the image
if section is None:
#if section is none, just use all pixels greater than zero
bdata = data[data > 0]
else:
y1, y2, x1, x2 = section
bdata = data[y1:y2, x1:x2]
bmean, bmedian, bstd = iterstat(bdata,
sig=thresh,
niter=niter,
verbose=False)
message="Image Background Statistics\n%30s %6s %8s %8s\n%30s %5.4f %5.4f %5.4f\n" % \
('Image', 'Mean', 'Median', 'Std',img, bmean, bmedian, bstd)
log.message(message, with_stdout=verbose)
mdata = data * (data - bmean > thresh * bstd)
#prepare the first guess for the image
ring_list = findrings(data,
thresh=thresh,
niter=niter,
minsize=minsize)
#if specified, find the center of the ring
if method is not None:
for i in range(len(ring_list)):
ring_list[i] = findcenter(data,
ring_list[i],
method,
niter=niter,
conv=conv)
#if one peak: no rings. If two peaks: one ring, if two peaks: four rings
if len(ring_list) == 1:
msg = "One ring dected in image"
else:
msg = "%i rings found in image" % len(ring_list)
log.message(message, with_stdout=verbose)
if displayimage:
regfile = img.replace('.fits', '.reg')
if clobber and os.path.isfile(regfile):
fout = saltio.delete(regfile)
fout = open(regfile, 'w')
fout.write("""# Region file format: DS9 version 4.1
# Filename: %s
global color=green dashlist=8 3 width=1 font="helvetica 10 normal roman" select=1 highlite=1 dash=0 fixed=0 edit=1 move=1 delete=1 include=1 source=1
physical
""" % img)
for ring in ring_list:
fout.write('circle(%f, %f, %f)\n' %
(ring.xc, ring.yc, ring.prad))
fout.write('circle(%f, %f, %f)\n' %
(ring.xc, ring.yc, ring.prad - 3 * ring.sigma))
fout.write('circle(%f, %f, %f)\n' %
(ring.xc, ring.yc, ring.prad + 3 * ring.sigma))
fout.close()
display(img, catname=regfile, rformat='reg')
message = 'Ring Parameters\n%30s %6s %6s %6s\n' % ('Image', 'XC',
'YC', 'Radius')
log.message(message, with_stdout=verbose)
for ring in ring_list:
msg = '%30s %6.2f %6.2f %6.2f\n' % (img, ring.xc, ring.yc,
ring.prad)
log.message(msg, with_header=False, with_stdout=verbose)
def saltcalibrations(
propcode,
outfile=None,
sdbhost="sdb.saao",
sdbname="sdb",
sdbuser="",
password="",
clobber=False,
logfile="saltlog.log",
verbose=True,
):
"""Seach the salt database for FITS files
"""
with logging(logfile, debug) as log:
# check the outfiles
if not saltio.checkfornone(outfile):
outfile = None
# check that the output file can be deleted
if outfile:
saltio.overwrite(outfile, clobber)
fout = open(oufile, "w")
# connect to the database
sdb = saltmysql.connectdb(sdbhost, sdbname, sdbuser, password)
# determine the associated
username = saltmysql.getpiptusername(sdb, propcode)
userdir = "/salt/ftparea/%s/spst" % username
if not os.path.exists(userdir):
saltio.createdir(userdir)
log.message("Will copy data to %s" % userdir)
# Find all the data assocated with a proposal
cmd_select = "d.FileName,d.FileData_Id, CCDTYPE, d.DETMODE, d.OBSMODE, CCDSUM, GAINSET, ROSPEED, FILTER, GRATING, GRTILT, CAMANG, MASKID"
cmd_table = """ FileData as d
left join FitsHeaderImage using (FileData_Id)
left join FitsHeaderRss using (FileData_Id)
left join ProposalCode using (ProposalCode_Id)
"""
cmd_logic = 'Proposal_Code="%s" and CCDTYPE="OBJECT" and d.OBSMODE="SPECTROSCOPY"' % (propcode)
record = saltmysql.select(sdb, cmd_select, cmd_table, cmd_logic)
# loop through all the results and return only the Set of identical results
caldict = create_caldict(record)
# prepare for writing out the results
outstr = ""
if outfile:
fout.write(outstr + "\n")
else:
print outstr
# now find all the cal_spst that have the same settings
cmd_select = "d.FileName,d.FileData_Id, CCDTYPE, d.DETMODE, d.OBSMODE, CCDSUM, GAINSET, ROSPEED, FILTER, GRATING, GRTILT, CAMANG, MASKID"
cmd_table = """ FileData as d
left join FitsHeaderImage using (FileData_Id)
left join FitsHeaderRss using (FileData_Id)
left join ProposalCode using (ProposalCode_Id)
"""
for r in caldict:
cmd_logic = "CCDSUM='%s' and GRATING='%s' and GRTILT='%s' and CAMANG='%s' and Proposal_Code='CAL_SPST'" % (
caldict[r][3],
caldict[r][7],
caldict[r][8],
caldict[r][9],
)
# cmd_logic="CCDSUM='%s' and GRATING='%s' and AR_STA='%s' " % (caldict[r][3], caldict[r][7], caldict[r][9])
log.message(cmd_logic, with_header=False)
record = saltmysql.select(sdb, cmd_select, cmd_table, cmd_logic)
# print record
# write out hte results
for r in record:
outstr = " ".join(["%s" % x for x in r])
if outfile:
fout.write(outstr + "\n")
else:
log.message(outstr, with_header=False)
# copy to the user directory
cfile = makefilename(r[0], state="product")
shutil.copy(cfile, userdir)
cfile = makefilename(r[0], state="raw")
shutil.copy(cfile, userdir)
# close outfile
if outfile:
fout.close()
def specslitnormalize(images,
outimages,
outpref,
response=None,
response_output=None,
order=2,
conv=1e-2,
niter=20,
startext=0,
clobber=False,
logfile='salt.log',
verbose=True):
with logging(logfile, debug) as log:
# Check the input images
infiles = saltio.argunpack('Input', images)
# create list of output files
outfiles = saltio.listparse('Outfile', outimages, outpref, infiles, '')
# read in the response function
response = saltio.checkfornone(response)
if response:
log.message('Loading response from %s' % response)
response = readresponse(response)
# Identify the lines in each file
for img, ofile in zip(infiles, outfiles):
# open the image
hdu = saltio.openfits(img)
for i in range(startext, len(hdu)):
if hdu[i].name == 'SCI':
log.message('Normalizing extension %i in %s' % (i, img))
# things that will change for each slit
# set up the data for the source
try:
data = hdu[i].data
except Exception as e:
message = \
'Unable to read in data array in %s because %s' % \
(img, e)
raise SALTSpecError(message)
if response is None:
response = create_response(data,
spatial_axis=1,
order=order,
conv=conv,
niter=niter)
if response_output:
write_response(response, clobber=clobber)
else:
# add a check that the response is the same shape as
# the data
if len(response) != data.shape[0]:
raise SALTSpecError(
'Length of response function does not equal size of image array'
)
# correct the data
data = data / response
# correct the variance frame
if saltkey.found('VAREXT', hdu[i]):
vhdu = saltkey.get('VAREXT', hdu[i])
hdu[vhdu].data = hdu[vhdu].data / response
saltio.writefits(hdu, ofile, clobber=clobber)
raise SALTMySQLError(message)
#get the obsstats
#ObsStatsId=getobsstatsid(db, date)
try:
filesize=os.path.getsize(PipelineFileNameString)/1024.0
except OSError:
filesize=os.path.getsize(FileNameString)/1024.0
except:
filesize=0
#get the block id
try:
BlockString=ImageHeader['BLOCKID'].strip()
BlockString=saltio.checkfornone(BlockString)
try:
if int(BlockString)==0: BlockString=None
except:
pass
except KeyError:
BlockString=''
#create the insertion command for the data
try:
insert_command='StepStats_Id=1,'
insert_command += "UTStart='"+UTStartString+"',"
insert_command += "ProposalCode_Id=%i," % ProposalCodeId
insert_command += "Target_Name='"+TargetNameString+"',"
def saltfpringfind(images, method=None, section=None, thresh=5, minsize=10, niter=5, conv=0.05,
displayimage=True, clobber=False, logfile='salt.log',verbose=True):
with logging(logfile,debug) as log:
# Check the input images
infiles = saltio.argunpack ('Input',images)
#check the method
method=saltio.checkfornone(method)
# read in the section
section=saltio.checkfornone(section)
if section is None:
pass
else:
section=saltio.getSection(section)
# open each raw image file
for img in infiles:
#open the fits file
struct=saltio.openfits(img)
data=struct[0].data
#determine the background value for the image
if section is None:
#if section is none, just use all pixels greater than zero
bdata=data[data>0]
else:
y1,y2,x1,x2=section
bdata=data[y1:y2,x1:x2]
bmean, bmedian, bstd=iterstat(bdata, sig=thresh, niter=niter, verbose=False)
message="Image Background Statistics\n%30s %6s %8s %8s\n%30s %5.4f %5.4f %5.4f\n" % \
('Image', 'Mean', 'Median', 'Std',img, bmean, bmedian, bstd)
log.message(message, with_stdout=verbose)
mdata=data*(data-bmean>thresh*bstd)
#prepare the first guess for the image
ring_list=findrings(data, thresh=thresh, niter=niter, minsize=minsize)
#if specified, find the center of the ring
if method is not None:
for i in range(len(ring_list)):
ring_list[i]=findcenter(data, ring_list[i], method, niter=niter, conv=conv)
#if one peak: no rings. If two peaks: one ring, if two peaks: four rings
if len(ring_list)==1:
msg="One ring dected in image"
else:
msg="%i rings found in image" % len(ring_list)
log.message(message, with_stdout=verbose)
if displayimage:
regfile=img.replace('.fits', '.reg')
if clobber and os.path.isfile(regfile): fout=saltio.delete(regfile)
fout=open(regfile, 'w')
fout.write("""# Region file format: DS9 version 4.1
# Filename: %s
global color=green dashlist=8 3 width=1 font="helvetica 10 normal roman" select=1 highlite=1 dash=0 fixed=0 edit=1 move=1 delete=1 include=1 source=1
physical
""" % img)
for ring in ring_list:
fout.write('circle(%f, %f, %f)\n' % (ring.xc,ring.yc,ring.prad))
fout.write('circle(%f, %f, %f)\n' % (ring.xc,ring.yc,ring.prad-3*ring.sigma))
fout.write('circle(%f, %f, %f)\n' % (ring.xc,ring.yc,ring.prad+3*ring.sigma))
fout.close()
display(img, catname=regfile, rformat='reg')
message = 'Ring Parameters\n%30s %6s %6s %6s\n' % ('Image', 'XC', 'YC', 'Radius')
log.message(message, with_stdout=verbose)
for ring in ring_list:
msg='%30s %6.2f %6.2f %6.2f\n' % (img, ring.xc, ring.yc, ring.prad)
log.message(msg, with_header=False, with_stdout=verbose)
raise SALTMySQLError(message)
#get the obsstats
#ObsStatsId=getobsstatsid(db, date)
try:
filesize=os.path.getsize(PipelineFileNameString)/1024.0
except OSError:
filesize=os.path.getsize(FileNameString)/1024.0
except:
filesize=0
#get the block id
try:
BlockString=ImageHeader['BLOCKID'].strip()
BlockString=saltio.checkfornone(BlockString)
try:
if int(BlockString)==0: BlockString=None
except:
pass
except KeyError:
BlockString=''
#get the block id
try:
BlockVisitString=ImageHeader['BVISITID'].strip()
BlockVisitString=saltio.checkfornone(BlockVisitString)
try:
if int(BlockVisitString)==0: BlockVisitString=None
except:
pass
def specsens(specfile,
outfile,
stdfile,
extfile,
airmass=None,
exptime=None,
stdzp=3.68e-20,
function='polynomial',
order=3,
thresh=3,
niter=5,
fitter='gaussian',
clobber=True,
logfile='salt.log',
verbose=True):
with logging(logfile, debug) as log:
# read in the specfile and create a spectrum object
obs_spectra = st.readspectrum(specfile.strip(),
error=True,
ftype='ascii')
# smooth the observed spectrum
# read in the std file and convert from magnitudes to fnu
# then convert it to fwave (ergs/s/cm2/A)
std_spectra = st.readspectrum(stdfile.strip(),
error=False,
ftype='ascii')
std_spectra.flux = Spectrum.magtoflux(std_spectra.flux, stdzp)
std_spectra.flux = Spectrum.fnutofwave(std_spectra.wavelength,
std_spectra.flux)
# Get the typical bandpass of the standard star,
std_bandpass = np.diff(std_spectra.wavelength).mean()
# Smooth the observed spectrum to that bandpass
obs_spectra.flux = st.boxcar_smooth(obs_spectra, std_bandpass)
# read in the extinction file (leave in magnitudes)
ext_spectra = st.readspectrum(extfile.strip(),
error=False,
ftype='ascii')
# determine the airmass if not specified
if saltio.checkfornone(airmass) is None:
message = 'Airmass was not supplied'
raise SALTSpecError(message)
# determine the exptime if not specified
if saltio.checkfornone(exptime) is None:
message = 'Exposure Time was not supplied'
raise SALTSpecError(message)
# calculate the calibrated spectra
log.message('Calculating the calibration curve for %s' % specfile)
cal_spectra = sensfunc(obs_spectra, std_spectra, ext_spectra, airmass,
exptime)
# plot(cal_spectra.wavelength, cal_spectra.flux * std_spectra.flux)
# fit the spectra--first take a first cut of the spectra
# using the median absolute deviation to throw away bad points
cmed = np.median(cal_spectra.flux)
cmad = saltstat.mad(cal_spectra.flux)
mask = (abs(cal_spectra.flux - cmed) < thresh * cmad)
mask = np.logical_and(mask, (cal_spectra.flux > 0))
# now fit the data
# Fit using a gaussian process.
if fitter == 'gaussian':
from sklearn.gaussian_process import GaussianProcess
#Instanciate a Gaussian Process model
dy = obs_spectra.var[mask]**0.5
dy /= obs_spectra.flux[mask] / cal_spectra.flux[mask]
y = cal_spectra.flux[mask]
gp = GaussianProcess(corr='squared_exponential',
theta0=1e-2,
thetaL=1e-4,
thetaU=0.1,
nugget=(dy / y)**2.0)
X = np.atleast_2d(cal_spectra.wavelength[mask]).T
# Fit to data using Maximum Likelihood Estimation of the parameters
gp.fit(X, y)
x = np.atleast_2d(cal_spectra.wavelength).T
# Make the prediction on the meshed x-axis (ask for MSE as well)
y_pred = gp.predict(x)
cal_spectra.flux = y_pred
else:
fit = interfit(cal_spectra.wavelength[mask],
cal_spectra.flux[mask],
function=function,
order=order,
thresh=thresh,
niter=niter)
fit.interfit()
cal_spectra.flux = fit(cal_spectra.wavelength)
# write the spectra out
st.writespectrum(cal_spectra, outfile, ftype='ascii')
def specreduce(
images,
badpixelimage=None,
caltype="rss",
function="polynomial",
order=3,
skysub=True,
skysection=None,
findobj=False,
objsection=None,
thresh=3.0,
clobber=True,
logfile="salt.log",
verbose=True,
):
with logging(logfile, debug) as log:
# Check the input images
infiles = saltio.argunpack("Input", images)
# open the badpixelstruct
if saltio.checkfornone(badpixelimage):
badpixelstruct = saltio.openfits(badpixelimage)
else:
badpixelstruct = None
# set up the section for sky estimate
if skysection is not None:
skysection = makesection(skysection)
else:
skysub = False
# set up the section for sky estimate
section = saltio.checkfornone(objsection)
if section is not None:
sections = saltio.getSection(section, iraf_format=False)
objsection = []
for i in range(0, len(sections), 2):
objsection.append((sections[i], sections[i + 1]))
# determine the wavelength solutions
if caltype == "line":
calc_wavesol(infiles)
# correct the images
for img in infiles:
# open the fits file
struct = saltio.openfits(img)
# prepare filep
log.message("Preparing %s" % img)
struct = prepare(struct, badpixelstruct)
# rectify the spectrum
log.message("Rectifying %s using %s" % (img, caltype))
struct = rectify(struct, None, caltype=caltype, function=function, order=order)
# sky subtract the spectrum
# assumes the data is long slit and in the middle of the field
if skysub:
log.message("Subtracting the sky from %s" % img)
struct = skysubtract(struct, method="normal", section=skysection)
# extract the spectrum
log.message("Extracting the spectrum from %s" % img)
print objsection
aplist = extract(struct, method="normal", section=objsection, thresh=thresh)
oimg = os.path.dirname(os.path.abspath(img)) + "/s" + os.path.basename(img.strip())
ofile = oimg[:-5] + ".txt"
write_extract(ofile, aplist, clobber=clobber)
# write FITS file
log.message("Writing 2-D corrected image as %s" % oimg)
saltio.writefits(struct, oimg, clobber=clobber)
saltio.closefits(struct)
def specslitnormalize(images, outimages, outpref, response=None,
response_output=None, order=2, conv=1e-2, niter=20,
startext=0, clobber=False, logfile='salt.log',
verbose=True):
with logging(logfile, debug) as log:
# Check the input images
infiles = saltio.argunpack('Input', images)
# create list of output files
outfiles = saltio.listparse('Outfile', outimages, outpref, infiles, '')
# read in the response function
response = saltio.checkfornone(response)
if response:
log.message('Loading response from %s' % response)
response = readresponse(response)
# Identify the lines in each file
for img, ofile in zip(infiles, outfiles):
# open the image
hdu = saltio.openfits(img)
for i in range(startext, len(hdu)):
if hdu[i].name == 'SCI':
log.message('Normalizing extension %i in %s' % (i, img))
# things that will change for each slit
# set up the data for the source
try:
data = hdu[i].data
except Exception as e:
message = \
'Unable to read in data array in %s because %s' % \
(img, e)
raise SALTSpecError(message)
if response is None:
response = create_response(
data,
spatial_axis=1,
order=order,
conv=conv,
niter=niter)
if response_output:
write_response(response, clobber=clobber)
else:
# add a check that the response is the same shape as
# the data
if len(response) != data.shape[0]:
raise SALTSpecError(
'Length of response function does not equal size of image array')
# correct the data
data = data / response
# correct the variance frame
if saltkey.found('VAREXT', hdu[i]):
vhdu = saltkey.get('VAREXT', hdu[i])
hdu[vhdu].data = hdu[vhdu].data / response
saltio.writefits(hdu, ofile, clobber=clobber)
def specreduce(images, badpixelimage=None, caltype='rss',
function='polynomial', order=3,
skysub=True, skysection=None, findobj=False,
objsection=None, thresh=3.0,
clobber=True, logfile='salt.log', verbose=True):
with logging(logfile, debug) as log:
# Check the input images
infiles = saltio.argunpack('Input', images)
# open the badpixelstruct
if saltio.checkfornone(badpixelimage):
badpixelstruct = saltio.openfits(badpixelimage)
else:
badpixelstruct = None
# set up the section for sky estimate
if skysection is not None:
skysection = makesection(skysection)
else:
skysub = False
# set up the section for sky estimate
section = saltio.checkfornone(objsection)
if section is not None:
sections = saltio.getSection(section, iraf_format=False)
objsection = []
for i in range(0, len(sections), 2):
objsection.append((sections[i], sections[i + 1]))
# determine the wavelength solutions
if caltype == 'line':
calc_wavesol(infiles)
# correct the images
for img in infiles:
# open the fits file
struct = saltio.openfits(img)
# prepare filep
log.message('Preparing %s' % img)
struct = prepare(struct, badpixelstruct)
# rectify the spectrum
log.message('Rectifying %s using %s' % (img, caltype))
struct = rectify(
struct,
None,
caltype=caltype,
function=function,
order=order)
# sky subtract the spectrum
# assumes the data is long slit and in the middle of the field
if skysub:
log.message('Subtracting the sky from %s' % img)
struct = skysubtract(
struct,
method='normal',
section=skysection)
# extract the spectrum
log.message('Extracting the spectrum from %s' % img)
print objsection
aplist = extract(
struct,
method='normal',
section=objsection,
thresh=thresh)
oimg = os.path.dirname(
os.path.abspath(img)) + '/s' + os.path.basename(img.strip())
ofile = oimg[:-5] + '.txt'
write_extract(ofile, aplist, clobber=clobber)
# write FITS file
log.message('Writing 2-D corrected image as %s' % oimg)
saltio.writefits(struct, oimg, clobber=clobber)
saltio.closefits(struct)
def specsens(specfile, outfile, stdfile, extfile, airmass=None, exptime=None,
stdzp=3.68e-20, function='polynomial', order=3, thresh=3, niter=5,
fitter='gaussian', clobber=True, logfile='salt.log', verbose=True):
with logging(logfile, debug) as log:
# read in the specfile and create a spectrum object
obs_spectra = st.readspectrum(specfile.strip(), error=True, ftype='ascii')
# smooth the observed spectrum
# read in the std file and convert from magnitudes to fnu
# then convert it to fwave (ergs/s/cm2/A)
std_spectra = st.readspectrum(stdfile.strip(), error=False, ftype='ascii')
std_spectra.flux = Spectrum.magtoflux(std_spectra.flux, stdzp)
std_spectra.flux = Spectrum.fnutofwave(
std_spectra.wavelength, std_spectra.flux)
# Get the typical bandpass of the standard star,
std_bandpass = np.diff(std_spectra.wavelength).mean()
# Smooth the observed spectrum to that bandpass
obs_spectra.flux = st.boxcar_smooth(obs_spectra, std_bandpass)
# read in the extinction file (leave in magnitudes)
ext_spectra = st.readspectrum(extfile.strip(), error=False, ftype='ascii')
# determine the airmass if not specified
if saltio.checkfornone(airmass) is None:
message = 'Airmass was not supplied'
raise SALTSpecError(message)
# determine the exptime if not specified
if saltio.checkfornone(exptime) is None:
message = 'Exposure Time was not supplied'
raise SALTSpecError(message)
# calculate the calibrated spectra
log.message('Calculating the calibration curve for %s' % specfile)
cal_spectra = sensfunc(
obs_spectra, std_spectra, ext_spectra, airmass, exptime)
# plot(cal_spectra.wavelength, cal_spectra.flux * std_spectra.flux)
# fit the spectra--first take a first cut of the spectra
# using the median absolute deviation to throw away bad points
cmed = np.median(cal_spectra.flux)
cmad = saltstat.mad(cal_spectra.flux)
mask = (abs(cal_spectra.flux - cmed) < thresh * cmad)
mask = np.logical_and(mask, (cal_spectra.flux > 0))
# now fit the data
# Fit using a gaussian process.
if fitter=='gaussian':
from sklearn.gaussian_process import GaussianProcess
#Instanciate a Gaussian Process model
dy = obs_spectra.var[mask] ** 0.5
dy /= obs_spectra.flux[mask] / cal_spectra.flux[mask]
y = cal_spectra.flux[mask]
gp = GaussianProcess(corr='squared_exponential', theta0=1e-2,
thetaL=1e-4, thetaU=0.1, nugget=(dy / y) ** 2.0)
X = np.atleast_2d(cal_spectra.wavelength[mask]).T
# Fit to data using Maximum Likelihood Estimation of the parameters
gp.fit(X, y)
x = np.atleast_2d(cal_spectra.wavelength).T
# Make the prediction on the meshed x-axis (ask for MSE as well)
y_pred = gp.predict(x)
cal_spectra.flux = y_pred
else:
fit=interfit(cal_spectra.wavelength[mask], cal_spectra.flux[mask], function=function, order=order, thresh=thresh, niter=niter)
fit.interfit()
cal_spectra.flux=fit(cal_spectra.wavelength)
# write the spectra out
st.writespectrum(cal_spectra, outfile, ftype='ascii')
def quickap(profile, lampid=None, findobj=False, objsection=None, skysection=None, clobber=False, logfile='saltclean.log', verbose=True):
profile = os.path.basename(profile)
outfile = profile.replace('fits', 'txt')
#open the file
struct=pyfits.open(profile)
data=struct[1].data
xlen=len(struct[1].data[0])
ylen=len(struct[1].data)
#determine the resolution element
dres = 1.0*calc_resolution(struct)
#correct for the response function
response = create_response(data, spatial_axis=1, order=3, conv=1e-2, niter=10)
data = data / response
if objsection is None:
y1=0.5*ylen-0.05*ylen
y2=0.5*ylen+0.05*ylen
objsection='[%i:%i]' % (y1,y2)
else:
y1=int(objsection[1:].split(':')[0])
y2=int(objsection[:-1].split(':')[1])
#extract object
minsize=5
thresh=5
ap_spec=extract(struct, method='normal', section=[(y1,y2)], minsize=minsize, thresh=thresh, convert=True)[0]
#if it is a lamp, do not sky subtract it
if saltsafeio.checkfornone(lampid):
write_extract(outfile, [ap_spec], outformat='ascii', clobber=clobber)
return y1,y2
if skysection is None:
skysection='%i:%i' % (y2+0.1*ylen,y2+0.2*ylen)
#sky spectrum
sy1, sy2 = skysection.split(':')
sy1=int(sy1)
sy2=int(sy2)
sk_spec=extract(struct, method='normal', section=[(sy1,sy2)], minsize=minsize, thresh=thresh, convert=True)[0]
sk_spec.ldata - sk_spec.ldata
w1=ap_spec.wave.min()
w2=ap_spec.wave.max()
nsect=10
dw=(w2-w1)/nsect
for w in np.arange(w1,w2,dw):
mask=(ap_spec.wave>w)*(ap_spec.wave<w+dw)
sk_spec=quickcross(ap_spec, sk_spec, mask=mask, dx=0.02, nstep=500)
ap_spec.ldata[mask]=ap_spec.ldata[mask]-float(y2-y1)/(sy2-sy1)*sk_spec.ldata[mask]
#set up masks for sky subtraction
amask = (ap_spec.ldata==0)
smask = (sk_spec.ldata==0)
#find offset between two spectra
d1 = ap_spec.ldata
d2 = sk_spec.ldata*float(y2-y1)/(sy2-sy1)
y=d1[d2>0]/d2[d2>0]
y=np.median(y)
#subtract the skyspectrum
clean_spectra(ap_spec, dres=2*dres, grow=dres)
#pl.plot(ap_spec.wave, ap_spec.ldata, ls='', marker='o', ms=2)
#pl.show()
#write it out and return
write_extract(outfile, [ap_spec], outformat='ascii', clobber=clobber)
#clean up the data
if clobber:
print 'Replacing pixels'
os.remove(profile)
struct[1].data[data==0]=np.median(data)
struct.writeto(profile)
return y1,y2
