def saltcombine(images,outimage, method='average', reject=None, mask=True, \ weight=True, blank=0, scale=None, statsec=None, lthresh=3, \ hthresh=3, clobber=False, logfile='salt.log',verbose=True): with logging(logfile,debug) as log: # Check the input images infiles = saltio.argunpack ('Input',images) #set reject as None reject=saltio.checkfornone(reject) if reject is not None: reject=reject.lower() #set scale scale=saltio.checkfornone(scale) if scale is not None: scale=scale.lower() #set statsec statsec=saltio.checkfornone(statsec) statsec=saltio.getSection(statsec, iraf_format=True) #Fast combine the images outstruct=imcombine(infiles, method=method, reject=reject, mask=mask, \ weight=weight, blank=blank, scale=scale, \ statsec=statsec, lthresh=lthresh, hthresh=hthresh) # housekeeping keywords fname, hist=history(level=1, wrap=False) saltkey.housekeeping(outstruct[0],'SCOMBINE', 'File Combined by SALTCOMBINE', hist) # write FITS file saltio.writefits(outstruct, outimage)
def saltcombine(images,outimage, method='average', reject=None, mask=True, \ weight=True, blank=0, scale=None, statsec=None, lthresh=3, \ hthresh=3, clobber=False, logfile='salt.log',verbose=True): with logging(logfile, debug) as log: # Check the input images infiles = saltio.argunpack('Input', images) #set reject as None reject = saltio.checkfornone(reject) if reject is not None: reject = reject.lower() #set scale scale = saltio.checkfornone(scale) if scale is not None: scale = scale.lower() #set statsec statsec = saltio.checkfornone(statsec) statsec = saltio.getSection(statsec, iraf_format=True) #Fast combine the images outstruct=imcombine(infiles, method=method, reject=reject, mask=mask, \ weight=weight, blank=blank, scale=scale, \ statsec=statsec, lthresh=lthresh, hthresh=hthresh) # housekeeping keywords fname, hist = history(level=1, wrap=False) saltkey.housekeeping(outstruct[0], 'SCOMBINE', 'File Combined by SALTCOMBINE', hist) # write FITS file saltio.writefits(outstruct, outimage)
def specsens(specfile, outfile, stdfile, extfile, airmass=None, exptime=None, stdzp=3.68e-20, function='polynomial', order=3, thresh=3, niter=5, clobber=True, logfile='salt.log',verbose=True): with logging(logfile,debug) as log: #read in the specfile and create a spectrum object obs_spectra=st.readspectrum(specfile, error=True, ftype='ascii') #read in the std file and convert from magnitudes to fnu #then convert it to fwave (ergs/s/cm2/A) std_spectra=st.readspectrum(stdfile, error=False, ftype='ascii') std_spectra.flux=Spectrum.magtoflux(std_spectra.flux, stdzp) std_spectra.flux=Spectrum.fnutofwave(std_spectra.wavelength, std_spectra.flux) #read in the extinction file (leave in magnitudes) ext_spectra=st.readspectrum(extfile, error=False, ftype='ascii') #determine the airmass if not specified if saltio.checkfornone(airmass) is None: message='Airmass was not supplied' raise SALTSpecError(message) #determine the exptime if not specified if saltio.checkfornone(airmass) is None: message='Exposure Time was not supplied' raise SALTSpecError(message) #calculate the calibrated spectra log.message('Calculating the calibration curve for %s' % specfile) cal_spectra=sensfunc(obs_spectra, std_spectra, ext_spectra, airmass, exptime) #fit the spectra--first take a first cut of the spectra #using the median absolute deviation to throw away bad points cmed=np.median(cal_spectra.flux) cmad=saltstat.mad(cal_spectra.flux) mask=(abs(cal_spectra.flux-cmed)<thresh*cmad) mask=(cal_spectra.flux>0) #now fit the data fit=interfit(cal_spectra.wavelength[mask], cal_spectra.flux[mask], function=function, order=order, thresh=thresh, niter=niter) fit.interfit() #print 'plotting...' #figure() #plot(cal_spectra.wavelength, cal_spectra.flux) #plot(obs_spectra.wavelength, obs_spectra.flux*cal_spectra.flux.mean()/obs_spectra.flux.mean()) #plot(std_spectra.wavelength, std_spectra.flux*cal_spectra.flux.mean()/std_spectra.flux.mean()) #plot(cal_spectra.wavelength, fit(cal_spectra.wavelength)) #show() #write the spectra out cal_spectra.flux=fit(cal_spectra.wavelength) st.writespectrum(cal_spectra, outfile, ftype='ascii')
def quickspec(profile, lampid=None, findobj=False, objsection=None, skysection=None, clobber=False, logfile='saltclean.log', verbose=True): """From mosaicked data, produce wavelength calibrated files""" struct=pyfits.open(profile) xlen=len(struct[1].data[0]) ylen=len(struct[1].data) print xlen,ylen badpixelimage=None caltype='rss' function='polynomial' order=3 skysub=True if saltsafeio.checkfornone(lampid): skysub=False if objsection is None: y1=0.5*ylen-0.05*ylen y2=0.5*ylen+0.05*ylen objsection='[%i:%i]' % (y1,y2) else: y1=int(objsection[1:].split(':')[0]) y2=int(objsection[:-1].split(':')[1]) if skysection is None: skysection='%i:%i' % (y2+0.1*ylen,y2+0.2*ylen) thresh=3.0 logfile='saltspec.log' specreduce(images=profile,badpixelimage=badpixelimage,caltype=caltype, function=function, order=order, skysub=skysub, skysection=skysection, findobj=findobj, objsection=objsection, thresh=thresh, clobber=clobber, logfile=logfile, verbose=verbose) return y1,y2
def saltprepare(images,outimages,outpref,createvar=False, badpixelimage=None, clobber=True,logfile='salt.log',verbose=True): with logging(logfile,debug) as log: # Check the input images infiles = saltio.argunpack ('Input',images) # create list of output files outfiles=saltio.listparse('Outfile', outimages, outpref,infiles,'') #verify that the input and output lists are the same length saltio.comparelists(infiles,outfiles,'Input','output') # open the badpixel image if saltio.checkfornone(badpixelimage) is None: badpixelstruct=None else: try: badpixelstruct = saltio.openfits(badpixelimage) except saltio.SaltIOError,e: msg='badpixel image must be specificied\n %s' % e raise SaltError(msg) # open each raw image file for img, oimg, in zip(infiles, outfiles): #open the fits file struct=saltio.openfits(img) #if createvar, throw a warning if it is using slotmode if saltkey.fastmode(saltkey.get('DETMODE', struct[0])) and createvar: msg='Creating variance frames for slotmode data in %s' % img log.warning(msg) # identify instrument instrume,keyprep,keygain,keybias,keyxtalk,keyslot = saltkey.instrumid(struct) # has file been prepared already? try: key = struct[0].header[keyprep] message = 'ERROR -- SALTPREPARE: File ' + infile message += ' has already been prepared' raise SaltError(message) except: pass # prepare file struct=prepare(struct,createvar=createvar, badpixelstruct=badpixelstruct) # housekeeping keywords fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref']) saltkey.housekeeping(struct[0],keyprep, 'File prepared for IRAF', hist) # write FITS file saltio.writefits(struct,oimg, clobber=clobber) saltio.closefits(struct) message = 'SALTPREPARE -- %s => %s' % (img, oimg) log.message(message, with_header=False)
def checkforpropid(image,propids): """subroutine to check to see if the propid keyword exists returns status """ #open up the image struct=pyfits.open(image, mode='update') if struct: #get proposal code propid=saltkey.get('PROPID', struct[0]) #check to see if it is none if saltio.checkfornone(propid) is None or propid is 'None': message='\nNo value in PROPID keyword for %s' % image raise SaltError(message) #check to see if it is an eng or cal proposal if propid.count('ENG_') or propid.count('CAL_'): return #clean up junk ones if propid in ['BIAS','COMMON','JUNK','TEST','UNKNOWN']: return #check to see if it is in the sdb if propid not in propids: message='\n%s for PROPID keyword for %s is invalid' % (propid, image) raise SaltError(message)
def specselfid(images, outimages, outpref, refimage=None, ystart='middlerow', rstep=3, clobber=False, logfile='salt.log', verbose=True): with logging(logfile, debug) as log: # set up the variables infiles = [] outfiles = [] # Check the input images infiles = saltio.argunpack('Input', images) # create list of output files outfiles = saltio.listparse( 'Outimages', outimages, outpref, infiles, '') # set up defaults if saltio.checkfornone(refimage) is not None: rhdu = saltio.openfits(refimage) else: refimage = None # read in rectify each image for img, oimg, in zip(infiles, outfiles): hdu = saltio.openfits(img) log.message( 'Performing self-identification and rectification on %s' % img) for i in range(1, len(hdu)): if hdu[i].name == 'SCI': if refimage is None: sdata = hdu[i].data else: sdata = rhdu[i].data hdu[i].data = selfid( hdu[i].data, sdata, ystart=ystart, rstep=rstep) if saltkey.found('VAREXT', hdu[i]): varext = saltkey.get('VAREXT', hdu[i]) hdu[varext].data = selfid( hdu[varext].data, sdata, ystart=ystart, rstep=rstep) if saltkey.found('BPMEXT', hdu[i]): bpmext = saltkey.get('BPMEXT', hdu[i]) hdu[bpmext].data = selfid( hdu[bpmext].data, sdata, ystart=ystart, rstep=rstep) # write out the oimg saltio.writefits(hdu, oimg, clobber=clobber)
def specselfid(images, outimages, outpref, refimage=None, ystart='middlerow', rstep=3, clobber=False, logfile='salt.log', verbose=True): with logging(logfile, debug) as log: # set up the variables infiles = [] outfiles = [] # Check the input images infiles = saltio.argunpack('Input', images) # create list of output files outfiles = saltio.listparse('Outimages', outimages, outpref, infiles, '') # set up defaults if saltio.checkfornone(refimage) is not None: rhdu = saltio.openfits(refimage) else: refimage = None # read in rectify each image for img, oimg, in zip(infiles, outfiles): hdu = saltio.openfits(img) log.message( 'Performing self-identification and rectification on %s' % img) for i in range(1, len(hdu)): if hdu[i].name == 'SCI': if refimage is None: sdata = hdu[i].data else: sdata = rhdu[i].data hdu[i].data = selfid(hdu[i].data, sdata, ystart=ystart, rstep=rstep) if saltkey.found('VAREXT', hdu[i]): varext = saltkey.get('VAREXT', hdu[i]) hdu[varext].data = selfid(hdu[varext].data, sdata, ystart=ystart, rstep=rstep) if saltkey.found('BPMEXT', hdu[i]): bpmext = saltkey.get('BPMEXT', hdu[i]) hdu[bpmext].data = selfid(hdu[bpmext].data, sdata, ystart=ystart, rstep=rstep) # write out the oimg saltio.writefits(hdu, oimg, clobber=clobber)
def specextract(images, outfile, method='normal', section=None, thresh=3.0, minsize=3.0, outformat='ascii', ext=1, convert=True, clobber=True, logfile='salt.log', verbose=True): with logging(logfile, debug) as log: # Check the input images infiles = saltio.argunpack('Input', images) # create list of output files outfiles = saltio.argunpack('outfile', outfile) if method is 'weighted': msg = 'This mode is not supported yet' raise SALTSpecError(msg) section = saltio.checkfornone(section) if section is not None: sections = saltio.getSection(section, iraf_format=False) section = [] for i in range(0, len(sections), 2): section.append((sections[i], sections[i + 1])) # Identify the lines in each file for img, ofile in zip(infiles, outfiles): log.message('\nExtracting spectrum in image %s to %s' % (img, ofile), with_header=False, with_stdout=verbose) # open the images hdu = saltio.openfits(img) ap_list = extract(hdu, ext=ext, method=method, section=section, minsize=minsize, thresh=thresh, convert=convert) # write the spectra out if ap_list: write_extract(ofile.strip(), ap_list, outformat=outformat, clobber=clobber) log.message('', with_header=False, with_stdout=verbose)
def speccal(specfile, outfile, calfile, extfile, airmass=None, exptime=None, clobber=True, logfile='salt.log', verbose=True): with logging(logfile, debug) as log: # read in the specfile and create a spectrum object obs_spectra = st.readspectrum(specfile, error=True, ftype='ascii') # read in the std file and convert from magnitudes to fnu # then convert it to fwave (ergs/s/cm2/A) cal_spectra = st.readspectrum(calfile, error=False, ftype='ascii') # read in the extinction file (leave in magnitudes) ext_spectra = st.readspectrum(extfile, error=False, ftype='ascii') # determine the airmass if not specified if saltio.checkfornone(airmass) is None: message = 'Airmass was not supplied' raise SALTSpecError(message) # determine the exptime if not specified if saltio.checkfornone(airmass) is None: message = 'Exposure Time was not supplied' raise SALTSpecError(message) # calculate the calibrated spectra log.message('Calculating the calibration curve for %s' % specfile) error = False try: if obs_spectra.var is not None: error = True except: error = False flux_spectra = calfunc( obs_spectra, cal_spectra, ext_spectra, airmass, exptime, error) # write the spectra out st.writespectrum(flux_spectra, outfile, ftype='ascii', error=error)
def specextract( images, outfile, method="normal", section=None, thresh=3.0, minsize=3.0, outformat="ascii", ext=1, convert=True, clobber=True, logfile="salt.log", verbose=True, ): with logging(logfile, debug) as log: # Check the input images infiles = saltio.argunpack("Input", images) # create list of output files outfiles = saltio.argunpack("outfile", outfile) if method is "weighted": msg = "This mode is not supported yet" raise SALTSpecError(msg) section = saltio.checkfornone(section) if section is not None: sections = saltio.getSection(section, iraf_format=False) section = [] for i in range(0, len(sections), 2): section.append((sections[i], sections[i + 1])) # Identify the lines in each file for img, ofile in zip(infiles, outfiles): log.message( "\nExtracting spectrum in image %s to %s" % (img, ofile), with_header=False, with_stdout=verbose ) # open the images hdu = saltio.openfits(img) ap_list = extract( hdu, ext=ext, method=method, section=section, minsize=minsize, thresh=thresh, convert=convert ) # write the spectra out if ap_list: write_extract(ofile.strip(), ap_list, outformat=outformat, clobber=clobber) log.message("", with_header=False, with_stdout=verbose)
def speccal( specfile, outfile, calfile, extfile, airmass=None, exptime=None, clobber=True, logfile="salt.log", verbose=True ): with logging(logfile, debug) as log: # read in the specfile and create a spectrum object obs_spectra = st.readspectrum(specfile, error=True, ftype="ascii") # read in the std file and convert from magnitudes to fnu # then convert it to fwave (ergs/s/cm2/A) cal_spectra = st.readspectrum(calfile, error=False, ftype="ascii") # read in the extinction file (leave in magnitudes) ext_spectra = st.readspectrum(extfile, error=False, ftype="ascii") # determine the airmass if not specified if saltio.checkfornone(airmass) is None: message = "Airmass was not supplied" raise SALTSpecError(message) # determine the exptime if not specified if saltio.checkfornone(airmass) is None: message = "Exposure Time was not supplied" raise SALTSpecError(message) # calculate the calibrated spectra log.message("Calculating the calibration curve for %s" % specfile) error = False try: if obs_spectra.var is not None: error = True except: error = False flux_spectra = calfunc(obs_spectra, cal_spectra, ext_spectra, airmass, exptime, error) # write the spectra out st.writespectrum(flux_spectra, outfile, ftype="ascii", error=error)
def saltfpcalring(images,outfile, waves, method=None, thresh=5, minsize=10, niter=3, conv=0.05, axc=None, ayc=None, clobber=False,logfile='salt.log',verbose=True): """Fits rings in Fabry-Perot ring images""" with logging(logfile,debug) as log: # Check the input images infiles = saltio.argunpack ('Input',images) # if the outfile exists and not clobber, fail saltio.overwrite(outfile,clobber) #open the output file fout = saltio.openascii(outfile, 'w') #make sure that the list of waves is convertable to a numpy array #convert to floats in case the wave is given as a string if isinstance(waves, str): waves=[float(x) for x in waves.split(',')] try: waves=np.array(waves) except: raise SaltError('%s is not convertable to a numpy array' % waves) #check the value of method method=saltio.checkfornone(method) #setup the output file fout.write('#Comment\n') fout.write('# radius err xc yc z ut wave dn file\n') # open each image and detect the ring for img,w in zip(infiles, waves): #open the image hdu=saltio.openfits(img) #measure the ring in each file xc, yc, radius, err, z, ut=make_calring(hdu, method=method, thresh=thresh, niter=niter, conv=conv, minsize=minsize, axc=axc, ayc=ayc) #output the results outstr=' %7.2f %6.2f %7.2f %7.2f %6.2f %7.4f %8.3f 0 %s\n' % (radius, err, xc, yc, z, ut, w, img) fout.write(outstr) log.message(outstr.strip(), with_stdout=verbose, with_header=False) fout.close()
def specprepare(images,outimages,outpref,badpixelimage='', \ clobber=True,logfile='salt.log',verbose=True): with logging(logfile,debug) as log: # Check the input images infiles = saltio.argunpack ('Input',images) # create list of output files outfiles=saltio.listparse('Outfile', outimages, outpref,infiles,'') #verify that the input and output lists are the same length saltio.comparelists(infiles,outfiles,'Input','output') # open the badpixel image if saltio.checkfornone(badpixelimage) is None: badpixelstruct=None else: try: badpixelstruct = saltio.openfits(badpixelimage) except saltio.SaltIOError,e: msg='badpixel image must be specificied\n %s' % e raise SALTSpecError(msg) # open each raw image file for img, oimg, in zip(infiles, outfiles): #open the fits file struct=saltio.openfits(img) # prepare file struct=prepare(struct,badpixelstruct) # write FITS file saltio.writefits(struct,oimg, clobber=clobber) saltio.closefits(struct) message = 'SPECPREPARE -- %s => %s' % (img, oimg) log.message(message)
def specprepare(images, outimages, outpref, badpixelimage='', clobber=True, logfile='salt.log', verbose=True): with logging(logfile, debug) as log: # Check the input images infiles = saltio.argunpack('Input', images) # create list of output files outfiles = saltio.listparse('Outfile', outimages, outpref, infiles, '') # verify that the input and output lists are the same length saltio.comparelists(infiles, outfiles, 'Input', 'output') # open the badpixel image if saltio.checkfornone(badpixelimage) is None: badpixelstruct = None else: try: badpixelstruct = saltio.openfits(badpixelimage) except saltio.SaltIOError as e: msg = 'badpixel image must be specificied\n %s' % e raise SALTSpecError(msg) # open each raw image file for img, oimg, in zip(infiles, outfiles): # open the fits file struct = saltio.openfits(img) # prepare file struct = prepare(struct, badpixelstruct) # write FITS file saltio.writefits(struct, oimg, clobber=clobber) saltio.closefits(struct) message = 'SPECPREPARE -- %s => %s' % (img, oimg) log.message(message)
raise SALTMySQLError(message) #get the obsstats #ObsStatsId=getobsstatsid(db, date) try: filesize = os.path.getsize(PipelineFileNameString) / 1024.0 except OSError: filesize = os.path.getsize(FileNameString) / 1024.0 except: filesize = 0 #get the block id try: BlockString = ImageHeader['BLOCKID'].strip() BlockString = saltio.checkfornone(BlockString) try: if int(BlockString) == 0: BlockString = None except: pass except KeyError: BlockString = '' #get the block id try: BlockVisitString = ImageHeader['BVISITID'].strip() BlockVisitString = saltio.checkfornone(BlockVisitString) try: if int(BlockVisitString) == 0: BlockVisitString = None except: pass
def create_insert(db, ImageHeader, FileNameString, PipelineFileNameString): #set up all the image header variables ExposureTimeString=createstring(ImageHeader, 'EXPTIME', '%5.3f') TelEpochString=createstring(ImageHeader, 'EQUINOX', '%6.2f', '2000.0') TargetNameString=createstring(ImageHeader, 'OBJECT', '%s') InstrueNameString=createstring(ImageHeader, 'INSTRUME', '%s') ObsModeString=createstring(ImageHeader, 'OBSMODE', '%s') DetModeString=createstring(ImageHeader, 'DETMODE', '%s') InstrueNameString=createstring(ImageHeader, 'INSTRUME', '%s') ProposalCodeString=createstring(ImageHeader, 'PROPID', '%s', default='UNKNOWN') ObservatString=createstring(ImageHeader, 'OBSERVAT', '%s') ProposalString=createstring(ImageHeader, 'PROPOSAL', '%s') #get the ProposalCode_Id if not ProposalCodeString.strip(): ProposalCodeString='NONE' ProposalCodeId=getpropcodeid(db, ProposalCodeString) #set the pipeline strings that are a little more difficult to set try: UTStartString=ImageHeader['DATE-OBS']+' '+ImageHeader['TIME-OBS'] except: UTStartString='1000-01-01 00:00:00' if not UTStartString.strip(): UTStartString='1000-01-01 00:00:00' try: TelRA=15.0*sex2dec(ImageHeader['TELRA']) TelDec=sex2dec(ImageHeader['TELDEC']) TelRAString='%10.7f' % TelRA TelDecString='%10.7f' % TelDec except Exception as e: UTStartString='1000-01-01 00:00:00' TelRAString='0.00000' TelDecString='100.0000' try: PipelineStartString=datatimeobs2DateTime(ImageHeader['SAL-TLM']) except Exception as inst: PipelineStartString='1000-01-01 00:00:00' #set the number of exposures nexposures=1 try: if ImageHeader['DETMODE'].count('SLOT'): nexposures=ImageHeader['NEXTEND'] #this is hardwired but should be changed to use NAMPS if ImageHeader['INSTRUME'] is 'RSS': nexposures=nexposures/6 else: nexposures=nexposures/4 except: pass if not PipelineFileNameString: PipelineFileNameString=FileNameString try: name=os.path.basename(FileNameString).split('.')[0] date = '%s-%s-%s' % (name[1:5], name[5:7], name[7:9]) except Exception as e: message='Could not determine the date for %s because %s' \ % (FileNameString, e) raise SALTMySQLError(message) #get the obsstats #ObsStatsId=getobsstatsid(db, date) try: filesize=os.path.getsize(PipelineFileNameString)/1024.0 except OSError: filesize=os.path.getsize(FileNameString)/1024.0 except: filesize=0 #get the block id try: BlockString=ImageHeader['BLOCKID'].strip() BlockString=saltio.checkfornone(BlockString) try: if int(BlockString)==0: BlockString=None except: pass except KeyError: BlockString='' #get the block id try: BlockVisitString=ImageHeader['BVISITID'].strip() BlockVisitString=saltio.checkfornone(BlockVisitString) try: if int(BlockVisitString)==0: BlockVisitString=None except: pass except KeyError: BlockVisitString='' #create the insertion command for the data try: insert_command='StepStats_Id=1,' insert_command += "UTStart='"+UTStartString+"'," insert_command += "ProposalCode_Id=%i," % ProposalCodeId insert_command += "Target_Name='"+TargetNameString+"'," insert_command += "ExposureTime='"+ExposureTimeString+"'," insert_command += "TelRA='"+TelRAString+"'," insert_command += "TelDec='"+TelDecString+"'," insert_command += "TelEpoch='"+TelEpochString+"'," insert_command += "FileName='"+FileNameString+"'," insert_command += "PipelineDate='"+PipelineStartString+"'," insert_command += "PipelineFileName='"+PipelineFileNameString+"'," insert_command += "INSTRUME='"+InstrueNameString+"'," insert_command += "OBSMODE='"+ObsModeString+"'," insert_command += "DETMODE='"+DetModeString+"'," insert_command += "NExposures='%i',"%nexposures insert_command += "FileSize='%i'"%filesize if BlockString: insert_command += ",Block_Id='%s'"%BlockString if BlockVisitString: insert_command += ",BlockVisit_Id='%s'"%BlockVisitString except Exception as e: message='Could not create insert command because %s' % e raise SALTMySQLError(message) return insert_command
def hrsclean(images, outpath, obslogfile=None, subover=True, trim=True, masbias=None, subbias=True, median=False, function='polynomial', order=5, rej_lo=3, rej_hi=3, niter=5, interp='linear', clobber=False, logfile='salt.log',verbose=True): """Convert MEF HRS data into a single image. If variance frames and BPMs, then convert them to the same format as well. Returns an MEF image but that is combined into a single frame """ with logging(logfile,debug) as log: # Check the input images infiles = saltio.argunpack ('Input',images) # create list of output files outpath=saltio.abspath(outpath) if saltio.checkfornone(obslogfile) is None: raise SaltError('Obslog file is required') # Delete the obslog file if it already exists if (os.path.isfile(obslogfile) and clobber) or not os.path.isfile(obslogfile): if os.path.isfile(obslogfile): saltio.delete(obslogfile) #read in the obsveration log or create it headerDict=obslog(infiles, log) obsstruct=createobslogfits(headerDict) saltio.writefits(obsstruct, obslogfile) else: obsstruct=saltio.openfits(obslogfile) #create the list of bias frames and process them filename=obsstruct.data.field('FILENAME') detmode=obsstruct.data.field('DETMODE') ccdtype=obsstruct.data.field('OBJECT') biaslist=filename[ccdtype=='Bias'] masterbias_dict={} if log: log.message('Processing Bias Frames') for img in infiles: if os.path.basename(img) in biaslist: #open the image struct=pyfits.open(img) bimg=outpath+'bgph'+os.path.basename(img) #print the message if log: message='Processing Zero frame %s' % img log.message(message, with_stdout=verbose, with_header=False) #process the image struct=clean(struct, createvar=False, badpixelstruct=None, mult=True, subover=subover, trim=trim, subbias=False, imstack=False, bstruct=None, median=median, function=function, order=order, rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, log=log, verbose=verbose) #write the file out # housekeeping keywords fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref']) saltkey.housekeeping(struct[0],'HPREPARE', 'Images have been prepared', hist) saltkey.new('HGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0]) #saltkey.new('HXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0]) saltkey.new('HBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0]) # write FITS file saltio.writefits(struct,bimg, clobber=clobber) saltio.closefits(struct) #add files to the master bias list masterbias_dict=compareimages(struct, bimg, masterbias_dict, keylist=hrsbiasheader_list) #create the master bias frame for i in list(masterbias_dict.keys()): bkeys=masterbias_dict[i][0] blist=masterbias_dict[i][1:] mbiasname=outpath+createmasterbiasname(blist, bkeys, x1=5, x2=13) bfiles=','.join(blist) saltcombine(bfiles, mbiasname, method='median', reject='sigclip', mask=False, weight=False, blank=0, scale=None, statsec=None, lthresh=3, \ hthresh=3, clobber=False, logfile=logfile,verbose=verbose) #apply full reductions to the science data for img in infiles: nimg=os.path.basename(img) if not nimg in biaslist: #open the image struct=pyfits.open(img) simg=outpath+'mbgph'+os.path.basename(img) #print the message if log: message='Processing science frame %s' % img log.message(message, with_stdout=verbose) #get master bias frame masterbias=get_masterbias(struct, masterbias_dict, keylist=hrsbiasheader_list) if masterbias: subbias=True bstruct=saltio.openfits(masterbias) else: subbias=False bstruct=None #process the image struct=clean(struct, createvar=False, badpixelstruct=None, mult=True, subover=subover, trim=trim, subbias=subbias, imstack=True, bstruct=bstruct, median=median, function=function, order=order, rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, log=log, verbose=verbose) #write the file out # housekeeping keywords fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref']) saltkey.housekeeping(struct[0],'HPREPARE', 'Images have been prepared', hist) saltkey.new('HGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0]) #saltkey.new('HXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0]) saltkey.new('HBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0]) # write FITS file saltio.writefits(struct,simg, clobber=clobber) saltio.closefits(struct) return
def saltquery(selection, logic, startdate, enddate, outfile=None, sdbhost='sdb.saao', sdbname='sdb', \ sdbuser='', password='', clobber=False, logfile='saltlog.log', verbose=True): """Query the salt database for FITS files """ with logging(logfile,debug) as log: #check the outfiles if not saltio.checkfornone(outfile): outfile=None #check that the output file can be deleted if outfile: saltio.overwrite(outfile, clobber) #open outfile if outfile: fout=saltio.openascii(outfile, 'w') #connect to the database sdb=saltmysql.connectdb(sdbhost,sdbname,sdbuser,password) #Create a list of the selection and then unpack it selection_list = saltio.argunpack('Selection', selection) selection=','.join(selection_list) #write out the header for the outfile outstr='#'+' '.join(['%s' % x for x in selection_list]) if outfile: fout.write(outstr+'\n') else: print outstr #set up the table rsstable=''' FileData left join FitsHeaderImage using (FileData_Id) inner join FitsHeaderRss using (FileData_Id) ''' #set up the table scamtable=''' FileData left join FitsHeaderImage using (FileData_Id) inner join FitsHeaderSalticam using (FileData_Id) ''' #set up the logic logic=makelogic(logic, startdate, enddate) for tab in [rsstable, scamtable]: msg=''' Mysql querying data is: SELECT %s FROM %s WHERE %s ''' % (selection, tab, logic) log.message(msg, with_stdout=verbose) record=saltmysql.select(sdb, selection, tab, logic) print record for r in record: outstr=' '.join(['%s' % x for x in r]) if outfile: fout.write(outstr+'\n') else: print outstr #close outfile if outfile: fout.close()
def quickap(profile, lampid=None, findobj=False, objsection=None, skysection=None, clobber=False, logfile='saltclean.log', verbose=True): profile = os.path.basename(profile) outfile = profile.replace('fits', 'txt') #open the file struct=pyfits.open(profile) data=struct[1].data xlen=len(struct[1].data[0]) ylen=len(struct[1].data) #determine the resolution element dres = 1.0*calc_resolution(struct) #correct for the response function response = create_response(data, spatial_axis=1, order=3, conv=1e-2, niter=10) data = data / response if objsection is None: y1=0.5*ylen-0.05*ylen y2=0.5*ylen+0.05*ylen objsection='[%i:%i]' % (y1,y2) else: y1=int(objsection[1:].split(':')[0]) y2=int(objsection[:-1].split(':')[1]) #extract object minsize=5 thresh=5 ap_spec=extract(struct, method='normal', section=[(y1,y2)], minsize=minsize, thresh=thresh, convert=True)[0] #if it is a lamp, do not sky subtract it if saltsafeio.checkfornone(lampid): write_extract(outfile, [ap_spec], outformat='ascii', clobber=clobber) return y1,y2 if skysection is None: skysection='%i:%i' % (y2+0.1*ylen,y2+0.2*ylen) #sky spectrum sy1, sy2 = skysection.split(':') sy1=int(sy1) sy2=int(sy2) sk_spec=extract(struct, method='normal', section=[(sy1,sy2)], minsize=minsize, thresh=thresh, convert=True)[0] sk_spec.ldata - sk_spec.ldata w1=ap_spec.wave.min() w2=ap_spec.wave.max() nsect=10 dw=(w2-w1)/nsect for w in np.arange(w1,w2,dw): mask=(ap_spec.wave>w)*(ap_spec.wave<w+dw) sk_spec=quickcross(ap_spec, sk_spec, mask=mask, dx=0.02, nstep=500) ap_spec.ldata[mask]=ap_spec.ldata[mask]-float(y2-y1)/(sy2-sy1)*sk_spec.ldata[mask] #set up masks for sky subtraction amask = (ap_spec.ldata==0) smask = (sk_spec.ldata==0) #find offset between two spectra d1 = ap_spec.ldata d2 = sk_spec.ldata*float(y2-y1)/(sy2-sy1) y=d1[d2>0]/d2[d2>0] y=np.median(y) #subtract the skyspectrum clean_spectra(ap_spec, dres=2*dres, grow=dres) #pl.plot(ap_spec.wave, ap_spec.ldata, ls='', marker='o', ms=2) #pl.show() #write it out and return write_extract(outfile, [ap_spec], outformat='ascii', clobber=clobber) #clean up the data if clobber: print 'Replacing pixels' os.remove(profile) struct[1].data[data==0]=np.median(data) struct.writeto(profile) return y1,y2
def saltcharge( obsdate, outfile, sdbhost="sdb.saao", sdbname="sdb", sdbuser="", password="", clobber=False, logfile="saltlog.log", verbose=True, ): """Calculate the charged time for proposals observed during a night """ with logging(logfile, debug) as log: # check the outfiles if not saltio.checkfornone(outfile): outfile = None # check that the output file can be deleted if outfile: saltio.overwrite(outfile, clobber) # connect the database sdb = saltmysql.connectdb(sdbhost, sdbname, sdbuser, password) # get all the proposals observed during the night select_state = "distinct Proposal_Code" table_state = "FileData Join ProposalCode using (ProposalCode_Id)" logic_state = "FileName like '%" + obsdate + "%'" records = saltmysql.select(sdb, select_state, table_state, logic_state) pids = [] pid_time = {} for r in records: pids.append(r[0]) pid_time[r[0]] = 0 if len(pids) == 0: message = "There are no proposal to charge time for %s" % obsdate log.message(message) return # get the nightinfo_id night_id = saltmysql.getnightinfoid(sdb, obsdate) print night_id # get a list of all the images taken during the night select_state = "FileName, Proposal_Code, Target_Name, ExposureTime, UTSTART, INSTRUME, OBSMODE, DETMODE, CCDTYPE, NExposures" table_state = "FileData Join ProposalCode using (ProposalCode_Id) join FitsHeaderImage using (FileData_Id)" logic_state = "FileName like '%" + obsdate + "%'" img_list = saltmysql.select(sdb, select_state, table_state, logic_state) # get all the blocks visited select_state = "Block_Id, Accepted, Proposal_Code" table_state = "Block join BlockVisit using (Block_Id) join Proposal using (Proposal_Id) join ProposalCode using (ProposalCode_Id)" logic_state = "NightInfo_Id=%i" % night_id block_list = saltmysql.select(sdb, select_state, table_state, logic_state) print block_list # get the start and end of twilight nightstart = saltmysql.select(sdb, "EveningTwilightEnd", "NightInfo", "NightInfo_Id=%i" % night_id)[0][0] nightend = saltmysql.select(sdb, "MorningTwilightStart", "NightInfo", "NightInfo_Id=%i" % night_id)[0][0] print nightstart, nightend, (nightend - nightstart).seconds print # download the SOMMI log from the night try: sommi_log = saltmysql.select(sdb, "SONightLog", "NightLogs", "NightInfo_Id=%i" % night_id)[0][0] except Exception, e: msg = "Unable to read in SOMMI log for %s" % obsdate raise SaltError(msg) # add to account for error in SOMMI log if int(obsdate) < 20111027: sommi_log = sommi_log.replace("Object name", "\nObject name") # parse the sommi log point_list = parseforpointings(sommi_log) # now find all blocks observed during a night for point in point_list: # proposal for that block pid = point[0].strip() # start time for that block starttime = point[1] # find the end time for that block endtime = findnexttime(point[1], point_list, nightend) # find the start time of the first object to be observed # for this block startimage = findfirstimage(pid, starttime, img_list) lastimage = findlastimage(pid, endtime, img_list) # check to see if the end time for the last file observed for # this block if startimage is not None: if startimage > endtime: startimage = None # determine the acquisition time for the block if startimage is not None: acqdelta = (startimage - starttime).seconds else: acqdelta = -1 # find the shutter open time shuttertime = calculate_shutteropen(img_list, starttime, endtime) # if the shutter and time of the last image is substantially different from the # end of the block, then take the last image as the end of the block # *TODO* Change to use expected block time st = starttime + datetime.timedelta(0, shuttertime + acqdelta) if (endtime - st).seconds > 900: if lastimage is None: endtime = st elif (lastimage - st).seconds > 3600: endtime = st else: endtime = lastimage # account for the time for that block tblock = (endtime - starttime).seconds if acqdelta > -1: # find the associated block block_id, blockvisit_id, accepted = getblockid(sdb, night_id, pid, img_list, starttime, endtime) if accepted and pid.count("-3-"): # charged time for that block try: pid_time[pid] += tblock except KeyError: pid_time[pid] = tblock print block_id, blockvisit_id, pid, starttime, tblock, acqdelta, shuttertime, shuttertime / tblock, block_id, accepted # update the charge time slewdelta = 0 scidelta = tblock - acqdelta update_cmd = "TotalSlewTime=%i, TotalAcquisitionTime=%i, TotalScienceTime=%i" % ( slewdelta, acqdelta, scidelta, ) table_cmd = "BlockVisit" logic_cmd = "BlockVisit_Id=%i" % blockvisit_id saltmysql.update(sdb, update_cmd, table_cmd, logic_cmd) return # update the charged time information # TODO: Check to see if the block was approved ptime_tot = 0 stime_tot = 0 for k in pid_time: ptime = pid_time[k] if ptime > 0: obsratio = stime / ptime else: obsratio = 0 print "%25s %5i %5i %3.2f" % (k, ptime, stime, obsratio) if k.count("-3-"): ptime_tot += ptime stime_tot += stime # print out the total night statistics tdelta = nightend - nightstart print tdelta.seconds, ptime_tot, stime_tot
def hrsclean(images, outpath, obslogfile=None, subover=True, trim=True, masbias=None, subbias=True, median=False, function='polynomial', order=5, rej_lo=3, rej_hi=3, niter=5, interp='linear', clobber=False, logfile='salt.log', verbose=True): """Convert MEF HRS data into a single image. If variance frames and BPMs, then convert them to the same format as well. Returns an MEF image but that is combined into a single frame """ with logging(logfile, debug) as log: # Check the input images infiles = saltio.argunpack('Input', images) # create list of output files outpath = saltio.abspath(outpath) if saltio.checkfornone(obslogfile) is None: raise SaltError('Obslog file is required') # Delete the obslog file if it already exists if (os.path.isfile(obslogfile) and clobber) or not os.path.isfile(obslogfile): if os.path.isfile(obslogfile): saltio.delete(obslogfile) #read in the obsveration log or create it headerDict = obslog(infiles, log) obsstruct = createobslogfits(headerDict) saltio.writefits(obsstruct, obslogfile) else: obsstruct = saltio.openfits(obslogfile) #create the list of bias frames and process them filename = obsstruct.data.field('FILENAME') detmode = obsstruct.data.field('DETMODE') ccdtype = obsstruct.data.field('OBJECT') biaslist = filename[ccdtype == 'Bias'] masterbias_dict = {} if log: log.message('Processing Bias Frames') for img in infiles: if os.path.basename(img) in biaslist: #open the image struct = pyfits.open(img) bimg = outpath + 'bgph' + os.path.basename(img) #print the message if log: message = 'Processing Zero frame %s' % img log.message(message, with_stdout=verbose, with_header=False) #process the image struct = clean(struct, createvar=False, badpixelstruct=None, mult=True, subover=subover, trim=trim, subbias=False, imstack=False, bstruct=None, median=median, function=function, order=order, rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, log=log, verbose=verbose) #write the file out # housekeeping keywords fname, hist = history( level=1, wrap=False, exclude=['images', 'outimages', 'outpref']) saltkey.housekeeping(struct[0], 'HPREPARE', 'Images have been prepared', hist) saltkey.new('HGAIN', time.asctime(time.localtime()), 'Images have been gain corrected', struct[0]) #saltkey.new('HXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0]) saltkey.new('HBIAS', time.asctime(time.localtime()), 'Images have been de-biased', struct[0]) # write FITS file saltio.writefits(struct, bimg, clobber=clobber) saltio.closefits(struct) #add files to the master bias list masterbias_dict = compareimages(struct, bimg, masterbias_dict, keylist=hrsbiasheader_list) #create the master bias frame for i in masterbias_dict.keys(): bkeys = masterbias_dict[i][0] blist = masterbias_dict[i][1:] mbiasname = outpath + createmasterbiasname( blist, bkeys, x1=5, x2=13) bfiles = ','.join(blist) saltcombine(bfiles, mbiasname, method='median', reject='sigclip', mask=False, weight=False, blank=0, scale=None, statsec=None, lthresh=3, \ hthresh=3, clobber=False, logfile=logfile,verbose=verbose) #apply full reductions to the science data for img in infiles: nimg = os.path.basename(img) if not nimg in biaslist: #open the image struct = pyfits.open(img) simg = outpath + 'mbgph' + os.path.basename(img) #print the message if log: message = 'Processing science frame %s' % img log.message(message, with_stdout=verbose) #get master bias frame masterbias = get_masterbias(struct, masterbias_dict, keylist=hrsbiasheader_list) if masterbias: subbias = True bstruct = saltio.openfits(masterbias) else: subbias = False bstruct = None #process the image struct = clean(struct, createvar=False, badpixelstruct=None, mult=True, subover=subover, trim=trim, subbias=subbias, imstack=True, bstruct=bstruct, median=median, function=function, order=order, rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, log=log, verbose=verbose) #write the file out # housekeeping keywords fname, hist = history( level=1, wrap=False, exclude=['images', 'outimages', 'outpref']) saltkey.housekeeping(struct[0], 'HPREPARE', 'Images have been prepared', hist) saltkey.new('HGAIN', time.asctime(time.localtime()), 'Images have been gain corrected', struct[0]) #saltkey.new('HXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0]) saltkey.new('HBIAS', time.asctime(time.localtime()), 'Images have been de-biased', struct[0]) # write FITS file saltio.writefits(struct, simg, clobber=clobber) saltio.closefits(struct) return
def saltfpringfind(images, method=None, section=None, thresh=5, minsize=10, niter=5, conv=0.05, displayimage=True, clobber=False, logfile='salt.log', verbose=True): with logging(logfile, debug) as log: # Check the input images infiles = saltio.argunpack('Input', images) #check the method method = saltio.checkfornone(method) # read in the section section = saltio.checkfornone(section) if section is None: pass else: section = saltio.getSection(section) # open each raw image file for img in infiles: #open the fits file struct = saltio.openfits(img) data = struct[0].data #determine the background value for the image if section is None: #if section is none, just use all pixels greater than zero bdata = data[data > 0] else: y1, y2, x1, x2 = section bdata = data[y1:y2, x1:x2] bmean, bmedian, bstd = iterstat(bdata, sig=thresh, niter=niter, verbose=False) message="Image Background Statistics\n%30s %6s %8s %8s\n%30s %5.4f %5.4f %5.4f\n" % \ ('Image', 'Mean', 'Median', 'Std',img, bmean, bmedian, bstd) log.message(message, with_stdout=verbose) mdata = data * (data - bmean > thresh * bstd) #prepare the first guess for the image ring_list = findrings(data, thresh=thresh, niter=niter, minsize=minsize) #if specified, find the center of the ring if method is not None: for i in range(len(ring_list)): ring_list[i] = findcenter(data, ring_list[i], method, niter=niter, conv=conv) #if one peak: no rings. If two peaks: one ring, if two peaks: four rings if len(ring_list) == 1: msg = "One ring dected in image" else: msg = "%i rings found in image" % len(ring_list) log.message(message, with_stdout=verbose) if displayimage: regfile = img.replace('.fits', '.reg') if clobber and os.path.isfile(regfile): fout = saltio.delete(regfile) fout = open(regfile, 'w') fout.write("""# Region file format: DS9 version 4.1 # Filename: %s global color=green dashlist=8 3 width=1 font="helvetica 10 normal roman" select=1 highlite=1 dash=0 fixed=0 edit=1 move=1 delete=1 include=1 source=1 physical """ % img) for ring in ring_list: fout.write('circle(%f, %f, %f)\n' % (ring.xc, ring.yc, ring.prad)) fout.write('circle(%f, %f, %f)\n' % (ring.xc, ring.yc, ring.prad - 3 * ring.sigma)) fout.write('circle(%f, %f, %f)\n' % (ring.xc, ring.yc, ring.prad + 3 * ring.sigma)) fout.close() display(img, catname=regfile, rformat='reg') message = 'Ring Parameters\n%30s %6s %6s %6s\n' % ('Image', 'XC', 'YC', 'Radius') log.message(message, with_stdout=verbose) for ring in ring_list: msg = '%30s %6.2f %6.2f %6.2f\n' % (img, ring.xc, ring.yc, ring.prad) log.message(msg, with_header=False, with_stdout=verbose)
def saltcalibrations( propcode, outfile=None, sdbhost="sdb.saao", sdbname="sdb", sdbuser="", password="", clobber=False, logfile="saltlog.log", verbose=True, ): """Seach the salt database for FITS files """ with logging(logfile, debug) as log: # check the outfiles if not saltio.checkfornone(outfile): outfile = None # check that the output file can be deleted if outfile: saltio.overwrite(outfile, clobber) fout = open(oufile, "w") # connect to the database sdb = saltmysql.connectdb(sdbhost, sdbname, sdbuser, password) # determine the associated username = saltmysql.getpiptusername(sdb, propcode) userdir = "/salt/ftparea/%s/spst" % username if not os.path.exists(userdir): saltio.createdir(userdir) log.message("Will copy data to %s" % userdir) # Find all the data assocated with a proposal cmd_select = "d.FileName,d.FileData_Id, CCDTYPE, d.DETMODE, d.OBSMODE, CCDSUM, GAINSET, ROSPEED, FILTER, GRATING, GRTILT, CAMANG, MASKID" cmd_table = """ FileData as d left join FitsHeaderImage using (FileData_Id) left join FitsHeaderRss using (FileData_Id) left join ProposalCode using (ProposalCode_Id) """ cmd_logic = 'Proposal_Code="%s" and CCDTYPE="OBJECT" and d.OBSMODE="SPECTROSCOPY"' % (propcode) record = saltmysql.select(sdb, cmd_select, cmd_table, cmd_logic) # loop through all the results and return only the Set of identical results caldict = create_caldict(record) # prepare for writing out the results outstr = "" if outfile: fout.write(outstr + "\n") else: print outstr # now find all the cal_spst that have the same settings cmd_select = "d.FileName,d.FileData_Id, CCDTYPE, d.DETMODE, d.OBSMODE, CCDSUM, GAINSET, ROSPEED, FILTER, GRATING, GRTILT, CAMANG, MASKID" cmd_table = """ FileData as d left join FitsHeaderImage using (FileData_Id) left join FitsHeaderRss using (FileData_Id) left join ProposalCode using (ProposalCode_Id) """ for r in caldict: cmd_logic = "CCDSUM='%s' and GRATING='%s' and GRTILT='%s' and CAMANG='%s' and Proposal_Code='CAL_SPST'" % ( caldict[r][3], caldict[r][7], caldict[r][8], caldict[r][9], ) # cmd_logic="CCDSUM='%s' and GRATING='%s' and AR_STA='%s' " % (caldict[r][3], caldict[r][7], caldict[r][9]) log.message(cmd_logic, with_header=False) record = saltmysql.select(sdb, cmd_select, cmd_table, cmd_logic) # print record # write out hte results for r in record: outstr = " ".join(["%s" % x for x in r]) if outfile: fout.write(outstr + "\n") else: log.message(outstr, with_header=False) # copy to the user directory cfile = makefilename(r[0], state="product") shutil.copy(cfile, userdir) cfile = makefilename(r[0], state="raw") shutil.copy(cfile, userdir) # close outfile if outfile: fout.close()
def specslitnormalize(images, outimages, outpref, response=None, response_output=None, order=2, conv=1e-2, niter=20, startext=0, clobber=False, logfile='salt.log', verbose=True): with logging(logfile, debug) as log: # Check the input images infiles = saltio.argunpack('Input', images) # create list of output files outfiles = saltio.listparse('Outfile', outimages, outpref, infiles, '') # read in the response function response = saltio.checkfornone(response) if response: log.message('Loading response from %s' % response) response = readresponse(response) # Identify the lines in each file for img, ofile in zip(infiles, outfiles): # open the image hdu = saltio.openfits(img) for i in range(startext, len(hdu)): if hdu[i].name == 'SCI': log.message('Normalizing extension %i in %s' % (i, img)) # things that will change for each slit # set up the data for the source try: data = hdu[i].data except Exception as e: message = \ 'Unable to read in data array in %s because %s' % \ (img, e) raise SALTSpecError(message) if response is None: response = create_response(data, spatial_axis=1, order=order, conv=conv, niter=niter) if response_output: write_response(response, clobber=clobber) else: # add a check that the response is the same shape as # the data if len(response) != data.shape[0]: raise SALTSpecError( 'Length of response function does not equal size of image array' ) # correct the data data = data / response # correct the variance frame if saltkey.found('VAREXT', hdu[i]): vhdu = saltkey.get('VAREXT', hdu[i]) hdu[vhdu].data = hdu[vhdu].data / response saltio.writefits(hdu, ofile, clobber=clobber)
raise SALTMySQLError(message) #get the obsstats #ObsStatsId=getobsstatsid(db, date) try: filesize=os.path.getsize(PipelineFileNameString)/1024.0 except OSError: filesize=os.path.getsize(FileNameString)/1024.0 except: filesize=0 #get the block id try: BlockString=ImageHeader['BLOCKID'].strip() BlockString=saltio.checkfornone(BlockString) try: if int(BlockString)==0: BlockString=None except: pass except KeyError: BlockString='' #create the insertion command for the data try: insert_command='StepStats_Id=1,' insert_command += "UTStart='"+UTStartString+"'," insert_command += "ProposalCode_Id=%i," % ProposalCodeId insert_command += "Target_Name='"+TargetNameString+"',"
def saltfpringfind(images, method=None, section=None, thresh=5, minsize=10, niter=5, conv=0.05, displayimage=True, clobber=False, logfile='salt.log',verbose=True): with logging(logfile,debug) as log: # Check the input images infiles = saltio.argunpack ('Input',images) #check the method method=saltio.checkfornone(method) # read in the section section=saltio.checkfornone(section) if section is None: pass else: section=saltio.getSection(section) # open each raw image file for img in infiles: #open the fits file struct=saltio.openfits(img) data=struct[0].data #determine the background value for the image if section is None: #if section is none, just use all pixels greater than zero bdata=data[data>0] else: y1,y2,x1,x2=section bdata=data[y1:y2,x1:x2] bmean, bmedian, bstd=iterstat(bdata, sig=thresh, niter=niter, verbose=False) message="Image Background Statistics\n%30s %6s %8s %8s\n%30s %5.4f %5.4f %5.4f\n" % \ ('Image', 'Mean', 'Median', 'Std',img, bmean, bmedian, bstd) log.message(message, with_stdout=verbose) mdata=data*(data-bmean>thresh*bstd) #prepare the first guess for the image ring_list=findrings(data, thresh=thresh, niter=niter, minsize=minsize) #if specified, find the center of the ring if method is not None: for i in range(len(ring_list)): ring_list[i]=findcenter(data, ring_list[i], method, niter=niter, conv=conv) #if one peak: no rings. If two peaks: one ring, if two peaks: four rings if len(ring_list)==1: msg="One ring dected in image" else: msg="%i rings found in image" % len(ring_list) log.message(message, with_stdout=verbose) if displayimage: regfile=img.replace('.fits', '.reg') if clobber and os.path.isfile(regfile): fout=saltio.delete(regfile) fout=open(regfile, 'w') fout.write("""# Region file format: DS9 version 4.1 # Filename: %s global color=green dashlist=8 3 width=1 font="helvetica 10 normal roman" select=1 highlite=1 dash=0 fixed=0 edit=1 move=1 delete=1 include=1 source=1 physical """ % img) for ring in ring_list: fout.write('circle(%f, %f, %f)\n' % (ring.xc,ring.yc,ring.prad)) fout.write('circle(%f, %f, %f)\n' % (ring.xc,ring.yc,ring.prad-3*ring.sigma)) fout.write('circle(%f, %f, %f)\n' % (ring.xc,ring.yc,ring.prad+3*ring.sigma)) fout.close() display(img, catname=regfile, rformat='reg') message = 'Ring Parameters\n%30s %6s %6s %6s\n' % ('Image', 'XC', 'YC', 'Radius') log.message(message, with_stdout=verbose) for ring in ring_list: msg='%30s %6.2f %6.2f %6.2f\n' % (img, ring.xc, ring.yc, ring.prad) log.message(msg, with_header=False, with_stdout=verbose)
raise SALTMySQLError(message) #get the obsstats #ObsStatsId=getobsstatsid(db, date) try: filesize=os.path.getsize(PipelineFileNameString)/1024.0 except OSError: filesize=os.path.getsize(FileNameString)/1024.0 except: filesize=0 #get the block id try: BlockString=ImageHeader['BLOCKID'].strip() BlockString=saltio.checkfornone(BlockString) try: if int(BlockString)==0: BlockString=None except: pass except KeyError: BlockString='' #get the block id try: BlockVisitString=ImageHeader['BVISITID'].strip() BlockVisitString=saltio.checkfornone(BlockVisitString) try: if int(BlockVisitString)==0: BlockVisitString=None except: pass
def specsens(specfile, outfile, stdfile, extfile, airmass=None, exptime=None, stdzp=3.68e-20, function='polynomial', order=3, thresh=3, niter=5, fitter='gaussian', clobber=True, logfile='salt.log', verbose=True): with logging(logfile, debug) as log: # read in the specfile and create a spectrum object obs_spectra = st.readspectrum(specfile.strip(), error=True, ftype='ascii') # smooth the observed spectrum # read in the std file and convert from magnitudes to fnu # then convert it to fwave (ergs/s/cm2/A) std_spectra = st.readspectrum(stdfile.strip(), error=False, ftype='ascii') std_spectra.flux = Spectrum.magtoflux(std_spectra.flux, stdzp) std_spectra.flux = Spectrum.fnutofwave(std_spectra.wavelength, std_spectra.flux) # Get the typical bandpass of the standard star, std_bandpass = np.diff(std_spectra.wavelength).mean() # Smooth the observed spectrum to that bandpass obs_spectra.flux = st.boxcar_smooth(obs_spectra, std_bandpass) # read in the extinction file (leave in magnitudes) ext_spectra = st.readspectrum(extfile.strip(), error=False, ftype='ascii') # determine the airmass if not specified if saltio.checkfornone(airmass) is None: message = 'Airmass was not supplied' raise SALTSpecError(message) # determine the exptime if not specified if saltio.checkfornone(exptime) is None: message = 'Exposure Time was not supplied' raise SALTSpecError(message) # calculate the calibrated spectra log.message('Calculating the calibration curve for %s' % specfile) cal_spectra = sensfunc(obs_spectra, std_spectra, ext_spectra, airmass, exptime) # plot(cal_spectra.wavelength, cal_spectra.flux * std_spectra.flux) # fit the spectra--first take a first cut of the spectra # using the median absolute deviation to throw away bad points cmed = np.median(cal_spectra.flux) cmad = saltstat.mad(cal_spectra.flux) mask = (abs(cal_spectra.flux - cmed) < thresh * cmad) mask = np.logical_and(mask, (cal_spectra.flux > 0)) # now fit the data # Fit using a gaussian process. if fitter == 'gaussian': from sklearn.gaussian_process import GaussianProcess #Instanciate a Gaussian Process model dy = obs_spectra.var[mask]**0.5 dy /= obs_spectra.flux[mask] / cal_spectra.flux[mask] y = cal_spectra.flux[mask] gp = GaussianProcess(corr='squared_exponential', theta0=1e-2, thetaL=1e-4, thetaU=0.1, nugget=(dy / y)**2.0) X = np.atleast_2d(cal_spectra.wavelength[mask]).T # Fit to data using Maximum Likelihood Estimation of the parameters gp.fit(X, y) x = np.atleast_2d(cal_spectra.wavelength).T # Make the prediction on the meshed x-axis (ask for MSE as well) y_pred = gp.predict(x) cal_spectra.flux = y_pred else: fit = interfit(cal_spectra.wavelength[mask], cal_spectra.flux[mask], function=function, order=order, thresh=thresh, niter=niter) fit.interfit() cal_spectra.flux = fit(cal_spectra.wavelength) # write the spectra out st.writespectrum(cal_spectra, outfile, ftype='ascii')
def specreduce( images, badpixelimage=None, caltype="rss", function="polynomial", order=3, skysub=True, skysection=None, findobj=False, objsection=None, thresh=3.0, clobber=True, logfile="salt.log", verbose=True, ): with logging(logfile, debug) as log: # Check the input images infiles = saltio.argunpack("Input", images) # open the badpixelstruct if saltio.checkfornone(badpixelimage): badpixelstruct = saltio.openfits(badpixelimage) else: badpixelstruct = None # set up the section for sky estimate if skysection is not None: skysection = makesection(skysection) else: skysub = False # set up the section for sky estimate section = saltio.checkfornone(objsection) if section is not None: sections = saltio.getSection(section, iraf_format=False) objsection = [] for i in range(0, len(sections), 2): objsection.append((sections[i], sections[i + 1])) # determine the wavelength solutions if caltype == "line": calc_wavesol(infiles) # correct the images for img in infiles: # open the fits file struct = saltio.openfits(img) # prepare filep log.message("Preparing %s" % img) struct = prepare(struct, badpixelstruct) # rectify the spectrum log.message("Rectifying %s using %s" % (img, caltype)) struct = rectify(struct, None, caltype=caltype, function=function, order=order) # sky subtract the spectrum # assumes the data is long slit and in the middle of the field if skysub: log.message("Subtracting the sky from %s" % img) struct = skysubtract(struct, method="normal", section=skysection) # extract the spectrum log.message("Extracting the spectrum from %s" % img) print objsection aplist = extract(struct, method="normal", section=objsection, thresh=thresh) oimg = os.path.dirname(os.path.abspath(img)) + "/s" + os.path.basename(img.strip()) ofile = oimg[:-5] + ".txt" write_extract(ofile, aplist, clobber=clobber) # write FITS file log.message("Writing 2-D corrected image as %s" % oimg) saltio.writefits(struct, oimg, clobber=clobber) saltio.closefits(struct)
def specslitnormalize(images, outimages, outpref, response=None, response_output=None, order=2, conv=1e-2, niter=20, startext=0, clobber=False, logfile='salt.log', verbose=True): with logging(logfile, debug) as log: # Check the input images infiles = saltio.argunpack('Input', images) # create list of output files outfiles = saltio.listparse('Outfile', outimages, outpref, infiles, '') # read in the response function response = saltio.checkfornone(response) if response: log.message('Loading response from %s' % response) response = readresponse(response) # Identify the lines in each file for img, ofile in zip(infiles, outfiles): # open the image hdu = saltio.openfits(img) for i in range(startext, len(hdu)): if hdu[i].name == 'SCI': log.message('Normalizing extension %i in %s' % (i, img)) # things that will change for each slit # set up the data for the source try: data = hdu[i].data except Exception as e: message = \ 'Unable to read in data array in %s because %s' % \ (img, e) raise SALTSpecError(message) if response is None: response = create_response( data, spatial_axis=1, order=order, conv=conv, niter=niter) if response_output: write_response(response, clobber=clobber) else: # add a check that the response is the same shape as # the data if len(response) != data.shape[0]: raise SALTSpecError( 'Length of response function does not equal size of image array') # correct the data data = data / response # correct the variance frame if saltkey.found('VAREXT', hdu[i]): vhdu = saltkey.get('VAREXT', hdu[i]) hdu[vhdu].data = hdu[vhdu].data / response saltio.writefits(hdu, ofile, clobber=clobber)
def specreduce(images, badpixelimage=None, caltype='rss', function='polynomial', order=3, skysub=True, skysection=None, findobj=False, objsection=None, thresh=3.0, clobber=True, logfile='salt.log', verbose=True): with logging(logfile, debug) as log: # Check the input images infiles = saltio.argunpack('Input', images) # open the badpixelstruct if saltio.checkfornone(badpixelimage): badpixelstruct = saltio.openfits(badpixelimage) else: badpixelstruct = None # set up the section for sky estimate if skysection is not None: skysection = makesection(skysection) else: skysub = False # set up the section for sky estimate section = saltio.checkfornone(objsection) if section is not None: sections = saltio.getSection(section, iraf_format=False) objsection = [] for i in range(0, len(sections), 2): objsection.append((sections[i], sections[i + 1])) # determine the wavelength solutions if caltype == 'line': calc_wavesol(infiles) # correct the images for img in infiles: # open the fits file struct = saltio.openfits(img) # prepare filep log.message('Preparing %s' % img) struct = prepare(struct, badpixelstruct) # rectify the spectrum log.message('Rectifying %s using %s' % (img, caltype)) struct = rectify( struct, None, caltype=caltype, function=function, order=order) # sky subtract the spectrum # assumes the data is long slit and in the middle of the field if skysub: log.message('Subtracting the sky from %s' % img) struct = skysubtract( struct, method='normal', section=skysection) # extract the spectrum log.message('Extracting the spectrum from %s' % img) print objsection aplist = extract( struct, method='normal', section=objsection, thresh=thresh) oimg = os.path.dirname( os.path.abspath(img)) + '/s' + os.path.basename(img.strip()) ofile = oimg[:-5] + '.txt' write_extract(ofile, aplist, clobber=clobber) # write FITS file log.message('Writing 2-D corrected image as %s' % oimg) saltio.writefits(struct, oimg, clobber=clobber) saltio.closefits(struct)
def specsens(specfile, outfile, stdfile, extfile, airmass=None, exptime=None, stdzp=3.68e-20, function='polynomial', order=3, thresh=3, niter=5, fitter='gaussian', clobber=True, logfile='salt.log', verbose=True): with logging(logfile, debug) as log: # read in the specfile and create a spectrum object obs_spectra = st.readspectrum(specfile.strip(), error=True, ftype='ascii') # smooth the observed spectrum # read in the std file and convert from magnitudes to fnu # then convert it to fwave (ergs/s/cm2/A) std_spectra = st.readspectrum(stdfile.strip(), error=False, ftype='ascii') std_spectra.flux = Spectrum.magtoflux(std_spectra.flux, stdzp) std_spectra.flux = Spectrum.fnutofwave( std_spectra.wavelength, std_spectra.flux) # Get the typical bandpass of the standard star, std_bandpass = np.diff(std_spectra.wavelength).mean() # Smooth the observed spectrum to that bandpass obs_spectra.flux = st.boxcar_smooth(obs_spectra, std_bandpass) # read in the extinction file (leave in magnitudes) ext_spectra = st.readspectrum(extfile.strip(), error=False, ftype='ascii') # determine the airmass if not specified if saltio.checkfornone(airmass) is None: message = 'Airmass was not supplied' raise SALTSpecError(message) # determine the exptime if not specified if saltio.checkfornone(exptime) is None: message = 'Exposure Time was not supplied' raise SALTSpecError(message) # calculate the calibrated spectra log.message('Calculating the calibration curve for %s' % specfile) cal_spectra = sensfunc( obs_spectra, std_spectra, ext_spectra, airmass, exptime) # plot(cal_spectra.wavelength, cal_spectra.flux * std_spectra.flux) # fit the spectra--first take a first cut of the spectra # using the median absolute deviation to throw away bad points cmed = np.median(cal_spectra.flux) cmad = saltstat.mad(cal_spectra.flux) mask = (abs(cal_spectra.flux - cmed) < thresh * cmad) mask = np.logical_and(mask, (cal_spectra.flux > 0)) # now fit the data # Fit using a gaussian process. if fitter=='gaussian': from sklearn.gaussian_process import GaussianProcess #Instanciate a Gaussian Process model dy = obs_spectra.var[mask] ** 0.5 dy /= obs_spectra.flux[mask] / cal_spectra.flux[mask] y = cal_spectra.flux[mask] gp = GaussianProcess(corr='squared_exponential', theta0=1e-2, thetaL=1e-4, thetaU=0.1, nugget=(dy / y) ** 2.0) X = np.atleast_2d(cal_spectra.wavelength[mask]).T # Fit to data using Maximum Likelihood Estimation of the parameters gp.fit(X, y) x = np.atleast_2d(cal_spectra.wavelength).T # Make the prediction on the meshed x-axis (ask for MSE as well) y_pred = gp.predict(x) cal_spectra.flux = y_pred else: fit=interfit(cal_spectra.wavelength[mask], cal_spectra.flux[mask], function=function, order=order, thresh=thresh, niter=niter) fit.interfit() cal_spectra.flux=fit(cal_spectra.wavelength) # write the spectra out st.writespectrum(cal_spectra, outfile, ftype='ascii')