def slotutcfix(images,update,outfile,ampperccd,ignorexp,droplimit,inter,plotdata,logfile,verbose,debug): with logging(logfile,debug) as log: # set up the variables utc_list = [] # is the input file specified? saltsafeio.filedefined('Input',images) # if the input file is a list, does it exist? if images[0] == '@': saltsafeio.listexists('Input',images) # parse list of input files and place them in order infiles=saltsafeio.listparse('Raw image',images,'','','') infiles.sort() # check input files exist saltsafeio.filesexist(infiles,'','r') # check to see if the output file exists and if so, clobber it if os.path.isfile(outfile): try: os.remove(outfile) except: raise SaltIOError('File ' + outfile + ' can not be removed') # open the outfile if outfile: try: fout=open(outfile,'w') except: raise SaltIOError('File ' + outfile + ' can not be opened') # get time of first exposure and basic information about the observations infile=infiles[0] struct=saltsafeio.openfits(infile) # check to make sure slotmode data detmode=saltsafekey.get('DETMODE',struct[0], infile) if detmode != 'Slot Mode': raise SaltIOError('Data are not Slot Mode Observations') # Check to see if SLOTUTCFIX has already been run # and print a warning if they have if saltsafekey.found('SLOTUTC', struct[0]): message='Data have already been processed by SLOTUTCFIX' log.warning(message) # check to make sure that it is the right version of the software scamver=saltsafekey.get('DETSWV', struct[0], infile) try: scamver=float(scamver.split('-')[-1]) if 4.42 <= scamver <= 5.00: pass else: raise SaltError('cannot currently correct this version of the SCAM software.') except: raise SaltError('Not able to read software version') # requested exposure time req_texp=saltsafekey.get('EXPTIME',struct[0],infile) # how many extensions? nextend=saltsafekey.get('NEXTEND',struct[0],infile) # how many amplifiers amplifiers=saltsafekey.get('NCCDS',struct[0],infile) amplifiers = int(ampperccd*float(amplifiers)) if ampperccd>0: nframes = nextend/amplifiers nstep=amplifiers else: nframes = nextend nstep=1 # how many total frame and unique times ntotal=nextend*len(infiles) nunique=len(infiles)*nframes-ignorexp+1 # Create arrays necessary for analysis id_arr=np.arange(nunique) utc_arr=np.zeros(nunique,dtype=float) # Read in each file and make a list of the UTC values if verbose: log.message('Reading in files to create list of UTC values.') j=0 for n,infile in enumerate(infiles): # Show progress if verbose: percent=100.*float(n)/float(len(infiles)) ctext='Percentage Complete: %.2f\r' % percent sys.stdout.write(ctext) sys.stdout.flush() struct=saltsafeio.openfits(infile) if not len(struct)-1==nextend: raise SaltIOError(infile,' has a different number of extensions from the first file') # Skip through the frames and read in the utc istart=1 if infile==infiles[0]: istart=ignorexp*amplifiers+1 for i in range(istart,len(struct), amplifiers): try: utc_list.append(saltsafekey.get('UTC-OBS', struct[i], infile)) utc_arr[j]=slottool.getobstime(struct[i], infile) j += 1 except Exception, e: raise SaltIOError('Unable to create array of UTC times. Please check the number of extensions in the files') # close FITS file saltsafeio.closefits(struct) # set up the other important arrays try: diff_arr=utc_arr.copy() diff_arr[1:]=diff_arr[1:]-utc_arr[:-1] diff_arr[0]=-1 dsec_arr=utc_arr-utc_arr.astype(int) except: raise SaltIOError('Unable to create timing arrays') # calculate the real exposure time if verbose: log.message('Calculating real exposure time.') real_expt, med_expt, t_start, t_arr, ysum_arr=calculate_realexptime(id_arr, utc_arr, dsec_arr, diff_arr, req_texp, utc_list) # plot the results if plotdata: if verbose: log.message('Plotting data.') plt.ion() plt.plot(t_arr,ysum_arr,linewidth=0.5,linestyle='-',marker='',color='b') plt.xlabel('Time (s)') plt.ylabel('Fit') # Calculate the corrrect values if verbose: log.message('Calculating correct values') i_start = abs(utc_arr-t_start).argmin() t_diff=utc_arr*0.0+real_expt nd=utc_arr*0.0 ndrop=0 for i in range(len(utc_arr)): if utc_arr[i] >= t_start: t_new=t_start+real_expt*(i-i_start+ndrop) t_diff[i]=utc_arr[i]-t_new while (t_diff[i]>real_expt and nd[i] < droplimit): nd[i]+= 1 t_new=t_start+real_expt*(i-i_start+ndrop+nd[i]) t_diff[i]=utc_arr[i]-t_new if (nd[i]<droplimit): ndrop += nd[i] else: t_new=t_start+real_expt*(i-i_start) t_diff[i]=utc_arr[i]-t_new while (t_diff[i]>real_expt and nd[i] < droplimit): nd[i]+= 1 t_new=t_start+real_expt*(i-i_start-nd[i]) t_diff[i]=utc_arr[i]-t_new # calculate the corrected timestamp by counting 6 record files forward and # 8 recored + unrecorded files back--or just 8*t_exp forward. # if the object is near the end of the run, then just replace it with # the correct value assuming no dropped exposures. # first make the array of new times new_arr=utc_arr-t_diff # Next loop through them to find the corrected time corr_arr=utc_arr*0.0 for i in range(len(new_arr)): if i+6 < len(new_arr)-1: corr_arr[i]=new_arr[i+6]-8*real_expt else: corr_arr[i]=new_arr[i]-2*real_expt t_diff=utc_arr-corr_arr # write out the first results msg="Dwell Time=%5.3f Requested Exposure Time=%5.3f Nobs = %i Dropped = %i" % (real_expt, req_texp, nunique, ndrop) if verbose: log.message(msg) if outfile: fout.write('#'+msg+'\n') fout.write('#%23s %2s %12s %12s %10s %8s %4s \n' % ('File', 'N', 'UTC_old', 'UTC_new', 'UTC_new(s)', 'Diff', 'drop' )) # Give the user a chance to update the value if inter: message='Update headers with a dwell time of %5.3f s [y/n]? ' % real_expt update=saltsafeio.yn_ask(message) if not update: message='Set Dwell Time manually [y/n]? ' update=saltsafeio.yn_ask(message) if update: message='New Dwell Time: ' real_expt=saltsafeio.ask(message) try: real_expt=float(real_expt) except Exception, e: msg='Could not set user dwell time because %s' % e raise SaltError(msg)
sys.stdout.flush() struct=saltsafeio.openupdatefits(infile) # Skip through the frames and read in the utc istart=1 if infile==infiles[0]: istart=ignorexp*amplifiers+1 for i in range(istart,len(struct), amplifiers): for k in range(0,amplifiers): if update: struct[i+k]=updateheaders(struct[i+k],i+k, t_diff[j], real_expt, utc_list[j], infile) if outfile: utc_new=saltsafekey.get('UTC-OBS', struct[i+k], infile) utc_new_sec=slottool.getobstime(struct[i], infile) fout.write('%25s %2i %12s %12s %7.3f %5.4f %4i \n' % (infile, i, utc_list[j], utc_new, utc_new_sec, t_diff[j], nd[j] )) j += 1 # Housekeeping key words if update: history = 'SALTUTCFIX -- ' history += 'images='+infile+' ' saltsafekey.housekeeping(struct[0],'SLOTUTC','UTC has been corrected',history,infile) # update fits file if update: saltsafeio.updatefits(struct) saltsafeio.closefits(struct) # close outfile
# Skip through the frames and read in the utc istart = 1 if infile == infiles[0]: istart = ignorexp * amplifiers + 1 for i in range(istart, len(struct), amplifiers): for k in range(0, amplifiers): if update: struct[i + k] = updateheaders( struct[i + k], i + k, t_diff[j], real_expt, utc_list[j], infile) if outfile: utc_new = saltsafekey.get('UTC-OBS', struct[i + k], infile) utc_new_sec = slottool.getobstime(struct[i], infile) fout.write('%25s %2i %12s %12s %7.3f %5.4f %4i \n' % (infile, i, utc_list[j], utc_new, utc_new_sec, t_diff[j], nd[j])) j += 1 # Housekeeping key words if update: history = 'SALTUTCFIX -- ' history += 'images=' + infile + ' ' saltsafekey.housekeeping(struct[0], 'SLOTUTC', 'UTC has been corrected', history, infile) # update fits file if update:
def slotutcfix(images, update, outfile, ampperccd, ignorexp, droplimit, inter, plotdata, logfile, verbose, debug): with logging(logfile, debug) as log: # set up the variables utc_list = [] # is the input file specified? saltsafeio.filedefined('Input', images) # if the input file is a list, does it exist? if images[0] == '@': saltsafeio.listexists('Input', images) # parse list of input files and place them in order infiles = saltsafeio.listparse('Raw image', images, '', '', '') infiles.sort() # check input files exist saltsafeio.filesexist(infiles, '', 'r') # check to see if the output file exists and if so, clobber it if os.path.isfile(outfile): try: os.remove(outfile) except: raise SaltIOError('File ' + outfile + ' can not be removed') # open the outfile if outfile: try: fout = open(outfile, 'w') except: raise SaltIOError('File ' + outfile + ' can not be opened') # get time of first exposure and basic information about the observations infile = infiles[0] struct = saltsafeio.openfits(infile) # check to make sure slotmode data detmode = saltsafekey.get('DETMODE', struct[0], infile) if detmode != 'Slot Mode': raise SaltIOError('Data are not Slot Mode Observations') # Check to see if SLOTUTCFIX has already been run # and print a warning if they have if saltsafekey.found('SLOTUTC', struct[0]): message = 'Data have already been processed by SLOTUTCFIX' log.warning(message) # check to make sure that it is the right version of the software scamver = saltsafekey.get('DETSWV', struct[0], infile) try: scamver = float(scamver.split('-')[-1]) if 4.42 <= scamver <= 5.00: pass else: raise SaltError( 'cannot currently correct this version of the SCAM software.' ) except: raise SaltError('Not able to read software version') # requested exposure time req_texp = saltsafekey.get('EXPTIME', struct[0], infile) # how many extensions? nextend = saltsafekey.get('NEXTEND', struct[0], infile) # how many amplifiers amplifiers = saltsafekey.get('NCCDS', struct[0], infile) amplifiers = int(ampperccd * float(amplifiers)) if ampperccd > 0: nframes = nextend / amplifiers nstep = amplifiers else: nframes = nextend nstep = 1 # how many total frame and unique times ntotal = nextend * len(infiles) nunique = len(infiles) * nframes - ignorexp + 1 # Create arrays necessary for analysis id_arr = np.arange(nunique) utc_arr = np.zeros(nunique, dtype=float) # Read in each file and make a list of the UTC values if verbose: log.message('Reading in files to create list of UTC values.') j = 0 for n, infile in enumerate(infiles): # Show progress if verbose: percent = 100. * float(n) / float(len(infiles)) ctext = 'Percentage Complete: %.2f\r' % percent sys.stdout.write(ctext) sys.stdout.flush() struct = saltsafeio.openfits(infile) if not len(struct) - 1 == nextend: raise SaltIOError( infile, ' has a different number of extensions from the first file' ) # Skip through the frames and read in the utc istart = 1 if infile == infiles[0]: istart = ignorexp * amplifiers + 1 for i in range(istart, len(struct), amplifiers): try: utc_list.append( saltsafekey.get('UTC-OBS', struct[i], infile)) utc_arr[j] = slottool.getobstime(struct[i], infile) j += 1 except Exception, e: raise SaltIOError( 'Unable to create array of UTC times. Please check the number of extensions in the files' ) # close FITS file saltsafeio.closefits(struct) # set up the other important arrays try: diff_arr = utc_arr.copy() diff_arr[1:] = diff_arr[1:] - utc_arr[:-1] diff_arr[0] = -1 dsec_arr = utc_arr - utc_arr.astype(int) except: raise SaltIOError('Unable to create timing arrays') # calculate the real exposure time if verbose: log.message('Calculating real exposure time.') real_expt, med_expt, t_start, t_arr, ysum_arr = calculate_realexptime( id_arr, utc_arr, dsec_arr, diff_arr, req_texp, utc_list) # plot the results if plotdata: if verbose: log.message('Plotting data.') plt.ion() plt.plot(t_arr, ysum_arr, linewidth=0.5, linestyle='-', marker='', color='b') plt.xlabel('Time (s)') plt.ylabel('Fit') # Calculate the corrrect values if verbose: log.message('Calculating correct values') i_start = abs(utc_arr - t_start).argmin() t_diff = utc_arr * 0.0 + real_expt nd = utc_arr * 0.0 ndrop = 0 for i in range(len(utc_arr)): if utc_arr[i] >= t_start: t_new = t_start + real_expt * (i - i_start + ndrop) t_diff[i] = utc_arr[i] - t_new while (t_diff[i] > real_expt and nd[i] < droplimit): nd[i] += 1 t_new = t_start + real_expt * (i - i_start + ndrop + nd[i]) t_diff[i] = utc_arr[i] - t_new if (nd[i] < droplimit): ndrop += nd[i] else: t_new = t_start + real_expt * (i - i_start) t_diff[i] = utc_arr[i] - t_new while (t_diff[i] > real_expt and nd[i] < droplimit): nd[i] += 1 t_new = t_start + real_expt * (i - i_start - nd[i]) t_diff[i] = utc_arr[i] - t_new # calculate the corrected timestamp by counting 6 record files forward and # 8 recored + unrecorded files back--or just 8*t_exp forward. # if the object is near the end of the run, then just replace it with # the correct value assuming no dropped exposures. # first make the array of new times new_arr = utc_arr - t_diff # Next loop through them to find the corrected time corr_arr = utc_arr * 0.0 for i in range(len(new_arr)): if i + 6 < len(new_arr) - 1: corr_arr[i] = new_arr[i + 6] - 8 * real_expt else: corr_arr[i] = new_arr[i] - 2 * real_expt t_diff = utc_arr - corr_arr # write out the first results msg = "Dwell Time=%5.3f Requested Exposure Time=%5.3f Nobs = %i Dropped = %i" % ( real_expt, req_texp, nunique, ndrop) if verbose: log.message(msg) if outfile: fout.write('#' + msg + '\n') fout.write('#%23s %2s %12s %12s %10s %8s %4s \n' % ('File', 'N', 'UTC_old', 'UTC_new', 'UTC_new(s)', 'Diff', 'drop')) # Give the user a chance to update the value if inter: message = 'Update headers with a dwell time of %5.3f s [y/n]? ' % real_expt update = saltsafeio.yn_ask(message) if not update: message = 'Set Dwell Time manually [y/n]? ' update = saltsafeio.yn_ask(message) if update: message = 'New Dwell Time: ' real_expt = saltsafeio.ask(message) try: real_expt = float(real_expt) except Exception, e: msg = 'Could not set user dwell time because %s' % e raise SaltError(msg)
def slotphot(images,outfile,srcfile,newfits=None,phottype='square', subbacktype='median',sigback=3,mbin=7,sorder=3,niter=5,sigdet=5, contpix=10,ampperccd=2,ignorexp=6,driftlimit=10.,finddrift=True, outtype='ascii',reltime=True,clobber=True,logfile='salt.log', verbose=True): """Perform photometry on listed SALT slotmode *images*.""" with logging(logfile,debug) as log: # set up the variables entries = [] vig_lo = {} vig_hi = {} amp = {} x = {} y = {} x_o = {} y_o = {} r = {} br1 = {} br2 = {} hour = 0 min = 0 sec = 0. time0 = 0. nframes = 0 bin=mbin order=sorder # is the input file specified? saltsafeio.filedefined('Input',images) # if the input file is a list, does it exist? if images[0] == '@': saltsafeio.listexists('Input',images) # parse list of input files infiles=saltsafeio.listparse('Raw image',images,'','','') # check input files exist saltsafeio.filesexist(infiles,'','r') # is the output file specified? saltsafeio.filedefined('Output',outfile) # check output file does not exist, optionally remove it if it does exist if os.path.exists(outfile) and clobber: os.remove(outfile) elif os.path.exists(outfile) and not clobber: raise SaltIOError('File '+outfile+' already exists, use clobber=y') # open output ascii file if outtype=='ascii': try: lc = open(outfile,'a') except: raise SaltIOError('Cannot open ouput file '+outfile) # is the extraction region defintion file specified? saltsafeio.filedefined('Extraction region defintion',srcfile) # check extraction region defintion file exists srcfile = srcfile.strip() saltsafeio.fileexists(srcfile) # read extraction region defintion file amp, x, y, x_o, y_o, r, br1, br2=slottool.readsrcfile(srcfile) # set the writenewfits parameter if not newfits or newfits=='none': writenewfits=False else: writenewfits=newfits # get time of first exposure and basic information about the observations infile=infiles[0] struct=saltsafeio.openfits(infile) # identify instrument instrume,keyprep,keygain,keybias,keyxtalk,keyslot=saltsafekey.instrumid(struct,infile) # how many extensions? nextend=saltsafekey.get('NEXTEND',struct[0],infile) if nextend < amp['comparison']: msg='Insufficient number of extensions in %s' % (infile) raise SaltIOError(msg) # how many amplifiers? amplifiers=saltsafekey.get('NCCDS',struct[0],infile) amplifiers = int(ampperccd*float(amplifiers)) if ampperccd>0: nframes = int(nextend/amplifiers) nstep=amplifiers else: nframes = nextend nstep=1 ntotal=nframes*len(infiles) # image size naxis1=saltsafekey.get('NAXIS1',struct[amp['comparison']],infile) naxis2=saltsafekey.get('NAXIS2',struct[amp['comparison']],infile) # CCD binning ccdsum=saltsafekey.get('CCDSUM',struct[0],infile) binx=int(ccdsum.split(' ')[0]) biny=int(ccdsum.split(' ')[1]) # Identify the time of the observations ext = 1 try: time0=slottool.getobstime(struct[ext], infile+'['+str(ext)+']') dateobs=saltsafekey.get('DATE-OBS',struct[ext],infile) dateobs=dateobs.replace('-','/') except: raise SaltIOError('No time or obsdate in first image') # If a total file is to be written out, create it and update it if writenewfits: if os.path.isfile(writenewfits): if clobber: saltsafeio.delete(writenewfits) else: raise SaltIOError('Newfits file exists, use clobber') try: hdu=pyfits.PrimaryHDU() hdu.header=struct[0].header hdu.header['NCCDS']=1 hdu.header['NSCIEXT']=ntotal-ignorexp hdu.header['NEXTEND']=ntotal-ignorexp hduList=pyfits.HDUList(hdu) hduList.verify() hduList.writeto(writenewfits) except: raise SaltIOError('Could not create newfits file, '+writenewfits) # Close image file saltsafeio.closefits(struct) # Read newfits file back in for updating if writenewfits: try: hduList=pyfits.open(writenewfits,mode='update') except: raise SaltIOError('Cannot open newfits file '+writenewfits+' for updating.') # set up the arrays j=0 time=np.zeros(ntotal ,dtype='float')-1.0 dx=np.zeros(ntotal ,dtype='float') dy=np.zeros(ntotal ,dtype='float') tflux=np.zeros(ntotal ,dtype='float') terr =np.zeros(ntotal ,dtype='float') cflux=np.zeros(ntotal ,dtype='float') cerr =np.zeros(ntotal ,dtype='float') ratio=np.zeros(ntotal ,dtype='float') rerr =np.zeros(ntotal ,dtype='float') tgt_x=np.zeros(ntotal ,dtype='float') tgt_y=np.zeros(ntotal ,dtype='float') cmp_x=np.zeros(ntotal ,dtype='float') cmp_y=np.zeros(ntotal ,dtype='float') p_one=100./ntotal # One percent p_old=-1 # Previous completed percentage p_new=0 # New completed percentage p_n=1 # Counter number for infile in infiles: # Log if verbose: log.message('Starting photometry on file '+infile, with_stdout=False) struct=pyfits.open(infile) # Skip through the frames and process each frame individually for i in range(nframes): # Show progress if verbose: p_new=int(p_n*p_one) p_n+=1 if p_new!=p_old: ctext='Percentage Complete: %d\r' % p_new sys.stdout.write(ctext) sys.stdout.flush() p_old=p_new if not (infile==infiles[0] and i < ignorexp): ext=amp['comparison']+i*nstep try: header=struct[ext].header array=struct[ext].data array=array*1.0 except: msg='Unable to open extension %i in image %s' % (ext, infile) raise SaltIOError(msg) # starti the analysis of each frame # get the time time[j]=slottool.getobstime(struct[ext],infile+'['+str(ext)+']') # gain and readout noise try: gain=float(header['GAIN']) except: gain=1 raise SaltIOError('Gain not specified in image header') try: rdnoise=float(header['RDNOISE']) except: rdnoise=0 raise SaltIOError('RDNOISE not specified in image header') # background subtraction if not subbacktype=='none': try: array=subbackground(array, sigback, bin, order, niter, subbacktype) except SaltError: log.warning('Image '+infile+' extention '+str(ext)+' is blank, skipping') continue # x-y fit to the comparison star and update the x,y values if finddrift: carray, fx,fy=slottool.finddrift(array, x['comparison'], y['comparison'], r['comparison'], naxis1, naxis2, sigdet, contpix, sigback, driftlimit, niter) if fx > -1 and fy > -1: if fx < naxis1 and fy < naxis2: dx[j]=x['comparison']-fx dy[j]=y['comparison']-fy x['comparison']=fx y['comparison']=fy x['target']=x['target']-dx[j] y['target']=y['target']-dy[j] else: dx[j]=0 dy[j]=0 x['comparison']=x_o['comparison'] y['comparison']=y_o['comparison'] x['target']=x_o['target'] y['target']=y_o['target'] else: msg='No comparison object found in image file ' + infile+' on extension %i skipping.' % ext log.warning(msg) pass # do photometry try: tflux[j],terr[j],cflux[j],cerr[j],ratio[j],rerr[j]=slottool.dophot(phottype, array, x, y, r, br1, br2, gain, rdnoise, naxis1, naxis2) except SaltError, e: msg='Could not do photometry on extension %i in image %s because %s skipping.' % (ext, infile, e) log.warning(msg) tgt_x[j]=x['target'] tgt_y[j]=y['target'] cmp_x[j]=x['comparison'] cmp_y[j]=y['comparison'] # record results # TODO! This should be removed in favor of the write all to disk in the end if outtype=='ascii': slottool.writedataout(lc, j+1, time[j], x, y, tflux[j], terr[j], cflux[j],cerr[j],ratio[j],rerr[j],time0, reltime) # write newfits file if writenewfits: # add original name and extension number to header try: hdue=pyfits.ImageHDU(array) hdue.header=header hdue.header.update('ONAME',infile,'Original image name') hdue.header.update('OEXT',ext,'Original extension number') hduList.append(hdue) except: log.warning('Could not update image in newfits '+infile+' '+str(ext)) # increment counter j+=1 # close FITS file saltsafeio.closefits(struct) # close newfits file if writenewfits: try: hduList.flush() hduList.close() except: raise SaltIOError('Cannot close newfits file.') # write to output if outtype=='ascii': # close output ascii file try: lc.close() except: raise SaltIOError('Cannot close ouput file ' + outfile) elif outtype=='fits': print 'writing fits' try: c1=pyfits.Column(name='index',format='D',array=np.arange(ntotal)) if reltime: c2=pyfits.Column(name='time',format='D',array=time-time0) else: c2=pyfits.Column(name='time',format='D',array=time) c3=pyfits.Column(name='tgt_x',format='D',array=tgt_x) c4=pyfits.Column(name='tgt_y',format='D',array=tgt_y) c5=pyfits.Column(name='tgt_flux',format='D',array=tflux) c6=pyfits.Column(name='tgt_err',format='D',array=terr) c7=pyfits.Column(name='cmp_x',format='D',array=cmp_x) c8=pyfits.Column(name='cmp_y',format='D',array=cmp_y) c9=pyfits.Column(name='cmp_flux',format='D',array=cflux) c10=pyfits.Column(name='cmp_err',format='D',array=cerr) c11=pyfits.Column(name='flux_ratio',format='D',array=ratio) c12=pyfits.Column(name='flux_ratio_err',format='D',array=rerr) tbhdu=pyfits.new_table([c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12]) # Add header information tbhdu.header.update('RELTIME',str(reltime),'Time relative to first datapoint or absolute.') tbhdu.writeto(outfile) print 'fits written to ',outfile except: raise SaltIOError('Could not write to fits table.')