Esempio n. 1
0
       table='FileData join ProposalCode using (ProposalCode_Id)'
       logic="FileName like '%" + obsdate + "%'"
       records=saltmysql.select(sdb, select, table, logic)

       if len(records)<1:  
          msg="No observations available for %s" % obsdate
          log.warning(msg)
          return
       else:
          for p in records:
              propids.append(p[0])

       #clean the proposal list
       print pids, propids
       try:
           pids=saltio.cleanpropcode(pids, propids)
           pids=saltio.removebadpids(pids)
           pids=saltio.removeengineeringpids(pids)
       except SaltIOError:
           msg="No notifications necessary for %s" % obsdate
           log.warning
           return

       #loop through each of the pids and send the email
       for pid in pids:
         propinfo=findpropinfo(pid, sdb)
         if propinfo:
           for pi, email in zip(propinfo[pid][0], propinfo[pid][1]):
               letter=saltio.openascii(readme,'r')
               msg=letter.read()
               msg=msg.replace('yourname',pi)
Esempio n. 2
0
def saltobsid(propcode,obslog,rawpath,prodpath,outpath,prefix='mbxgp', fprefix='bxgp',clobber=False,logfile='salt.log',verbose=True):
   """Split data into their different data directories
   """


   with logging(logfile,debug) as log:

       # are the arguments defined
       pids = saltio.argunpack('propcode',propcode)

       # check observation log file exists
       obslog = obslog.strip()
       saltio.fileexists(obslog)

       #open the observing log
       obstruct = saltio.openfits(obslog)
       obstab = saltio.readtab(obstruct[1],obslog)
       saltio.closefits(obstruct)

       #read in the file information
       filenames = saltstring.listfunc(obstab.field('filename'),'lstrip')
       instrumes = saltstring.listfunc(obstab.field('instrume'),'lstrip')
       proposers = saltstring.listfunc(obstab.field('proposer'),'clean')
       propids = saltstring.listfunc(obstab.field('propid'),'clean')
       ccdtypes = saltstring.listfunc(obstab.field('ccdtype'),'clean')
       ccdsums = saltstring.listfunc(obstab.field('ccdsum'),'clean')
       gainsets = saltstring.listfunc(obstab.field('gainset'),'clean')
       rospeeds = saltstring.listfunc(obstab.field('rospeed'),'clean')
       detmodes = saltstring.listfunc(obstab.field('detmode'),'clean')
       filters = saltstring.listfunc(obstab.field('filter'),'clean')
       gratings = saltstring.listfunc(obstab.field('grating'),'clean')
       gr_angles = obstab.field('gr-angle')
       ar_angles = obstab.field('ar-angle')
 
       # Create the list of proposals
       try:
           pids=saltio.cleanpropcode(pids, propids)
       except SaltIOError:
           #throw a warning adn exit if not data needs to be filterd
           log.warning('No data to filter\n', with_stdout=verbose)
           return

       # check paths exist, end with a "/" and convert them to absolute paths
       rawpath = saltio.abspath(rawpath)
       prodpath = saltio.abspath(prodpath)
       outpath = saltio.abspath(outpath)

       #create the symlink raw path
       rawsplit=rawpath.strip().split('/')
       symrawpath='../../%s/%s/' % (rawsplit[-3], rawsplit[-2])
       prodsplit=prodpath.strip().split('/')
       symprodpath='../../%s/%s/' % (prodsplit[-3], prodsplit[-2])
  

       # create PI directories
       for pid in pids:
           saltio.createdir(outpath+pid)
           saltio.createdir(outpath+pid+'/raw')
           saltio.createdir(outpath+pid+'/product')

       #copy the data that belongs to a pid into that directory
       log.message('SALTOBSID -- filtering images to proposal directories\n', with_stdout=verbose)

       #copy data for a given proposal to the raw and produce directories
       for i in range(len(obstab)):
           if os.path.exists(outpath+obstab[i]['propid']):
             if obstab[i]['object'].upper() not in ['ZERO', 'BIAS']:
               fname=obstab[i]['filename']
               pdir=obstab[i]['propid']
               detmode=obstab[i]['detmode']
               linkfiles(fname, pdir,detmode, symrawpath, symprodpath, outpath, prefix, fprefix, clobber)
               message='Copying %s to %s' % (fname, pdir)
               log.message(message, with_header=False, with_stdout=verbose)

       #look through the bias/flat/arc/standard data to see if there is any relavent data
       log.message('SALTOBSID -- filtering calibration files to proposal directories\n', with_stdout=verbose)

       caldata=['ZERO', 'FLAT', 'ARC']
       biasheader_list=['DETMODE', 'CCDSUM', 'GAINSET', 'ROSPEED']
       flatheader_list=['DETMODE', 'CCDSUM', 'GAINSET', 'ROSPEED', 'FILTER', 'GRATING', 'GR-ANGLE', 'AR-ANGLE']
       archeader_list=['OBSMODE', 'DETMODE', 'CCDSUM', 'GAINSET', 'ROSPEED', 'FILTER', 'GRATING', 'GR-ANGLE', 'AR-ANGLE']
       
       calproplist=['CAL_SPST']
       #Include bias frames
       log.message('SALTOBSID -- filtering bias files to proposal directories\n', with_stdout=verbose)

       for i in range(len(obstab)):
           fname=obstab[i]['filename']
           prop_list=[]
           #if it is a zero, check to see what other data have the same settings 
           if obstab[i]['CCDTYPE'].strip().upper()=='ZERO' or obstab[i]['OBJECT'].strip().upper() in ['BIAS', 'ZERO']:
               for j in range(len(obstab)):
                   if comparefiles(obstab[i], obstab[j], biasheader_list):
                       prop_list.append(obstab[i]['PROPID'])

           prop_list=saltio.removebadpids(set(prop_list))
           for pdir in prop_list:
                   detmode=obstab[i]['detmode']
                   linkfiles(fname, pdir, detmode, symrawpath, symprodpath, outpath,  fprefix, fprefix, clobber)
                   message='Copying %s to %s' % (fname, pdir)
                   log.message(message, with_header=False, with_stdout=verbose)

       #Include calibration  frames
       log.message('SALTOBSID -- filtering  calibration files to proposal directories\n', with_stdout=verbose)
 
       for i in range(len(obstab)):
           fname=obstab[i]['filename']
           prop_list=[]

           #if it is a flat, check to see what other data have the same settings 
           #this is turned off
           if obstab[i]['CCDTYPE'].strip().upper()=='FLAT' and False:
               for j in range(len(obstab)):
                   if comparefiles(obstab[i], obstab[j],  flatheader_list):
                       prop_list.append(obstab[j]['PROPID'])

           #if it is a arc, check to see what other data have the same settings 
           #this is turned off
           if obstab[i]['CCDTYPE'].strip().upper()=='ARC' and False:
               for j in range(len(obstab)):
                   if comparefiles(obstab[i], obstab[j],  archeader_list):
                       prop_list.append(obstab[j]['PROPID'])


           #if it is a calibration standard, see what other data have the same settings
           if obstab[i]['PROPID'].strip().upper() in calproplist:
               for j in range(len(obstab)):
                   if comparefiles(obstab[i], obstab[j],  flatheader_list):
                       prop_list.append(obstab[j]['PROPID'])


           prop_list=saltio.removebadpids(set(prop_list))
           for pdir in prop_list:
               if pdir!=obstab[i]['propid']:
                   detmode=obstab[i]['detmode']
                   linkfiles(fname, pdir, detmode, symrawpath, symprodpath, outpath,  prefix, fprefix, clobber)
                   message='Copying %s to %s' % (fname, pdir)
                   log.message(message, with_header=False, with_stdout=verbose)

       #Include master (bias or flat) frames
       log.message('SALTOBSID -- filtering master calibration files to proposal directories\n', with_stdout=verbose)
       masterlist=glob.glob(prodpath+'*Bias*')+glob.glob(prodpath+'*Flat*')
       for bimg in masterlist:
           struct=pyfits.open(bimg)
           bdict={}
           prop_list=[]
           for k in biasheader_list:
               bdict[k]=saltkey.get(k, struct[0])
           for i in range(len(obstab)):
               if comparefiles(obstab[i], bdict,  biasheader_list):
                   prop_list.append(obstab[i]['PROPID'])
           struct.close()

           #copy the files over to the directory
           prop_list=saltio.removebadpids(set(prop_list))
           for pdir in prop_list:
               fname=os.path.basename(bimg)
               infile = symprodpath+fname
               link = outpath+pdir+'/product/'+fname
               saltio.symlink(infile,link,clobber)
               message='Copying %s to %s' % (fname ,pdir)
               log.message(message, with_header=False, with_stdout=verbose)