table='FileData join ProposalCode using (ProposalCode_Id)' logic="FileName like '%" + obsdate + "%'" records=saltmysql.select(sdb, select, table, logic) if len(records)<1: msg="No observations available for %s" % obsdate log.warning(msg) return else: for p in records: propids.append(p[0]) #clean the proposal list print pids, propids try: pids=saltio.cleanpropcode(pids, propids) pids=saltio.removebadpids(pids) pids=saltio.removeengineeringpids(pids) except SaltIOError: msg="No notifications necessary for %s" % obsdate log.warning return #loop through each of the pids and send the email for pid in pids: propinfo=findpropinfo(pid, sdb) if propinfo: for pi, email in zip(propinfo[pid][0], propinfo[pid][1]): letter=saltio.openascii(readme,'r') msg=letter.read() msg=msg.replace('yourname',pi)
def saltobsid(propcode,obslog,rawpath,prodpath,outpath,prefix='mbxgp', fprefix='bxgp',clobber=False,logfile='salt.log',verbose=True): """Split data into their different data directories """ with logging(logfile,debug) as log: # are the arguments defined pids = saltio.argunpack('propcode',propcode) # check observation log file exists obslog = obslog.strip() saltio.fileexists(obslog) #open the observing log obstruct = saltio.openfits(obslog) obstab = saltio.readtab(obstruct[1],obslog) saltio.closefits(obstruct) #read in the file information filenames = saltstring.listfunc(obstab.field('filename'),'lstrip') instrumes = saltstring.listfunc(obstab.field('instrume'),'lstrip') proposers = saltstring.listfunc(obstab.field('proposer'),'clean') propids = saltstring.listfunc(obstab.field('propid'),'clean') ccdtypes = saltstring.listfunc(obstab.field('ccdtype'),'clean') ccdsums = saltstring.listfunc(obstab.field('ccdsum'),'clean') gainsets = saltstring.listfunc(obstab.field('gainset'),'clean') rospeeds = saltstring.listfunc(obstab.field('rospeed'),'clean') detmodes = saltstring.listfunc(obstab.field('detmode'),'clean') filters = saltstring.listfunc(obstab.field('filter'),'clean') gratings = saltstring.listfunc(obstab.field('grating'),'clean') gr_angles = obstab.field('gr-angle') ar_angles = obstab.field('ar-angle') # Create the list of proposals try: pids=saltio.cleanpropcode(pids, propids) except SaltIOError: #throw a warning adn exit if not data needs to be filterd log.warning('No data to filter\n', with_stdout=verbose) return # check paths exist, end with a "/" and convert them to absolute paths rawpath = saltio.abspath(rawpath) prodpath = saltio.abspath(prodpath) outpath = saltio.abspath(outpath) #create the symlink raw path rawsplit=rawpath.strip().split('/') symrawpath='../../%s/%s/' % (rawsplit[-3], rawsplit[-2]) prodsplit=prodpath.strip().split('/') symprodpath='../../%s/%s/' % (prodsplit[-3], prodsplit[-2]) # create PI directories for pid in pids: saltio.createdir(outpath+pid) saltio.createdir(outpath+pid+'/raw') saltio.createdir(outpath+pid+'/product') #copy the data that belongs to a pid into that directory log.message('SALTOBSID -- filtering images to proposal directories\n', with_stdout=verbose) #copy data for a given proposal to the raw and produce directories for i in range(len(obstab)): if os.path.exists(outpath+obstab[i]['propid']): if obstab[i]['object'].upper() not in ['ZERO', 'BIAS']: fname=obstab[i]['filename'] pdir=obstab[i]['propid'] detmode=obstab[i]['detmode'] linkfiles(fname, pdir,detmode, symrawpath, symprodpath, outpath, prefix, fprefix, clobber) message='Copying %s to %s' % (fname, pdir) log.message(message, with_header=False, with_stdout=verbose) #look through the bias/flat/arc/standard data to see if there is any relavent data log.message('SALTOBSID -- filtering calibration files to proposal directories\n', with_stdout=verbose) caldata=['ZERO', 'FLAT', 'ARC'] biasheader_list=['DETMODE', 'CCDSUM', 'GAINSET', 'ROSPEED'] flatheader_list=['DETMODE', 'CCDSUM', 'GAINSET', 'ROSPEED', 'FILTER', 'GRATING', 'GR-ANGLE', 'AR-ANGLE'] archeader_list=['OBSMODE', 'DETMODE', 'CCDSUM', 'GAINSET', 'ROSPEED', 'FILTER', 'GRATING', 'GR-ANGLE', 'AR-ANGLE'] calproplist=['CAL_SPST'] #Include bias frames log.message('SALTOBSID -- filtering bias files to proposal directories\n', with_stdout=verbose) for i in range(len(obstab)): fname=obstab[i]['filename'] prop_list=[] #if it is a zero, check to see what other data have the same settings if obstab[i]['CCDTYPE'].strip().upper()=='ZERO' or obstab[i]['OBJECT'].strip().upper() in ['BIAS', 'ZERO']: for j in range(len(obstab)): if comparefiles(obstab[i], obstab[j], biasheader_list): prop_list.append(obstab[i]['PROPID']) prop_list=saltio.removebadpids(set(prop_list)) for pdir in prop_list: detmode=obstab[i]['detmode'] linkfiles(fname, pdir, detmode, symrawpath, symprodpath, outpath, fprefix, fprefix, clobber) message='Copying %s to %s' % (fname, pdir) log.message(message, with_header=False, with_stdout=verbose) #Include calibration frames log.message('SALTOBSID -- filtering calibration files to proposal directories\n', with_stdout=verbose) for i in range(len(obstab)): fname=obstab[i]['filename'] prop_list=[] #if it is a flat, check to see what other data have the same settings #this is turned off if obstab[i]['CCDTYPE'].strip().upper()=='FLAT' and False: for j in range(len(obstab)): if comparefiles(obstab[i], obstab[j], flatheader_list): prop_list.append(obstab[j]['PROPID']) #if it is a arc, check to see what other data have the same settings #this is turned off if obstab[i]['CCDTYPE'].strip().upper()=='ARC' and False: for j in range(len(obstab)): if comparefiles(obstab[i], obstab[j], archeader_list): prop_list.append(obstab[j]['PROPID']) #if it is a calibration standard, see what other data have the same settings if obstab[i]['PROPID'].strip().upper() in calproplist: for j in range(len(obstab)): if comparefiles(obstab[i], obstab[j], flatheader_list): prop_list.append(obstab[j]['PROPID']) prop_list=saltio.removebadpids(set(prop_list)) for pdir in prop_list: if pdir!=obstab[i]['propid']: detmode=obstab[i]['detmode'] linkfiles(fname, pdir, detmode, symrawpath, symprodpath, outpath, prefix, fprefix, clobber) message='Copying %s to %s' % (fname, pdir) log.message(message, with_header=False, with_stdout=verbose) #Include master (bias or flat) frames log.message('SALTOBSID -- filtering master calibration files to proposal directories\n', with_stdout=verbose) masterlist=glob.glob(prodpath+'*Bias*')+glob.glob(prodpath+'*Flat*') for bimg in masterlist: struct=pyfits.open(bimg) bdict={} prop_list=[] for k in biasheader_list: bdict[k]=saltkey.get(k, struct[0]) for i in range(len(obstab)): if comparefiles(obstab[i], bdict, biasheader_list): prop_list.append(obstab[i]['PROPID']) struct.close() #copy the files over to the directory prop_list=saltio.removebadpids(set(prop_list)) for pdir in prop_list: fname=os.path.basename(bimg) infile = symprodpath+fname link = outpath+pdir+'/product/'+fname saltio.symlink(infile,link,clobber) message='Copying %s to %s' % (fname ,pdir) log.message(message, with_header=False, with_stdout=verbose)