def linkfiles(fname, pdir, detmode, rawpath, prodpath, outpath, prefix='mbxgp', fprefix='bxgp', clobber=False): #copy the raw data infile=rawpath+fname link=outpath+pdir+'/raw/'+fname saltio.symlink(infile,link,clobber) #copy the product data if not fastmode(detmode): pfname=prefix+fname else: pfname=fprefix+fname infile = prodpath+pfname link = outpath+pdir+'/product/'+pfname if fname[0] in ['S', 'P', 'H', 'R']: saltio.symlink(infile,link,clobber)
def saltobsid(propcode,obslog,rawpath,prodpath,outpath,prefix='mbxgp', fprefix='bxgp',clobber=False,logfile='salt.log',verbose=True): """Split data into their different data directories """ with logging(logfile,debug) as log: # are the arguments defined pids = saltio.argunpack('propcode',propcode) # check observation log file exists obslog = obslog.strip() saltio.fileexists(obslog) #open the observing log obstruct = saltio.openfits(obslog) obstab = saltio.readtab(obstruct[1],obslog) saltio.closefits(obstruct) #read in the file information filenames = saltstring.listfunc(obstab.field('filename'),'lstrip') instrumes = saltstring.listfunc(obstab.field('instrume'),'lstrip') proposers = saltstring.listfunc(obstab.field('proposer'),'clean') propids = saltstring.listfunc(obstab.field('propid'),'clean') ccdtypes = saltstring.listfunc(obstab.field('ccdtype'),'clean') ccdsums = saltstring.listfunc(obstab.field('ccdsum'),'clean') gainsets = saltstring.listfunc(obstab.field('gainset'),'clean') rospeeds = saltstring.listfunc(obstab.field('rospeed'),'clean') detmodes = saltstring.listfunc(obstab.field('detmode'),'clean') filters = saltstring.listfunc(obstab.field('filter'),'clean') gratings = saltstring.listfunc(obstab.field('grating'),'clean') gr_angles = obstab.field('gr-angle') ar_angles = obstab.field('ar-angle') # Create the list of proposals try: pids=saltio.cleanpropcode(pids, propids) except SaltIOError: #throw a warning adn exit if not data needs to be filterd log.warning('No data to filter\n', with_stdout=verbose) return # check paths exist, end with a "/" and convert them to absolute paths rawpath = saltio.abspath(rawpath) prodpath = saltio.abspath(prodpath) outpath = saltio.abspath(outpath) #create the symlink raw path rawsplit=rawpath.strip().split('/') symrawpath='../../%s/%s/' % (rawsplit[-3], rawsplit[-2]) prodsplit=prodpath.strip().split('/') symprodpath='../../%s/%s/' % (prodsplit[-3], prodsplit[-2]) # create PI directories for pid in pids: saltio.createdir(outpath+pid) saltio.createdir(outpath+pid+'/raw') saltio.createdir(outpath+pid+'/product') #copy the data that belongs to a pid into that directory log.message('SALTOBSID -- filtering images to proposal directories\n', with_stdout=verbose) #copy data for a given proposal to the raw and produce directories for i in range(len(obstab)): if os.path.exists(outpath+obstab[i]['propid']): if obstab[i]['object'].upper() not in ['ZERO', 'BIAS']: fname=obstab[i]['filename'] pdir=obstab[i]['propid'] detmode=obstab[i]['detmode'] linkfiles(fname, pdir,detmode, symrawpath, symprodpath, outpath, prefix, fprefix, clobber) message='Copying %s to %s' % (fname, pdir) log.message(message, with_header=False, with_stdout=verbose) #look through the bias/flat/arc/standard data to see if there is any relavent data log.message('SALTOBSID -- filtering calibration files to proposal directories\n', with_stdout=verbose) caldata=['ZERO', 'FLAT', 'ARC'] biasheader_list=['DETMODE', 'CCDSUM', 'GAINSET', 'ROSPEED'] flatheader_list=['DETMODE', 'CCDSUM', 'GAINSET', 'ROSPEED', 'FILTER', 'GRATING', 'GR-ANGLE', 'AR-ANGLE'] archeader_list=['OBSMODE', 'DETMODE', 'CCDSUM', 'GAINSET', 'ROSPEED', 'FILTER', 'GRATING', 'GR-ANGLE', 'AR-ANGLE'] calproplist=['CAL_SPST'] #Include bias frames log.message('SALTOBSID -- filtering bias files to proposal directories\n', with_stdout=verbose) for i in range(len(obstab)): fname=obstab[i]['filename'] prop_list=[] #if it is a zero, check to see what other data have the same settings if obstab[i]['CCDTYPE'].strip().upper()=='ZERO' or obstab[i]['OBJECT'].strip().upper() in ['BIAS', 'ZERO']: for j in range(len(obstab)): if comparefiles(obstab[i], obstab[j], biasheader_list): prop_list.append(obstab[i]['PROPID']) prop_list=saltio.removebadpids(set(prop_list)) for pdir in prop_list: detmode=obstab[i]['detmode'] linkfiles(fname, pdir, detmode, symrawpath, symprodpath, outpath, fprefix, fprefix, clobber) message='Copying %s to %s' % (fname, pdir) log.message(message, with_header=False, with_stdout=verbose) #Include calibration frames log.message('SALTOBSID -- filtering calibration files to proposal directories\n', with_stdout=verbose) for i in range(len(obstab)): fname=obstab[i]['filename'] prop_list=[] #if it is a flat, check to see what other data have the same settings #this is turned off if obstab[i]['CCDTYPE'].strip().upper()=='FLAT' and False: for j in range(len(obstab)): if comparefiles(obstab[i], obstab[j], flatheader_list): prop_list.append(obstab[j]['PROPID']) #if it is a arc, check to see what other data have the same settings #this is turned off if obstab[i]['CCDTYPE'].strip().upper()=='ARC' and False: for j in range(len(obstab)): if comparefiles(obstab[i], obstab[j], archeader_list): prop_list.append(obstab[j]['PROPID']) #if it is a calibration standard, see what other data have the same settings if obstab[i]['PROPID'].strip().upper() in calproplist: for j in range(len(obstab)): if comparefiles(obstab[i], obstab[j], flatheader_list): prop_list.append(obstab[j]['PROPID']) prop_list=saltio.removebadpids(set(prop_list)) for pdir in prop_list: if pdir!=obstab[i]['propid']: detmode=obstab[i]['detmode'] linkfiles(fname, pdir, detmode, symrawpath, symprodpath, outpath, prefix, fprefix, clobber) message='Copying %s to %s' % (fname, pdir) log.message(message, with_header=False, with_stdout=verbose) #Include master (bias or flat) frames log.message('SALTOBSID -- filtering master calibration files to proposal directories\n', with_stdout=verbose) masterlist=glob.glob(prodpath+'*Bias*')+glob.glob(prodpath+'*Flat*') for bimg in masterlist: struct=pyfits.open(bimg) bdict={} prop_list=[] for k in biasheader_list: bdict[k]=saltkey.get(k, struct[0]) for i in range(len(obstab)): if comparefiles(obstab[i], bdict, biasheader_list): prop_list.append(obstab[i]['PROPID']) struct.close() #copy the files over to the directory prop_list=saltio.removebadpids(set(prop_list)) for pdir in prop_list: fname=os.path.basename(bimg) infile = symprodpath+fname link = outpath+pdir+'/product/'+fname saltio.symlink(infile,link,clobber) message='Copying %s to %s' % (fname ,pdir) log.message(message, with_header=False, with_stdout=verbose)
def hrscalibrations( obsdate, sdbhost="sdb.saao", sdbname="sdb", sdbuser="", password="", clobber=False, logfile="saltlog.log", verbose=True, ): """Seach the salt database for FITS files """ with logging(logfile, debug) as log: # make obsdate a string if needed obsdate = str(obsdate) log.message("Sorting HRS calibration data") # connect to the database sdb = saltmysql.connectdb(sdbhost, sdbname, sdbuser, password) # first select propcodes for all HRS data from that day and check for any CAL data table = "FileData join ProposalCode using (ProposalCode_Id)" logic = "FileName like 'H" + obsdate + "%' or FileName like 'R" + obsdate + "%'" records = saltmysql.select(sdb, "FileName, Proposal_Code", table, logic) # exit if no data was taken for HRS if len(records) == 0: return # Loop through each of the files and create the directories if needed image_dict = {} for filename, propid in records: if propid.count("CAL"): # check for directory and create structure caldir = "/salt/HRS_Cals/%s/" % propid if not os.path.isdir(caldir): os.mkdir(caldir) yeardir = "%s%s/" % (caldir, obsdate[0:4]) if not os.path.isdir(yeardir): os.mkdir(yeardir) daydir = "%s%s/%s/" % (caldir, obsdate[0:4], obsdate[4:8]) if not os.path.isdir(daydir): os.mkdir(daydir) rawdir = "%s%s/%s/raw/" % (caldir, obsdate[0:4], obsdate[4:8]) if not os.path.isdir(rawdir): os.mkdir(rawdir) prodir = "%s%s/%s/product/" % (caldir, obsdate[0:4], obsdate[4:8]) if not os.path.isdir(prodir): os.mkdir(prodir) # create the symlinks to the files infile = "/salt/data/%s/%s/hrs/raw/%s" % (obsdate[0:4], obsdate[4:8], filename) link = "%s%s" % (rawdir, filename) saltio.symlink(infile, link, clobber) infile = "/salt/data/%s/%s/hrs/product/mbgph%s" % (obsdate[0:4], obsdate[4:8], filename) link = "%smbgph%s" % (prodir, filename) saltio.symlink(infile, link, clobber) log.message( "Copied %s to the HRS_CAL/%s directory" % (filename, propid), with_header=False, with_stdout=verbose ) # create log of files image_info = get_image_info(sdb, filename) try: image_dict[propid].append([image_info]) except: image_dict[propid] = [[image_info]] # create log of each file--currently not enough information # in database to do this nightlog = "/salt/logs/sanightlogs/%s.log" % obsdate for k in image_dict: nightlink = "/salt/HRS_Cals/%s/%s/%s/%s.log" % (k, obsdate[0:4], obsdate[4:8], obsdate) saltio.symlink(nightlog, nightlink, clobber) fout = open("/salt/HRS_Cals/%s/%s/%s/%s.log" % (k, obsdate[0:4], obsdate[4:8], k), "w") for imlist in image_dict[k]: for info in imlist: for f in info: fout.write("%20s " % str(f)) fout.write("\n") fout.close()