Example #1
0
def linkfiles(fname, pdir, detmode, rawpath, prodpath, outpath, prefix='mbxgp', fprefix='bxgp', clobber=False):

   #copy the raw data
   infile=rawpath+fname
   link=outpath+pdir+'/raw/'+fname
   saltio.symlink(infile,link,clobber)

   #copy the product data
   if not fastmode(detmode):
       pfname=prefix+fname
   else:
       pfname=fprefix+fname
   infile = prodpath+pfname
   link = outpath+pdir+'/product/'+pfname
   if fname[0] in ['S', 'P', 'H', 'R']: 
      saltio.symlink(infile,link,clobber)
Example #2
0
def saltobsid(propcode,obslog,rawpath,prodpath,outpath,prefix='mbxgp', fprefix='bxgp',clobber=False,logfile='salt.log',verbose=True):
   """Split data into their different data directories
   """


   with logging(logfile,debug) as log:

       # are the arguments defined
       pids = saltio.argunpack('propcode',propcode)

       # check observation log file exists
       obslog = obslog.strip()
       saltio.fileexists(obslog)

       #open the observing log
       obstruct = saltio.openfits(obslog)
       obstab = saltio.readtab(obstruct[1],obslog)
       saltio.closefits(obstruct)

       #read in the file information
       filenames = saltstring.listfunc(obstab.field('filename'),'lstrip')
       instrumes = saltstring.listfunc(obstab.field('instrume'),'lstrip')
       proposers = saltstring.listfunc(obstab.field('proposer'),'clean')
       propids = saltstring.listfunc(obstab.field('propid'),'clean')
       ccdtypes = saltstring.listfunc(obstab.field('ccdtype'),'clean')
       ccdsums = saltstring.listfunc(obstab.field('ccdsum'),'clean')
       gainsets = saltstring.listfunc(obstab.field('gainset'),'clean')
       rospeeds = saltstring.listfunc(obstab.field('rospeed'),'clean')
       detmodes = saltstring.listfunc(obstab.field('detmode'),'clean')
       filters = saltstring.listfunc(obstab.field('filter'),'clean')
       gratings = saltstring.listfunc(obstab.field('grating'),'clean')
       gr_angles = obstab.field('gr-angle')
       ar_angles = obstab.field('ar-angle')
 
       # Create the list of proposals
       try:
           pids=saltio.cleanpropcode(pids, propids)
       except SaltIOError:
           #throw a warning adn exit if not data needs to be filterd
           log.warning('No data to filter\n', with_stdout=verbose)
           return

       # check paths exist, end with a "/" and convert them to absolute paths
       rawpath = saltio.abspath(rawpath)
       prodpath = saltio.abspath(prodpath)
       outpath = saltio.abspath(outpath)

       #create the symlink raw path
       rawsplit=rawpath.strip().split('/')
       symrawpath='../../%s/%s/' % (rawsplit[-3], rawsplit[-2])
       prodsplit=prodpath.strip().split('/')
       symprodpath='../../%s/%s/' % (prodsplit[-3], prodsplit[-2])
  

       # create PI directories
       for pid in pids:
           saltio.createdir(outpath+pid)
           saltio.createdir(outpath+pid+'/raw')
           saltio.createdir(outpath+pid+'/product')

       #copy the data that belongs to a pid into that directory
       log.message('SALTOBSID -- filtering images to proposal directories\n', with_stdout=verbose)

       #copy data for a given proposal to the raw and produce directories
       for i in range(len(obstab)):
           if os.path.exists(outpath+obstab[i]['propid']):
             if obstab[i]['object'].upper() not in ['ZERO', 'BIAS']:
               fname=obstab[i]['filename']
               pdir=obstab[i]['propid']
               detmode=obstab[i]['detmode']
               linkfiles(fname, pdir,detmode, symrawpath, symprodpath, outpath, prefix, fprefix, clobber)
               message='Copying %s to %s' % (fname, pdir)
               log.message(message, with_header=False, with_stdout=verbose)

       #look through the bias/flat/arc/standard data to see if there is any relavent data
       log.message('SALTOBSID -- filtering calibration files to proposal directories\n', with_stdout=verbose)

       caldata=['ZERO', 'FLAT', 'ARC']
       biasheader_list=['DETMODE', 'CCDSUM', 'GAINSET', 'ROSPEED']
       flatheader_list=['DETMODE', 'CCDSUM', 'GAINSET', 'ROSPEED', 'FILTER', 'GRATING', 'GR-ANGLE', 'AR-ANGLE']
       archeader_list=['OBSMODE', 'DETMODE', 'CCDSUM', 'GAINSET', 'ROSPEED', 'FILTER', 'GRATING', 'GR-ANGLE', 'AR-ANGLE']
       
       calproplist=['CAL_SPST']
       #Include bias frames
       log.message('SALTOBSID -- filtering bias files to proposal directories\n', with_stdout=verbose)

       for i in range(len(obstab)):
           fname=obstab[i]['filename']
           prop_list=[]
           #if it is a zero, check to see what other data have the same settings 
           if obstab[i]['CCDTYPE'].strip().upper()=='ZERO' or obstab[i]['OBJECT'].strip().upper() in ['BIAS', 'ZERO']:
               for j in range(len(obstab)):
                   if comparefiles(obstab[i], obstab[j], biasheader_list):
                       prop_list.append(obstab[i]['PROPID'])

           prop_list=saltio.removebadpids(set(prop_list))
           for pdir in prop_list:
                   detmode=obstab[i]['detmode']
                   linkfiles(fname, pdir, detmode, symrawpath, symprodpath, outpath,  fprefix, fprefix, clobber)
                   message='Copying %s to %s' % (fname, pdir)
                   log.message(message, with_header=False, with_stdout=verbose)

       #Include calibration  frames
       log.message('SALTOBSID -- filtering  calibration files to proposal directories\n', with_stdout=verbose)
 
       for i in range(len(obstab)):
           fname=obstab[i]['filename']
           prop_list=[]

           #if it is a flat, check to see what other data have the same settings 
           #this is turned off
           if obstab[i]['CCDTYPE'].strip().upper()=='FLAT' and False:
               for j in range(len(obstab)):
                   if comparefiles(obstab[i], obstab[j],  flatheader_list):
                       prop_list.append(obstab[j]['PROPID'])

           #if it is a arc, check to see what other data have the same settings 
           #this is turned off
           if obstab[i]['CCDTYPE'].strip().upper()=='ARC' and False:
               for j in range(len(obstab)):
                   if comparefiles(obstab[i], obstab[j],  archeader_list):
                       prop_list.append(obstab[j]['PROPID'])


           #if it is a calibration standard, see what other data have the same settings
           if obstab[i]['PROPID'].strip().upper() in calproplist:
               for j in range(len(obstab)):
                   if comparefiles(obstab[i], obstab[j],  flatheader_list):
                       prop_list.append(obstab[j]['PROPID'])


           prop_list=saltio.removebadpids(set(prop_list))
           for pdir in prop_list:
               if pdir!=obstab[i]['propid']:
                   detmode=obstab[i]['detmode']
                   linkfiles(fname, pdir, detmode, symrawpath, symprodpath, outpath,  prefix, fprefix, clobber)
                   message='Copying %s to %s' % (fname, pdir)
                   log.message(message, with_header=False, with_stdout=verbose)

       #Include master (bias or flat) frames
       log.message('SALTOBSID -- filtering master calibration files to proposal directories\n', with_stdout=verbose)
       masterlist=glob.glob(prodpath+'*Bias*')+glob.glob(prodpath+'*Flat*')
       for bimg in masterlist:
           struct=pyfits.open(bimg)
           bdict={}
           prop_list=[]
           for k in biasheader_list:
               bdict[k]=saltkey.get(k, struct[0])
           for i in range(len(obstab)):
               if comparefiles(obstab[i], bdict,  biasheader_list):
                   prop_list.append(obstab[i]['PROPID'])
           struct.close()

           #copy the files over to the directory
           prop_list=saltio.removebadpids(set(prop_list))
           for pdir in prop_list:
               fname=os.path.basename(bimg)
               infile = symprodpath+fname
               link = outpath+pdir+'/product/'+fname
               saltio.symlink(infile,link,clobber)
               message='Copying %s to %s' % (fname ,pdir)
               log.message(message, with_header=False, with_stdout=verbose)
Example #3
0
def hrscalibrations(
    obsdate,
    sdbhost="sdb.saao",
    sdbname="sdb",
    sdbuser="",
    password="",
    clobber=False,
    logfile="saltlog.log",
    verbose=True,
):
    """Seach the salt database for FITS files

    """

    with logging(logfile, debug) as log:

        # make obsdate a string if needed
        obsdate = str(obsdate)
        log.message("Sorting HRS calibration data")

        # connect to the database
        sdb = saltmysql.connectdb(sdbhost, sdbname, sdbuser, password)

        # first select propcodes for all HRS data from that day and check for any CAL data
        table = "FileData join ProposalCode using (ProposalCode_Id)"
        logic = "FileName like 'H" + obsdate + "%'  or FileName like 'R" + obsdate + "%'"
        records = saltmysql.select(sdb, "FileName, Proposal_Code", table, logic)

        # exit if no data was taken for HRS
        if len(records) == 0:
            return

        # Loop through each of the files and create the directories if needed
        image_dict = {}
        for filename, propid in records:
            if propid.count("CAL"):

                # check for directory and create structure
                caldir = "/salt/HRS_Cals/%s/" % propid
                if not os.path.isdir(caldir):
                    os.mkdir(caldir)
                yeardir = "%s%s/" % (caldir, obsdate[0:4])
                if not os.path.isdir(yeardir):
                    os.mkdir(yeardir)
                daydir = "%s%s/%s/" % (caldir, obsdate[0:4], obsdate[4:8])
                if not os.path.isdir(daydir):
                    os.mkdir(daydir)
                rawdir = "%s%s/%s/raw/" % (caldir, obsdate[0:4], obsdate[4:8])
                if not os.path.isdir(rawdir):
                    os.mkdir(rawdir)
                prodir = "%s%s/%s/product/" % (caldir, obsdate[0:4], obsdate[4:8])
                if not os.path.isdir(prodir):
                    os.mkdir(prodir)

                # create the symlinks to the files
                infile = "/salt/data/%s/%s/hrs/raw/%s" % (obsdate[0:4], obsdate[4:8], filename)
                link = "%s%s" % (rawdir, filename)
                saltio.symlink(infile, link, clobber)

                infile = "/salt/data/%s/%s/hrs/product/mbgph%s" % (obsdate[0:4], obsdate[4:8], filename)
                link = "%smbgph%s" % (prodir, filename)
                saltio.symlink(infile, link, clobber)

                log.message(
                    "Copied %s to the HRS_CAL/%s directory" % (filename, propid), with_header=False, with_stdout=verbose
                )

                # create log of files
                image_info = get_image_info(sdb, filename)
                try:
                    image_dict[propid].append([image_info])
                except:
                    image_dict[propid] = [[image_info]]

        # create log of each file--currently not enough information
        # in database to do this
        nightlog = "/salt/logs/sanightlogs/%s.log" % obsdate
        for k in image_dict:
            nightlink = "/salt/HRS_Cals/%s/%s/%s/%s.log" % (k, obsdate[0:4], obsdate[4:8], obsdate)
            saltio.symlink(nightlog, nightlink, clobber)
            fout = open("/salt/HRS_Cals/%s/%s/%s/%s.log" % (k, obsdate[0:4], obsdate[4:8], k), "w")
            for imlist in image_dict[k]:
                for info in imlist:
                    for f in info:
                        fout.write("%20s " % str(f))
                fout.write("\n")
            fout.close()