def saltpipe(obsdate,pinames,archive,ftp,email,emserver,emuser,empasswd,bcc, qcpcuser,qcpcpasswd, ftpserver,ftpuser,ftppasswd,sdbhost, sdbname, sdbuser, sdbpass, elshost, elsname, elsuser, elspass, median,function,order,rej_lo,rej_hi,niter,interp, clobber, runstatus, logfile,verbose): # set up basedir=os.getcwd() propcode=pinames sender = emuser + '@salt.ac.za' recipient = sender emessage = '' emailfile = '../piemaillist/email.lis' # check the observation date is sensible if ('/' in obsdate or '20' not in obsdate or len(obsdate) != 8): emessage = 'Observation date does not look sensible - YYYYMMDD\n' raise SaltError(emessage) # stop if the obsdate temporary directory already exists obsdir='%s' % obsdate if os.path.exists(obsdir): emessage += 'The temporary working directory ' + os.getcwd() + '/' emessage += obsdate + ' already exists. ' raise SaltError(emessage) # create a temporary working directory and move to it saltio.createdir(obsdir) saltio.changedir(obsdir) workpath = saltio.abspath('.') # test the logfile logfile = workpath+logfile logfile = saltio.logname(logfile) #note the starttime starttime = time.time() #start logging with logging(logfile,debug) as log: #connect to the database sdb=saltmysql.connectdb(sdbhost, sdbname, sdbuser, sdbpass) #get the nightinfo id nightinfoid=saltmysql.getnightinfoid(sdb, obsdate) #Get the list of proposal codes state_select='Proposal_Code' state_tables='Proposal join ProposalCode using (ProposalCode_Id)' state_logic="current=1" records=saltmysql.select(sdb, state_select, state_tables, state_logic) propids=[k[0] for k in records] # Calculate the current date currentdate=salttime.currentobsdate() # are the arguments defined saltio.argdefined('obsdate',obsdate) # check email and ftp arguments are consistent if email and not ftp: message = 'ERROR: SALTPIPE -- cannot send email to PI(s) unless data is transferred ' message += 'to the FTP server; use ftp=\'yes\' email=\'yes\'' raise SaltError(message) # identify a potential list of keyword edits keyfile = '../newheadfiles/list_newhead_' + obsdate if not os.path.isfile(keyfile): message = '\nSALTPIPE -- keyword edits ' + keyfile + ' not found locally' log.message(message) # check directories for the raw RSS data rssrawpath = makerawdir(obsdate, 'rss') # check directories for the raw SALTICAM data scmrawpath = makerawdir(obsdate, 'scam') # check raw directories for the disk.file record and find last file number #check rss data lastrssnum = checkfordata(rssrawpath, 'P', obsdate, log) #check scame data lastscmnum = checkfordata(scmrawpath, 'S', obsdate, log) #check for HRS Data--not filedata yet, so cannot check if lastrssnum == 1 and lastscmnum == 1: message = 'SALTPIPE -- no SALTICAM or RSS data obtained on ' + obsdate emessage += '\n' + message + '\n' log.message(message) #copy the data to the working directory if lastrssnum > 1: message = 'Copy ' + rssrawpath + ' --> ' + workpath + 'raw/' log.message(message) saltio.copydir(rssrawpath,'rss/raw') if lastscmnum > 1: message = 'Copy ' + scmrawpath + ' --> ' + workpath + 'raw/' log.message(message) saltio.copydir(scmrawpath,'scam/raw') #copy and pre-process the HRS data try: hrsbrawpath = makerawdir(obsdate, 'hbdet') saltio.createdir('hrs') saltio.createdir('hrs/raw') message = 'Copy ' + hrsbrawpath + ' --> ' + workpath + 'raw/' log.message(message) salthrspreprocess(hrsbrawpath, 'hrs/raw/', clobber=True, log=log, verbose=verbose) lasthrbnum=len(glob.glob('hrs/raw/*fits')) except Exception,e: log.message('Could not copy HRS data because %s' % e) lasthrbnum=0 try: hrsrrawpath = makerawdir(obsdate, 'hrdet') message = 'Copy ' + hrsrrawpath + ' --> ' + workpath + 'raw/' log.message(message) salthrspreprocess(hrsrrawpath, 'hrs/raw/', clobber=True, log=log, verbose=verbose) lasthrsnum=max(lasthrbnum, len(glob.glob('hrs/raw/*fits'))) except Exception,e: log.message('Could not copy HRS data because %s' % e) lasthrsnum=lasthrbnum
def saltpipe(obsdate,pinames,archive,ftp,email,emserver,emuser,empasswd,bcc, qcpcuser,qcpcpasswd, ftpserver,ftpuser,ftppasswd,sdbhost, sdbname, sdbuser, sdbpass, elshost, elsname, elsuser, elspass, median,function,order,rej_lo,rej_hi,niter,interp, clobber, runstatus, logfile,verbose): # set up basedir=os.getcwd() propcode=pinames sender = emuser + '@salt.ac.za' recipient = sender emessage = '' emailfile = '../piemaillist/email.lis' # check the observation date is sensible if ('/' in obsdate or '20' not in obsdate or len(obsdate) != 8): emessage = 'Observation date does not look sensible - YYYYMMDD\n' raise SaltError(emessage) # stop if the obsdate temporary directory already exists obsdir='%s' % obsdate if os.path.exists(obsdir): emessage += 'The temporary working directory ' + os.getcwd() + '/' emessage += obsdate + ' already exists. ' raise SaltError(emessage) # create a temporary working directory and move to it saltio.createdir(obsdir) saltio.changedir(obsdir) workpath = saltio.abspath('.') # test the logfile logfile = workpath+logfile logfile = saltio.logname(logfile) #note the starttime starttime = time.time() #start logging with logging(logfile,debug) as log: #connect to the database sdb=saltmysql.connectdb(sdbhost, sdbname, sdbuser, sdbpass) #get the nightinfo id nightinfoid=saltmysql.getnightinfoid(sdb, obsdate) #Get the list of proposal codes state_select='Proposal_Code' state_tables='Proposal join ProposalCode using (ProposalCode_Id)' state_logic="current=1" records=saltmysql.select(sdb, state_select, state_tables, state_logic) propids=[k[0] for k in records] # Calculate the current date currentdate=salttime.currentobsdate() # are the arguments defined saltio.argdefined('obsdate',obsdate) # check email and ftp arguments are consistent if email and not ftp: message = 'ERROR: SALTPIPE -- cannot send email to PI(s) unless data is transferred ' message += 'to the FTP server; use ftp=\'yes\' email=\'yes\'' raise SaltError(message) # identify a potential list of keyword edits keyfile = '../newheadfiles/list_newhead_' + obsdate if not os.path.isfile(keyfile): message = '\nSALTPIPE -- keyword edits ' + keyfile + ' not found locally' log.message(message) # check directories for the raw RSS data rssrawpath = makerawdir(obsdate, 'rss') # check directories for the raw SALTICAM data scmrawpath = makerawdir(obsdate, 'scam') # check raw directories for the disk.file record and find last file number #check rss data lastrssnum = checkfordata(rssrawpath, 'P', obsdate, log) #check scame data lastscmnum = checkfordata(scmrawpath, 'S', obsdate, log) #check for HRS Data--not filedata yet, so cannot check if lastrssnum == 1 and lastscmnum == 1: message = 'SALTPIPE -- no SALTICAM or RSS data obtained on ' + obsdate emessage += '\n' + message + '\n' log.message(message) #copy the data to the working directory if lastrssnum > 1: message = 'Copy ' + rssrawpath + ' --> ' + workpath + 'raw/' log.message(message) saltio.copydir(rssrawpath,'rss/raw') if lastscmnum > 1: message = 'Copy ' + scmrawpath + ' --> ' + workpath + 'raw/' log.message(message) saltio.copydir(scmrawpath,'scam/raw') #copy and pre-process the HRS data try: hrsbrawpath = makerawdir(obsdate, 'hbdet') saltio.createdir('hrs') saltio.createdir('hrs/raw') message = 'Copy ' + hrsbrawpath + ' --> ' + workpath + 'raw/' log.message(message) salthrspreprocess(hrsbrawpath, 'hrs/raw/', clobber=True, log=log, verbose=verbose) hrsrrawpath = makerawdir(obsdate, 'hrdet') message = 'Copy ' + hrsrrawpath + ' --> ' + workpath + 'raw/' log.message(message) salthrspreprocess(hrsrrawpath, 'hrs/raw/', clobber=True, log=log, verbose=verbose) lasthrsnum=len(glob.glob('hrs/raw/*fits')) except Exception,e: log.message('Could not copy HRS data because %s' % e) lasthrsnum=0 if lastrssnum>1 or lastscmnum>1: message = 'Copy of data is complete' log.message(message) else: message = 'No data was taken on %s' % obsdate log.message(message) #process the data RSS data if lastrssnum>1: preprocessdata('rss', 'P', obsdate, keyfile, log, logfile, verbose) #process the SCAM data if lastscmnum>1: preprocessdata('scam', 'S', obsdate, keyfile, log, logfile, verbose) #process the HRS data if lasthrsnum>1: preprocessdata('hrs', 'H', obsdate, keyfile, log, logfile, verbose) preprocessdata('hrs', 'R', obsdate, keyfile, log, logfile, verbose) #check that all data was given a proper proposal id #only do it for semesters after the start of science operations if int(obsdate)>=20110901: # Check to see that the PROPID keyword exists and if not add it message = '\nSALTPIPE -- Checking for PROPID keyword' log.message(message) #check rss data rssstatus=runcheckforpropid(glob.glob('rss/raw/P*.fits'), propids, log) #check scam data scmstatus=runcheckforpropid(glob.glob('scam/raw/S*.fits'), propids, log) #check hrsB data hrsbstatus=runcheckforpropid(glob.glob('hrs/raw/H*.fits'), propids, log) #check hrsB data hrsrstatus=runcheckforpropid(glob.glob('hrs/raw/R*.fits'), propids, log) if not rssstatus or not scmstatus or not hrsbstatus or not hrsrstatus: msg='The PROPIDs for these files needs to be updated and re-start the pipeline' raise SaltError("Invalid PROPID in images:"+msg) #process the RSS data rssrawsize, rssrawnum, rssprodsize, rssprodnum=processdata('rss', obsdate, propcode, median, function, order, rej_lo, rej_hi, niter, interp,logfile, verbose) #advance process the data if rssrawnum > 0: advanceprocess('rss', obsdate, propcode, median, function, order, rej_lo, rej_hi, niter, interp,sdbhost, sdbname, sdbuser, sdbpass, logfile, verbose) #process the SCAM data scmrawsize, scmrawnum, scmprodsize, scmprodnum=processdata('scam', obsdate, propcode, median, function, order, rej_lo, rej_hi, niter, interp,logfile, verbose) #process the HRS data hrsrawsize, hrsrawnum, hrsprodsize, hrsprodnum=hrsprocess('hrs', obsdate, propcode, median, function, order, rej_lo, rej_hi, niter, interp, logfile, verbose) #upload the data to the database img_list=glob.glob(workpath+'scam/product/*bxgp*.fits') img_list.extend(glob.glob(workpath+'rss/product/*bxgp*.fits')) img_list.extend(glob.glob(workpath+'hrs/raw/*.fits')) if img_list: img=','.join('%s' % (k) for k in img_list) saltsdbloadfits(images=img, sdbname=sdbname, sdbhost=sdbhost, sdbuser=sdbuser, \ password=sdbpass, logfile=logfile, verbose=verbose) #add junk sources to the database raw_list=glob.glob(workpath+'scam/raw/S*.fits') raw_list.extend(glob.glob(workpath+'rss/raw/P*.fits')) if raw_list: img='' for img in raw_list: hdu=pyfits.open(img) if hdu[0].header['PROPID'].strip()=='JUNK': saltsdbloadfits(images=img, sdbname=sdbname, sdbhost=sdbhost, sdbuser=sdbuser, \ password=sdbpass, logfile=logfile, verbose=verbose) hdu.close() # construct observation and pipeline documentation if lastrssnum > 1 and rssrawnum>0: rssobslog = 'rss/product/P' + obsdate + 'OBSLOG.fits' else: rssobslog = 'none' if lastscmnum > 1 and scmrawnum>0: scmobslog = 'scam/product/S' + obsdate + 'OBSLOG.fits' else: scmobslog = 'None' if lasthrsnum > 1 and hrsrawnum>0: hrsobslog = 'hrs/product/H' + obsdate + 'OBSLOG.fits' else: hrsobslog = 'None' if rssrawnum==0 and scmrawnum==0 and hrsrawnum==0: msg='No data processed for %s' % obsdate email=False ftp=False log.message(msg) htmlpath = '.' nightlog = '../nightlogfiles/' + obsdate + '.log' readme = iraf.osfn('pipetools$html/readme.template') if not os.path.isfile(nightlog): nightlog = '' message = 'No night log file ~/nightlogfiles/' + obsdate + '.log found' log.warning(message) if (rssrawnum > 0 or scmrawnum > 0 or hrsrawnum>0): salthtml(propcode=propcode,scamobslog=scmobslog,rssobslog=rssobslog, hrsobslog=hrsobslog, htmlpath=htmlpath, nightlog=nightlog,readme=readme,clobber=True,logfile=logfile, verbose=verbose) #add a pause to allow syncing of the databases time.sleep(10) #Add in the environmental information if (rssrawnum > 0 or scmrawnum > 0 or hrsrawnum>0): propids=saltmysql.getpropcodes(sdb, obsdate) for pid in propids: saltelsdata(pid, obsdate, elshost, elsname, elsuser, elspass, sdbhost,sdbname,sdbuser, sdbpass, clobber, logfile,verbose) try: outfile='%s_%s_elsdata.fits' % (pid, obsdate) outdir='%s/doc/' % (pid) shutil.move(outfile, outdir) except: os.remove(outfile) #ftp the data beachdir='/salt/ftparea/' if ftp: try: saltftp(propcode=propcode,obsdate=obsdate, datapath=workpath, password=ftppasswd,beachdir=beachdir,sdbhost=sdbhost, sdbname=sdbname,sdbuser=sdbuser,splitfiles=False, cleanup=True,clobber=True,logfile=logfile, verbose=verbose) except Exception,e: message="Not able to copy data to FTP area:\n%s " % e raise SaltError(message) #run with the splitting of files try: saltftp(propcode=propcode,obsdate=obsdate, datapath=workpath, password=ftppasswd,beachdir=beachdir,sdbhost=sdbhost, sdbname=sdbname,sdbuser=sdbuser,splitfiles=True, cleanup=True,clobber=True,logfile=logfile, verbose=verbose) except Exception,e: message="Not able to copy data to FTP area:\n%s " % e raise SaltError(message)