def saltdataquality(obsdate, sdbhost = "sdb.saao", sdbname = "sdb", sdbuser = "",
password = '', clobber=False, logfile='salt.log',verbose=True):
"""For a given SALT data. Move the data to /salt/data after it has been run by the pipeline"""
#make a dataquality directory
dqdir='dq%s' % obsdate
if os.path.isdir(dqdir) and clobber:
saltio.deletedir(dqdir)
saltio.createdir(dqdir)
saltio.changedir(dqdir)
with logging(logfile,debug) as log:
#check the entries
saltio.argdefined('obsdate',str(obsdate))
#open up the database
sdb=saltmysql.connectdb(sdbhost, sdbname, sdbuser, password)
#run for each instrument
for instrume in ['rss', 'scam']:
log.message('Measuring Data Quality for %s observations' % instrume.upper())
dataquality(str(obsdate), sdb, instrume, clobber, logfile, verbose)
def saltfast(obsdate, readme, emailserver, username,password, bcc, sdbhost,
sdbname,sdbuser, clobber,logfile,verbose):
# set up
nightlog = ''
with logging(logfile,debug) as log:
# determine current directory
workdir = os.getcwd()
logfile = workdir + '/' + os.path.basename(logfile)
#log into the database
sdb=saltmysql.connectdb(sdbhost,sdbname,sdbuser,password)
#query the data base for all of the data taken last night and the proposal codes
select='FileName, Target_Name, Proposal_Code'
table='FileData join ProposalCode using (ProposalCode_Id)'
logic="FileName like '%"+obsdate+"%'"
records=saltmysql.select(sdb, select, table, logic)
if len(records)==0:
message='No data taken on %s\n' % obsdate
log.message(message)
return
#determine the list of files, targets, and propcodes
file_list=[]
target_list=[]
propcode_list=[]
propcode_dict={}
for name,targ,pid in records:
file_list.append(name)
target_list.append(targ)
propcode_list.append(pid)
try:
propcode_dict[pid].append(targ)
except KeyError:
propcode_dict[pid]=[targ]
# check to see if any of the PI directories requires fast response
for pid in propcode_dict.keys():
target=set(propcode_dict[pid])
if checkforfast(pid, target, sdb):
#log the move
message='Copying data from %s to the ftp site' % pid
log.message(message, with_stdout=verbose)
#get the username
piptuser=saltmysql.getpiptusername(sdb, pid)
#create the fast directory
fastdir='/salt/ftparea/%s/FAST/' % piptuser
if os.path.isdir(fastdir): saltio.deletedir(fastdir)
os.mkdir(fastdir)
for i in range(len(file_list)):
if propcode_list[i]==pid:
#create the data filename
if file_list[i].startswith('S'):
instr='scam'
elif file_list[i].startswith('P'):
instr='rss'
filepath='/salt/%s/data/%s/%s/raw/%s' % (instr, obsdate[0:4], obsdate[4:8], file_list[i])
saltio.copy(filepath, fastdir)
#make the temporary readme
mailmessage = maketempreadmefast(pid, sdb, readme)
#send the email
subject='SALT Raw Data available for %s' % pid
sender='*****@*****.**'
recipient=saltmysql.getpiptemail(sdb, piptuser)
bcc='*****@*****.**'#[email protected]'