def udb_init_ifdbfiles_1day(day_in, fdbtype='IFDB'):
'''Reads in the available IFDB files, for the given day, formatted YYYYMMDD'''
# check for what type of filedb you're looking for
# currently FDB, IFDB
if fdbtype == 'IFDB':
xdir = ifdbdir
elif fdbtype == 'FDB':
xdir = fdbdir
elif fdbtype == 'UFDB':
xdir = ufdbdir
else:
xdir = fdbdir
#endif
ppp = day_in
if fdbtype == 'FDB':
fdbfile = xdir + fdbtype + ppp + '.txt'
else:
yyyy = day_in[0:4] + '/'
fdbfile = xdir + yyyy + fdbtype + ppp + '.txt'
#endif
#Now read in the ifdb entries from the file
ifb = fdb.pfdb_read(fdbfile)
return ifb
def udb_reprocess_days(start_day, end_day):
'''Reprocess UDB data from the start to end day. Reads in the IFDB files
for the time interval, and the previous and next days, sets status flags
to zero for all of the IFDB entries for the given day, and reprocesses
until the files are finished. Start and end days are passed in as time
strings YYYY-MM-DD 00:00:00 (or %Y-%m-%d HH:MM:SS). Days are inclusive,
so that if start_day = end_day, then that 1 day is processed.'''
tform = '%Y-%m-%d %H:%M:%S'
time0 = time.mktime(time.strptime(start_day, tform))
time1 = time.mktime(time.strptime(end_day, tform))
#number of days to load
ndays0 = int((time1 - time0) / one_day) + 1
#get ifdb files for this time range, set status to 0 for all and rewrite
ifdb_files = udb_init_fdbfiles(ndays=ndays0, fdbtype='IFDB', day0=end_day)
if len(ifdb_files) == 0:
print "UDB_REPROCESS_DAYS: No files to process"
print "Start_day = ", start_day
print "End_day = ", end_day
sys.stdout.flush()
return []
#endif
ifb = []
for j in range(len(ifdb_files)):
ifbj = fdb.pfdb_read(ifdb_files[j])
if len(ifbj) > 0:
ifb = ifb + ifbj
#endif
#endfor
if len(ifb) == 0:
print "UDB_REPROCESS_DAYS: No files to process ???"
print "Start_day = ", start_day
print "End_day = ", end_day
sys.stdout.flush()
return []
#endif
#set status to zero
for j in range(len(ifb)):
ifb[j].pstatus = 0
print "Reset Pstatus: ", ifb[j].fileid
sys.stdout.flush()
#endfor
#And rewrite the IFDB files
ifdbfilesout = update_ifdb(ifb)
#Now reprocess the ifdbs, adding days insures that we get scans
#that overlap day boundaries
ndays2 = 2 + ndays0
#day02 will be day0 input
day02 = time.strftime(tform, time.localtime(time1 + one_day))
ufb_out = ['dummy']
while len(ufb_out) > 0:
ufb_out = udb_process(ndays=ndays2, day0=day02, reprocess=True)
if len(ufb_out) > 0:
print "UDB_REPROCESS_DAYS: Processed ", ufb_out
else:
print "UDB_REPROCESS_DAYS: No Processing "
sys.stdout.flush()
#endelse
#endwhile
print "UDB_REPROCESS_DAYS: Finished Processing "
print "Start_day = ", start_day
print "End_day = ", end_day
sys.stdout.flush()
return ['OK']
def update_ufdb(ufb, ndays=5, day0=None):
'''Updates ufdb files, need to read the last ndays files, check to
see if you're overwriting an entry, if not then append the new
scan to the list and output. If so, replace the old entry, then
output. Note that ufb is a 1-element list'''
if len(ufb) == 0:
print 'UPDATE_UFDB ERROR: No UFB input'
sys.stdout.flush()
files_out = []
#get filenames
ufdb_files = udb_init_fdbfiles(ndays=ndays, fdbtype='UFDB', day0=day0)
ufb_in = []
if len(ufdb_files) == 0:
#maybe there are none
print 'UPDATE_UFDB: No UFDB files in range:'
sys.stdout.flush()
else:
for j in range(len(ufdb_files)):
ufbj = fdb.pfdb_read(ufdb_files[j])
if len(ufbj) > 0:
ufb_in = ufb_in + ufbj
#endif
#endfor
#endif
# If there is no ufb_in, then it's easy
if len(ufb_in) == 0:
ufb_out = ufb
else:
# Check to see if there is an entry for the current ufb
uflist_in = fdb.fdb_list_fileid(ufb_in)
ufarray = array(uflist_in)
# return the subscript
ss_ufb_in = extract(ufarray == ufb[0].fileid,
array(range(len(ufb_in))))
#I think ss_ufb_in should be a scalar..
if len(ss_ufb_in) == 0:
ufb_out = ufb_in + ufb
else:
# Get all of the entries that don't have this fileid
ss_ufb_not = extract(ufarray != ufb[0].fileid,
array(range(len(ufb_in))))
if len(ss_ufb_not) == 0:
ufb_out = ufb
else:
ufb_out = []
for j in range(len(ss_ufb_not)):
ufb_out = ufb_out + [ufb_in[ss_ufb_not[j]]]
#endfor
ufb_out = ufb_out + ufb
#endif
#endif
#endif
#Here, now output the ufb_out entries
files_out = []
#Get days for each ufb entry
ufdays = fdb.fdb_list_fileid(ufb_out)
for j in range(len(ufdays)):
y = str.find(ufdays[j], 'DB')
x0 = y + 2
x1 = y + 10
ufdays[j] = ufdays[j][x0:x1]
#endfor
#Get the filenames for the files,
days_out = fdb.fdb_uniq_day(ufb_out)
#Adjustment due to lack
ndays_out = len(days_out)
if ndays_out == 0:
print 'UPDATE_UFDB Error: No days to process'
sys.stdout.flush()
else:
#need an array of days for extract commands
udyarr = array(ufdays)
for j in range(ndays_out):
#Find files with dates matching each day here, build a
#list to output
ufbj = extract(udyarr == days_out[j], ufb_out)
ufbj_out = ufbj.tolist()
if len(ufbj_out) == 0:
print 'UPDATE_UFDB Error: No output for: ' + days[j]
sys.stdout.flush()
else:
#filename
yyyy = days_out[j]
yyyy = yyyy[:4]
filenamej = ufdbdir + '/' + yyyy + '/' + 'UFDB' + days_out[
j] + '.txt'
#check for yyyy
if os.path.isdir(ufdbdir + yyyy) != True:
os.mkdir(ufdbdir + yyyy)
fdb.pfdb_write(ufbj_out, filenamej)
print 'Wrote: ' + filenamej
sys.stdout.flush()
files_out = files_out + [filenamej]
#endif
#endfor
#endif
return files_out
def udb_fb2process(ndays=5, day0=None):
'''Reads in the last N days of FDB files, and IFDB files, finds
the IDB files that need processing, and returns the fdb ifdb
classes for those files'''
#get FDB, IFDB filenames
fdbfiles = udb_init_fdbfiles(ndays=ndays, day0=day0)
if len(fdbfiles) == 0:
print 'UDB_FB2PROCESS Error: No FDB files:'
sys.stdout.flush()
ifboops = []
fboops = []
return fboops, ifboops
#endif
ifdbfiles = udb_init_fdbfiles(ndays=ndays, fdbtype='IFDB', day0=day0)
if len(ifdbfiles) == 0:
# Not necessarily an error
print 'UDB_FB2PROCESS: No IFDB files:'
sys.stdout.flush()
#endif
#read in the files
fb = []
for j in range(len(fdbfiles)):
fbj = fdb.fdb_read(fdbfiles[j])
if len(fbj) > 0:
fb = fb + fbj
#endif
#endfor
ifb = []
if len(ifdbfiles) > 0:
for j in range(len(ifdbfiles)):
ifbj = fdb.pfdb_read(ifdbfiles[j])
if len(ifbj) > 0:
ifb = ifb + ifbj
#endif
#endfor
#endif
#check for non-empty fb's first
if len(fb) == 0:
print "no data to process"
sys.stdout.flush()
otp = []
return fb, otp
#endif
#if I am here, I have IDB files from the FDB. Do I have PFDB entries?
if len(ifb) == 0:
print "no IFDB entries, process all FDB entries"
sys.stdout.flush()
#For each FDB entry, create a IFDB entry, with pstatus = 0
for j in range(len(fb)):
ifbj = fdb.pfiledb(fb[j].fileid, fb[j].scanid, fb[j].sourceid,
fb[j].projectid, fb[j].st_ts, fb[j].en_ts, 0)
ifb.append(ifbj)
#endfor
return fb, ifb
#endif
#if I am here, I have both FDB and IFDB entries, for each FDB,
#check to see if there is a corresponding IFDB. If not, create one
#with pstatus = 0
iflist = fdb.fdb_list_fileid(ifb)
iidarray = array(iflist)
ifb_out = []
for j in range(len(fb)):
fileidj = fb[j].fileid
ifbtemp = extract(iidarray == fileidj, ifb)
if (len(ifbtemp) > 0):
ifb_outj = ifbtemp[0]
else:
ifb_outj = fdb.pfiledb(fb[j].fileid, fb[j].scanid, fb[j].sourceid,
fb[j].projectid, fb[j].st_ts, fb[j].en_ts,
0)
#endif
ifb_out.append(ifb_outj)
#endfor
return fb, ifb_out
def udb_process_spec(ndays=3):
'''This process reads the UDB filedb and creates spectral FITS
files for each scan as necessary '''
import subprocess
#Read in UFDB files and check status.
ufdbfiles = udb_init_fdbfiles(ndays=ndays, fdbtype='UFDB')
if len(ufdbfiles) == 0:
print 'UDB_PROCESS_SPEC Error: No UFDB files:'
ufboops = []
return ufboops
ufb = []
for j in range(len(ufdbfiles)):
ufbj = fdb.pfdb_read(ufdbfiles[j])
if len(ufbj) > 0:
ufb = ufb + ufbj
#check for non-empty ufb's
if len(ufb) == 0:
print "UDB_PROCESS_SPEC: no data to process"
return ufb
#If I am here, I have a valid ufb, but do I have status = 0, If
#so, get the first one and process it.
do_this_file = -1
for j in range(len(ufb)):
if do_this_file == -1:
if ufb[j].pstatus == 0:
do_this_file = j
#the variable do_this_file now contains the first filename, unless
#it's -1, then no processing happens
if do_this_file == -1:
print "UDB_PROCESS_SPEC: no data to process"
print 'UDB_PROCESS_SPEC: finished successfully'
ufb_out = []
idlfile = "/home/user/workdir/udb_process_spec.pro"
f = open(idlfile, 'w')
s = "message, /info, 'No data to process'\n"
f.write(s)
s = "end\n"
f.write(s)
f.close
return ufb_out
ufileid = ufb[do_this_file].fileid
yyyy = ufileid[3:7]
print 'Processing: ', ufileid
ufilename = udbdir + yyyy + '/' + ufileid
# Write text files for xtsys, ytsys, sfreq and header for this file
xtsysfile = udbtxtdir + yyyy + '/' + ufileid + '_xtsys.txt'
ytsysfile = udbtxtdir + yyyy + '/' + ufileid + '_ytsys.txt'
sfreqfile = udbtxtdir + yyyy + '/' + ufileid + '_sfreq.txt'
TaskVarplt(vis=ufilename, xaxis='time', yaxis='xtsys', log=xtsysfile).run()
TaskVarplt(vis=ufilename, xaxis='time', yaxis='ytsys', log=ytsysfile).run()
TaskVarplt(vis=ufilename, xaxis='time', yaxis='sfreq', log=sfreqfile).run()
hdrfile = udbtxtdir + yyyy + '/' + ufileid + '_hdr.txt'
TaskPrintHead(in_=ufilename, log=hdrfile).run()
# Here figure out how to call the IDL process from here, write a
# batch file and run ssw_batch using the subprocess command
# 2014-06-26. Instead, just create the batch file and we will run
# ssw_batch directly from the shell script
idlfile = "/home/user/workdir/udb_process_spec.pro"
f = open(idlfile, 'w')
s = "fileid = '" + ufileid + "'\n"
f.write(s)
s = "hdrfile = '" + hdrfile + "'\n"
f.write(s)
s = "sfreqfile = '" + sfreqfile + "'\n"
f.write(s)
s = "xtsysfile = '" + xtsysfile + "'\n"
f.write(s)
s = "ytsysfile = '" + ytsysfile + "'\n"
f.write(s)
s = "eovsa_write_pwrfits, fileid, hdrfile, sfreqfile, xtsysfile, ytsysfile\n"
f.write(s)
s = "end\n"
f.write(s)
f.close
#subprocess is not dealing correctly with ssw_batch arguments, so write a shell script to spawn
# shellfile = "/home/user/workdir/udb_process_specpro.csh"
# f = open(shellfile, 'w')
# f.write("#! /bin/csh\n")
# f.write(" \n")
# f.write("ssw_batch "+idlfile+"\n")
# f.close
#hard coded, BECAUSE PYTHON SUCKS
# subprocess.call(["/bin/csh", "/home/user/workdir/udb_process_specpro.sh"])
# os.spawnl("ssw_batch udb_process_spec.pro")
# subprocess.call(["ssw_batch", "udb_process_spec.pro"])
# subprocess.call(["ssw_batch", "/home/user/workdir/udb_process_spec.pro"])
# subprocess.call(["rm", "-f", idlfile])
ufb[do_this_file].pstatus = 1
ufb_out = [ufb[do_this_file]]
#Write out ufdb entry, ufb_out is a 1 element list
ufb_files_out = update_ufdb(ufb_out, ndays=ndays)
print 'UDB_PROCESS_SPEC: Processed Spectra for: ' + ufileid
print 'UDB_PROCESS_SPEC: finished successfully'
return ufb_out