def udb_process1scan(ifb_1, reprocess=False):
'''Given the set of ifb entries, process them into a single UDB
Miriad dataset for this scan'''
n1 = len(ifb_1)
filelist = fdb.fdb_list_fileid(ifb_1)
print filelist
#Next, name the output files, this should be based on the start
#time of the first file, changed from using scan_id, 2014-06-17,
#jmm
scan_id = ifb_1[0].scanid
source_id = ifb_1[0].sourceid
project_id = ifb_1[0].projectid
st_ts = ifb_1[0].st_ts
en_ts = ifb_1[n1 - 1].en_ts
#Take care of two digit year here
# ufileid = 'UDB'+'20'+scan_id
ufileid = 'U' + ifb_1[0].fileid[1:]
yyyy = ufileid[3:7]
ufilename = udbdir + yyyy + '/' + ufileid
ufb = fdb.pfiledb(ufileid, scan_id, source_id, project_id, st_ts, en_ts, 0)
# check for reprocess
if reprocess == True:
idb_datadir = idbdir
else:
idb_datadir = idbfinaldir + yyyy + '/'
#endelse
# Call udbfile_create for this filelist
filelist_full = filelist
for j in range(len(filelist)):
filelist_full[j] = idb_datadir + filelist[j]
#endfor
if n1 > 50:
print "UDB_PROCESS1SCAN: Too many Files N1 = ", n1
print "UDB_PROCESS1SCAN: Reset to 50"
n1 = 50
filelist_full = filelist_full[0:n1 - 1]
#endif
#If the file exists, you need to delete it
if os.path.isdir(ufilename) == True:
print "UDB_PROCESS1SCAN: dataset: ", ufilename, " will be deleted"
shutil.rmtree(ufilename)
#endif
ufile_out = udb_util.udbfile_create(filelist_full, ufilename)
if (len(ufile_out) == 0):
print "UDB_PROCESS1SCAN: Error creating: ", ufilename
return []
else:
return [ufb]
def update_ifdb(ifb):
''' Updates ifdb files, Here we just write out the files for the
previous ndays days'''
files_out = []
#Get days for each ifb entry
ifdays = fdb.fdb_list_fileid(ifb)
# 'FDByyyymmddhhmmss' extract subscript range 3 to 11 to get the date
# 'IFDByyyymmddhhmmss' extract subscript range 4 to 12 to get the date
for j in range(len(ifdays)):
y = str.find(ifdays[j], 'DB')
x0 = y + 2
x1 = y + 10
ifdays[j] = ifdays[j][x0:x1]
#endfor
#Get the filenames for the files,
days = fdb.fdb_uniq_day(ifb)
ndays_out = len(days)
if ndays_out == 0:
print 'UPDATE_IFDB Error: No days to process'
sys.stdout.flush()
else:
#need an array of days for extract commands
idyarr = array(ifdays)
for j in range(ndays_out):
#Find files with dates matching each day here, build a
#list to output
ifbj = extract(idyarr == days[j], ifb)
ifbj_out = ifbj.tolist()
if len(ifbj_out) == 0:
print 'UPDATE_IFDB Error: No output for: ' + days[j]
sys.stdout.flush()
else:
#filename
yyyy = days[j]
yyyy = yyyy[:4]
filenamej = ifdbdir + '/' + yyyy + '/' + 'IFDB' + days[
j] + '.txt'
#check for yyyy directory
if os.path.isdir(ifdbdir + yyyy) != True:
os.mkdir(ifdbdir + yyyy)
fdb.pfdb_write(ifbj_out, filenamej)
print 'Wrote: ' + filenamej
sys.stdout.flush()
files_out = files_out + [filenamej]
#endif
#endfor
#endif
return files_out
def update_ufdb(ufb, ndays=5, day0=None):
'''Updates ufdb files, need to read the last ndays files, check to
see if you're overwriting an entry, if not then append the new
scan to the list and output. If so, replace the old entry, then
output. Note that ufb is a 1-element list'''
if len(ufb) == 0:
print 'UPDATE_UFDB ERROR: No UFB input'
sys.stdout.flush()
files_out = []
#get filenames
ufdb_files = udb_init_fdbfiles(ndays=ndays, fdbtype='UFDB', day0=day0)
ufb_in = []
if len(ufdb_files) == 0:
#maybe there are none
print 'UPDATE_UFDB: No UFDB files in range:'
sys.stdout.flush()
else:
for j in range(len(ufdb_files)):
ufbj = fdb.pfdb_read(ufdb_files[j])
if len(ufbj) > 0:
ufb_in = ufb_in + ufbj
#endif
#endfor
#endif
# If there is no ufb_in, then it's easy
if len(ufb_in) == 0:
ufb_out = ufb
else:
# Check to see if there is an entry for the current ufb
uflist_in = fdb.fdb_list_fileid(ufb_in)
ufarray = array(uflist_in)
# return the subscript
ss_ufb_in = extract(ufarray == ufb[0].fileid,
array(range(len(ufb_in))))
#I think ss_ufb_in should be a scalar..
if len(ss_ufb_in) == 0:
ufb_out = ufb_in + ufb
else:
# Get all of the entries that don't have this fileid
ss_ufb_not = extract(ufarray != ufb[0].fileid,
array(range(len(ufb_in))))
if len(ss_ufb_not) == 0:
ufb_out = ufb
else:
ufb_out = []
for j in range(len(ss_ufb_not)):
ufb_out = ufb_out + [ufb_in[ss_ufb_not[j]]]
#endfor
ufb_out = ufb_out + ufb
#endif
#endif
#endif
#Here, now output the ufb_out entries
files_out = []
#Get days for each ufb entry
ufdays = fdb.fdb_list_fileid(ufb_out)
for j in range(len(ufdays)):
y = str.find(ufdays[j], 'DB')
x0 = y + 2
x1 = y + 10
ufdays[j] = ufdays[j][x0:x1]
#endfor
#Get the filenames for the files,
days_out = fdb.fdb_uniq_day(ufb_out)
#Adjustment due to lack
ndays_out = len(days_out)
if ndays_out == 0:
print 'UPDATE_UFDB Error: No days to process'
sys.stdout.flush()
else:
#need an array of days for extract commands
udyarr = array(ufdays)
for j in range(ndays_out):
#Find files with dates matching each day here, build a
#list to output
ufbj = extract(udyarr == days_out[j], ufb_out)
ufbj_out = ufbj.tolist()
if len(ufbj_out) == 0:
print 'UPDATE_UFDB Error: No output for: ' + days[j]
sys.stdout.flush()
else:
#filename
yyyy = days_out[j]
yyyy = yyyy[:4]
filenamej = ufdbdir + '/' + yyyy + '/' + 'UFDB' + days_out[
j] + '.txt'
#check for yyyy
if os.path.isdir(ufdbdir + yyyy) != True:
os.mkdir(ufdbdir + yyyy)
fdb.pfdb_write(ufbj_out, filenamej)
print 'Wrote: ' + filenamej
sys.stdout.flush()
files_out = files_out + [filenamej]
#endif
#endfor
#endif
return files_out
def udb_process(ndays=5, day0=None, reprocess=False):
''' This is the main program, that reads the FDB files, decides
what to do, if anything, prcesses a scan, updates the DB and
finishes'''
#initialize output
ufb_1 = []
#first read the FDB files from DPP, and determine if there are any
#files that need to be processed. Here fb are from the DPP, and
#will not be touched. The ifb are the ones that we will work with
#and should have the same number of elements as the fb; both are
#lists
fb, ifb = udb_fb2process(ndays=ndays, day0=day0)
#Check for success
if len(ifb) == 0:
print 'UDB_PROCESS Error: No filedbs input, No processing'
sys.stdout.flush()
else:
#Now process the first scan with any files with pstatus = 0
ifb_1, ss_ifb_1 = udb_scan2process(ifb)
#Only process if you have to
if len(ifb_1) == 0:
print 'UDB_PROCESS: No Scans to process, finished successfully'
sys.stdout.flush()
else:
print 'UDB_PROCESS: Processing Scan: ' + ifb_1[0].scanid
sys.stdout.flush()
ufb_1, bad_filename = udb_process1scan(ifb_1, reprocess=reprocess)
if len(bad_filename) > 0:
ifb_1_list = fdb.fdb_list_fileid(ifb_1)
n1 = len(ifb_1) #change status for bad file to 666
print ifb_1_list
print bad_filename
for j in range(n1):
if ifb_1_list[j] == bad_filename:
ifb_1[j].pstatus = 666
ifb[ss_ifb_1[j]].pstatus = 666
#endif
#endfor
#Update IFDB
ifb_files_out = update_ifdb(ifb)
print "Updated IFDB for Bad filename: "
sys.stdout.flush()
return [
] # ufb_1 #This will probably segmentation fault, but maybe not -- yes, it DOES. Now return empty list
#endif bad filename
#Full database management
if len(ufb_1) == 0:
print 'UDB_PROCESS Error: No output from UDB_PROCESS1SCAN'
else:
#Update ifdb.pstatus for the processed files, output
#idbfits and copy IDB files if necessary
n1 = len(ifb_1)
# have_antlist = False
for j in range(n1):
if ifb_1[j].pstatus != 666: #check as bad files will crash
# if reprocess == False:
if ifb_1[j].pstatus == 0:
print 'UDB_PROCESS: Processing IDB: ' + ifb_1[
j].fileid
process_1ifb(ifb_1[j], reprocess=reprocess)
# fj, antlist = process_1ifb(ifb_1[j], reprocess=reprocess) # process_1ifb() no longer returns anything
# have_antlist = True
# else: #only do this once for antlist
# if ifb_1[j].pstatus == 0: # and have_antlist == False:
# print 'UDB_PROCESS: Processing IDB: '+ifb_1[j].fileid
# fj, antlist = process_1ifb(ifb_1[j], reprocess=reprocess)
# have_antlist = True
ifb_1[j].pstatus = 1
ifb[ss_ifb_1[j]].pstatus = 1
# else:
# antlist = '1 2 3 4 5 6 7 8 9 10 11 12 13 14 15, 16'
#endelse
# print 'UDB_PROCESS: Reset status for:'+ifb[ss_ifb_1[j]].fileid
# print 'UDB_PROCESS: Reset status for:'+ifb_1[j].fileid
#End j loop
#Write out the new ifb entries
ifb_files_out = update_ifdb(ifb)
#here process the UDB spec fits files by spawning an IDL process
# ufb_1a = udb_process_1spec(ufb_1, antlist, reprocess=reprocess)
# if len(ufb_1a) == 0:
# print 'UDB_PROCESS: Problem with Scan to FITS: '+ifb_1[0].scanid
# sys.stdout.flush()
#Write out ufdb entry, ufb_1 is a 1 element list
# ufb_files_out = update_ufdb(ufb_1, ndays=ndays, day0=day0)
# else:
ufb_files_out = update_ufdb(ufb_1, ndays=ndays, day0=day0)
# ufb_files_out = update_ufdb(ufb_1a, ndays=ndays, day0=day0)
print 'UDB_PROCESS: Processed Scan: ' + ifb_1[0].scanid
print 'UDB_PROCESS: finished successfully'
sys.stdout.flush()
#Endelse
#Endelse
#Endelse
#Endelse
return ufb_1
def udb_process1scan(ifb_1, reprocess=False):
'''Given the set of ifb entries, process them into a single UDB
Miriad dataset for this scan'''
n1 = len(ifb_1)
filelist = fdb.fdb_list_fileid(ifb_1)
print filelist
sys.stdout.flush()
#Next, name the output files, this should be based on the start
#time of the first file, changed from using scan_id, 2014-06-17,
#jmm
scan_id = ifb_1[0].scanid
source_id = ifb_1[0].sourceid
project_id = ifb_1[0].projectid
st_ts = ifb_1[0].st_ts
en_ts = ifb_1[n1 - 1].en_ts
#Take care of two digit year here
# ufileid = 'UDB'+'20'+scan_id
ufileid = 'U' + ifb_1[0].fileid[1:]
yyyy = ufileid[3:7]
ufilename = udbdir + yyyy + '/' + ufileid
ufb = fdb.pfiledb(ufileid, scan_id, source_id, project_id, st_ts, en_ts, 0)
# Call udbfile_create for this filelist
filelist_full = []
for j in range(len(filelist)):
#do not process files with pstatus = 666
if ifb_1[j].pstatus != 666:
if reprocess == True:
filelistj = filelist[j]
yyyymmdd = filelistj[len(filelistj) - 14:len(filelistj) - 6]
filelist_full.append(idbfinaldir + yyyymmdd + '/' +
filelist[j])
else:
filelist_full.append(idbdir + filelist[j])
#endelse
#endif
#endfor
n1 = len(filelist_full)
if n1 == 0:
print 'UDB_PROCESS1SCAN: No good files to process'
sys.stdout.flush()
return [], []
#endif
if n1 > 50:
print "UDB_PROCESS1SCAN: Too many Files N1 = ", n1
print "UDB_PROCESS1SCAN: Reset to 50"
sys.stdout.flush()
n1 = 50
filelist_full = filelist_full[0:n1 - 1]
#endif
#If the file exists, you need to delete it
if os.path.isdir(ufilename) == True:
print "UDB_PROCESS1SCAN: dataset: ", ufilename, " will be deleted"
sys.stdout.flush()
shutil.rmtree(ufilename)
#endif
ufile_out, bad_filename = udb_util.udbfile_create(filelist_full, ufilename)
if (len(bad_filename) > 0):
print type(ufile_out)
print "UDB_PROCESS1SCAN: Error creating: ", ufilename
print "UDB_PROCESS1SCAN: Bad Filename: ", bad_filename
sys.stdout.flush()
#strip the path out of the bad_filename here
bbb = bad_filename[len(bad_filename) - 17:]
return ufile_out, bbb
#endif
if (len(ufile_out) == 0):
print "UDB_PROCESS1SCAN: Error creating: ", ufilename
sys.stdout.flush()
return [], []
else:
return [ufb], []
def udb_fb2process(ndays=5, day0=None):
'''Reads in the last N days of FDB files, and IFDB files, finds
the IDB files that need processing, and returns the fdb ifdb
classes for those files'''
#get FDB, IFDB filenames
fdbfiles = udb_init_fdbfiles(ndays=ndays, day0=day0)
if len(fdbfiles) == 0:
print 'UDB_FB2PROCESS Error: No FDB files:'
sys.stdout.flush()
ifboops = []
fboops = []
return fboops, ifboops
#endif
ifdbfiles = udb_init_fdbfiles(ndays=ndays, fdbtype='IFDB', day0=day0)
if len(ifdbfiles) == 0:
# Not necessarily an error
print 'UDB_FB2PROCESS: No IFDB files:'
sys.stdout.flush()
#endif
#read in the files
fb = []
for j in range(len(fdbfiles)):
fbj = fdb.fdb_read(fdbfiles[j])
if len(fbj) > 0:
fb = fb + fbj
#endif
#endfor
ifb = []
if len(ifdbfiles) > 0:
for j in range(len(ifdbfiles)):
ifbj = fdb.pfdb_read(ifdbfiles[j])
if len(ifbj) > 0:
ifb = ifb + ifbj
#endif
#endfor
#endif
#check for non-empty fb's first
if len(fb) == 0:
print "no data to process"
sys.stdout.flush()
otp = []
return fb, otp
#endif
#if I am here, I have IDB files from the FDB. Do I have PFDB entries?
if len(ifb) == 0:
print "no IFDB entries, process all FDB entries"
sys.stdout.flush()
#For each FDB entry, create a IFDB entry, with pstatus = 0
for j in range(len(fb)):
ifbj = fdb.pfiledb(fb[j].fileid, fb[j].scanid, fb[j].sourceid,
fb[j].projectid, fb[j].st_ts, fb[j].en_ts, 0)
ifb.append(ifbj)
#endfor
return fb, ifb
#endif
#if I am here, I have both FDB and IFDB entries, for each FDB,
#check to see if there is a corresponding IFDB. If not, create one
#with pstatus = 0
iflist = fdb.fdb_list_fileid(ifb)
iidarray = array(iflist)
ifb_out = []
for j in range(len(fb)):
fileidj = fb[j].fileid
ifbtemp = extract(iidarray == fileidj, ifb)
if (len(ifbtemp) > 0):
ifb_outj = ifbtemp[0]
else:
ifb_outj = fdb.pfiledb(fb[j].fileid, fb[j].scanid, fb[j].sourceid,
fb[j].projectid, fb[j].st_ts, fb[j].en_ts,
0)
#endif
ifb_out.append(ifb_outj)
#endfor
return fb, ifb_out
def udb_process1scan(ifb_1):
'''Given the set of ifb entries, process
them into a single UDB Miriad dataset for this scan'''
# defines a varplt class for miriad python: class
# TaskVarplt(TaskBase): _keywords =
# ['vis','device','log','xaxis','yaxis','nxy','xrange','yrange']
# _options = ['dtime','compress','overlay','unwrap']
# a call to the varplt task TaskVarplt(vis='temp_pytest',
# xaxis='ut', yaxis='ytsys', device='/xs',
# xrange='0.975,0.977',options='overlay').run()
# Uvaver is the class we'll call here: vis is a set of files, out
# is the output file, line='channel,50,1,10,10' selects 50
# channels, starting with channel 1, summing over 10 channel, then
# skipping to every 10th channel, interval = '0.666667' specifies
# 4 second intervals
# TaskUVAver(vis='/dppdata1/IDB/IDB20131220232416,/dppdata1/IDB/IDB20131220232516',
# out='temp_pytest1',
# line='channel,50,1,10,10',interval='0.066667').run()
# switched to 1 second intervals, jmm, 2014-06-17
#first get the list of input files
n1 = len(ifb_1)
filelist = fdb.fdb_list_fileid(ifb_1)
#Next, name the output files, this should be based on the start
#time of the first file, changed from using scan_id, 2014-06-17,
#jmm
scan_id = ifb_1[0].scanid
source_id = ifb_1[0].sourceid
st_ts = ifb_1[0].st_ts
en_ts = ifb_1[n1 - 1].en_ts
#Take care of two digit year here
# ufileid = 'UDB'+'20'+scan_id
ufileid = 'U' + ifb_1[0].fileid[1:]
yyyy = ufileid[3:7]
ufilename = udbdir + yyyy + '/' + ufileid
ufb = fdb.pfiledb(ufileid, scan_id, source_id, st_ts, en_ts, 0)
#call taskuvaver on the file list, which looks like it needs to be
#a comma-separated string
filelist_str = ''
filelist_str = idbdir + filelist[0]
if n1 > 1:
#just for testing
if n1 > 5:
n1 = 5
#end of just for testing
for j in range(n1 - 1):
filelist_str = filelist_str + ',' + idbdir + filelist[j + 1]
#If the file exists, you need to delete it before uvavering
if os.path.isdir(ufilename) == True:
print "dataset: ", ufilename, " will be deleted"
shutil.rmtree(ufilename)
#Dropped 4 second averaging, 2014-06-17, jmm
TaskUVAver(vis=filelist_str, out=ufilename,
line='channel,50,1,10,10').run()
return [ufb]
