def udb_process1scan(ifb_1, reprocess=False): '''Given the set of ifb entries, process them into a single UDB Miriad dataset for this scan''' n1 = len(ifb_1) filelist = fdb.fdb_list_fileid(ifb_1) print filelist #Next, name the output files, this should be based on the start #time of the first file, changed from using scan_id, 2014-06-17, #jmm scan_id = ifb_1[0].scanid source_id = ifb_1[0].sourceid project_id = ifb_1[0].projectid st_ts = ifb_1[0].st_ts en_ts = ifb_1[n1 - 1].en_ts #Take care of two digit year here # ufileid = 'UDB'+'20'+scan_id ufileid = 'U' + ifb_1[0].fileid[1:] yyyy = ufileid[3:7] ufilename = udbdir + yyyy + '/' + ufileid ufb = fdb.pfiledb(ufileid, scan_id, source_id, project_id, st_ts, en_ts, 0) # check for reprocess if reprocess == True: idb_datadir = idbdir else: idb_datadir = idbfinaldir + yyyy + '/' #endelse # Call udbfile_create for this filelist filelist_full = filelist for j in range(len(filelist)): filelist_full[j] = idb_datadir + filelist[j] #endfor if n1 > 50: print "UDB_PROCESS1SCAN: Too many Files N1 = ", n1 print "UDB_PROCESS1SCAN: Reset to 50" n1 = 50 filelist_full = filelist_full[0:n1 - 1] #endif #If the file exists, you need to delete it if os.path.isdir(ufilename) == True: print "UDB_PROCESS1SCAN: dataset: ", ufilename, " will be deleted" shutil.rmtree(ufilename) #endif ufile_out = udb_util.udbfile_create(filelist_full, ufilename) if (len(ufile_out) == 0): print "UDB_PROCESS1SCAN: Error creating: ", ufilename return [] else: return [ufb]
def update_ifdb(ifb): ''' Updates ifdb files, Here we just write out the files for the previous ndays days''' files_out = [] #Get days for each ifb entry ifdays = fdb.fdb_list_fileid(ifb) # 'FDByyyymmddhhmmss' extract subscript range 3 to 11 to get the date # 'IFDByyyymmddhhmmss' extract subscript range 4 to 12 to get the date for j in range(len(ifdays)): y = str.find(ifdays[j], 'DB') x0 = y + 2 x1 = y + 10 ifdays[j] = ifdays[j][x0:x1] #endfor #Get the filenames for the files, days = fdb.fdb_uniq_day(ifb) ndays_out = len(days) if ndays_out == 0: print 'UPDATE_IFDB Error: No days to process' sys.stdout.flush() else: #need an array of days for extract commands idyarr = array(ifdays) for j in range(ndays_out): #Find files with dates matching each day here, build a #list to output ifbj = extract(idyarr == days[j], ifb) ifbj_out = ifbj.tolist() if len(ifbj_out) == 0: print 'UPDATE_IFDB Error: No output for: ' + days[j] sys.stdout.flush() else: #filename yyyy = days[j] yyyy = yyyy[:4] filenamej = ifdbdir + '/' + yyyy + '/' + 'IFDB' + days[ j] + '.txt' #check for yyyy directory if os.path.isdir(ifdbdir + yyyy) != True: os.mkdir(ifdbdir + yyyy) fdb.pfdb_write(ifbj_out, filenamej) print 'Wrote: ' + filenamej sys.stdout.flush() files_out = files_out + [filenamej] #endif #endfor #endif return files_out
def update_ufdb(ufb, ndays=5, day0=None): '''Updates ufdb files, need to read the last ndays files, check to see if you're overwriting an entry, if not then append the new scan to the list and output. If so, replace the old entry, then output. Note that ufb is a 1-element list''' if len(ufb) == 0: print 'UPDATE_UFDB ERROR: No UFB input' sys.stdout.flush() files_out = [] #get filenames ufdb_files = udb_init_fdbfiles(ndays=ndays, fdbtype='UFDB', day0=day0) ufb_in = [] if len(ufdb_files) == 0: #maybe there are none print 'UPDATE_UFDB: No UFDB files in range:' sys.stdout.flush() else: for j in range(len(ufdb_files)): ufbj = fdb.pfdb_read(ufdb_files[j]) if len(ufbj) > 0: ufb_in = ufb_in + ufbj #endif #endfor #endif # If there is no ufb_in, then it's easy if len(ufb_in) == 0: ufb_out = ufb else: # Check to see if there is an entry for the current ufb uflist_in = fdb.fdb_list_fileid(ufb_in) ufarray = array(uflist_in) # return the subscript ss_ufb_in = extract(ufarray == ufb[0].fileid, array(range(len(ufb_in)))) #I think ss_ufb_in should be a scalar.. if len(ss_ufb_in) == 0: ufb_out = ufb_in + ufb else: # Get all of the entries that don't have this fileid ss_ufb_not = extract(ufarray != ufb[0].fileid, array(range(len(ufb_in)))) if len(ss_ufb_not) == 0: ufb_out = ufb else: ufb_out = [] for j in range(len(ss_ufb_not)): ufb_out = ufb_out + [ufb_in[ss_ufb_not[j]]] #endfor ufb_out = ufb_out + ufb #endif #endif #endif #Here, now output the ufb_out entries files_out = [] #Get days for each ufb entry ufdays = fdb.fdb_list_fileid(ufb_out) for j in range(len(ufdays)): y = str.find(ufdays[j], 'DB') x0 = y + 2 x1 = y + 10 ufdays[j] = ufdays[j][x0:x1] #endfor #Get the filenames for the files, days_out = fdb.fdb_uniq_day(ufb_out) #Adjustment due to lack ndays_out = len(days_out) if ndays_out == 0: print 'UPDATE_UFDB Error: No days to process' sys.stdout.flush() else: #need an array of days for extract commands udyarr = array(ufdays) for j in range(ndays_out): #Find files with dates matching each day here, build a #list to output ufbj = extract(udyarr == days_out[j], ufb_out) ufbj_out = ufbj.tolist() if len(ufbj_out) == 0: print 'UPDATE_UFDB Error: No output for: ' + days[j] sys.stdout.flush() else: #filename yyyy = days_out[j] yyyy = yyyy[:4] filenamej = ufdbdir + '/' + yyyy + '/' + 'UFDB' + days_out[ j] + '.txt' #check for yyyy if os.path.isdir(ufdbdir + yyyy) != True: os.mkdir(ufdbdir + yyyy) fdb.pfdb_write(ufbj_out, filenamej) print 'Wrote: ' + filenamej sys.stdout.flush() files_out = files_out + [filenamej] #endif #endfor #endif return files_out
def udb_process(ndays=5, day0=None, reprocess=False): ''' This is the main program, that reads the FDB files, decides what to do, if anything, prcesses a scan, updates the DB and finishes''' #initialize output ufb_1 = [] #first read the FDB files from DPP, and determine if there are any #files that need to be processed. Here fb are from the DPP, and #will not be touched. The ifb are the ones that we will work with #and should have the same number of elements as the fb; both are #lists fb, ifb = udb_fb2process(ndays=ndays, day0=day0) #Check for success if len(ifb) == 0: print 'UDB_PROCESS Error: No filedbs input, No processing' sys.stdout.flush() else: #Now process the first scan with any files with pstatus = 0 ifb_1, ss_ifb_1 = udb_scan2process(ifb) #Only process if you have to if len(ifb_1) == 0: print 'UDB_PROCESS: No Scans to process, finished successfully' sys.stdout.flush() else: print 'UDB_PROCESS: Processing Scan: ' + ifb_1[0].scanid sys.stdout.flush() ufb_1, bad_filename = udb_process1scan(ifb_1, reprocess=reprocess) if len(bad_filename) > 0: ifb_1_list = fdb.fdb_list_fileid(ifb_1) n1 = len(ifb_1) #change status for bad file to 666 print ifb_1_list print bad_filename for j in range(n1): if ifb_1_list[j] == bad_filename: ifb_1[j].pstatus = 666 ifb[ss_ifb_1[j]].pstatus = 666 #endif #endfor #Update IFDB ifb_files_out = update_ifdb(ifb) print "Updated IFDB for Bad filename: " sys.stdout.flush() return [ ] # ufb_1 #This will probably segmentation fault, but maybe not -- yes, it DOES. Now return empty list #endif bad filename #Full database management if len(ufb_1) == 0: print 'UDB_PROCESS Error: No output from UDB_PROCESS1SCAN' else: #Update ifdb.pstatus for the processed files, output #idbfits and copy IDB files if necessary n1 = len(ifb_1) # have_antlist = False for j in range(n1): if ifb_1[j].pstatus != 666: #check as bad files will crash # if reprocess == False: if ifb_1[j].pstatus == 0: print 'UDB_PROCESS: Processing IDB: ' + ifb_1[ j].fileid process_1ifb(ifb_1[j], reprocess=reprocess) # fj, antlist = process_1ifb(ifb_1[j], reprocess=reprocess) # process_1ifb() no longer returns anything # have_antlist = True # else: #only do this once for antlist # if ifb_1[j].pstatus == 0: # and have_antlist == False: # print 'UDB_PROCESS: Processing IDB: '+ifb_1[j].fileid # fj, antlist = process_1ifb(ifb_1[j], reprocess=reprocess) # have_antlist = True ifb_1[j].pstatus = 1 ifb[ss_ifb_1[j]].pstatus = 1 # else: # antlist = '1 2 3 4 5 6 7 8 9 10 11 12 13 14 15, 16' #endelse # print 'UDB_PROCESS: Reset status for:'+ifb[ss_ifb_1[j]].fileid # print 'UDB_PROCESS: Reset status for:'+ifb_1[j].fileid #End j loop #Write out the new ifb entries ifb_files_out = update_ifdb(ifb) #here process the UDB spec fits files by spawning an IDL process # ufb_1a = udb_process_1spec(ufb_1, antlist, reprocess=reprocess) # if len(ufb_1a) == 0: # print 'UDB_PROCESS: Problem with Scan to FITS: '+ifb_1[0].scanid # sys.stdout.flush() #Write out ufdb entry, ufb_1 is a 1 element list # ufb_files_out = update_ufdb(ufb_1, ndays=ndays, day0=day0) # else: ufb_files_out = update_ufdb(ufb_1, ndays=ndays, day0=day0) # ufb_files_out = update_ufdb(ufb_1a, ndays=ndays, day0=day0) print 'UDB_PROCESS: Processed Scan: ' + ifb_1[0].scanid print 'UDB_PROCESS: finished successfully' sys.stdout.flush() #Endelse #Endelse #Endelse #Endelse return ufb_1
def udb_process1scan(ifb_1, reprocess=False): '''Given the set of ifb entries, process them into a single UDB Miriad dataset for this scan''' n1 = len(ifb_1) filelist = fdb.fdb_list_fileid(ifb_1) print filelist sys.stdout.flush() #Next, name the output files, this should be based on the start #time of the first file, changed from using scan_id, 2014-06-17, #jmm scan_id = ifb_1[0].scanid source_id = ifb_1[0].sourceid project_id = ifb_1[0].projectid st_ts = ifb_1[0].st_ts en_ts = ifb_1[n1 - 1].en_ts #Take care of two digit year here # ufileid = 'UDB'+'20'+scan_id ufileid = 'U' + ifb_1[0].fileid[1:] yyyy = ufileid[3:7] ufilename = udbdir + yyyy + '/' + ufileid ufb = fdb.pfiledb(ufileid, scan_id, source_id, project_id, st_ts, en_ts, 0) # Call udbfile_create for this filelist filelist_full = [] for j in range(len(filelist)): #do not process files with pstatus = 666 if ifb_1[j].pstatus != 666: if reprocess == True: filelistj = filelist[j] yyyymmdd = filelistj[len(filelistj) - 14:len(filelistj) - 6] filelist_full.append(idbfinaldir + yyyymmdd + '/' + filelist[j]) else: filelist_full.append(idbdir + filelist[j]) #endelse #endif #endfor n1 = len(filelist_full) if n1 == 0: print 'UDB_PROCESS1SCAN: No good files to process' sys.stdout.flush() return [], [] #endif if n1 > 50: print "UDB_PROCESS1SCAN: Too many Files N1 = ", n1 print "UDB_PROCESS1SCAN: Reset to 50" sys.stdout.flush() n1 = 50 filelist_full = filelist_full[0:n1 - 1] #endif #If the file exists, you need to delete it if os.path.isdir(ufilename) == True: print "UDB_PROCESS1SCAN: dataset: ", ufilename, " will be deleted" sys.stdout.flush() shutil.rmtree(ufilename) #endif ufile_out, bad_filename = udb_util.udbfile_create(filelist_full, ufilename) if (len(bad_filename) > 0): print type(ufile_out) print "UDB_PROCESS1SCAN: Error creating: ", ufilename print "UDB_PROCESS1SCAN: Bad Filename: ", bad_filename sys.stdout.flush() #strip the path out of the bad_filename here bbb = bad_filename[len(bad_filename) - 17:] return ufile_out, bbb #endif if (len(ufile_out) == 0): print "UDB_PROCESS1SCAN: Error creating: ", ufilename sys.stdout.flush() return [], [] else: return [ufb], []
def udb_fb2process(ndays=5, day0=None): '''Reads in the last N days of FDB files, and IFDB files, finds the IDB files that need processing, and returns the fdb ifdb classes for those files''' #get FDB, IFDB filenames fdbfiles = udb_init_fdbfiles(ndays=ndays, day0=day0) if len(fdbfiles) == 0: print 'UDB_FB2PROCESS Error: No FDB files:' sys.stdout.flush() ifboops = [] fboops = [] return fboops, ifboops #endif ifdbfiles = udb_init_fdbfiles(ndays=ndays, fdbtype='IFDB', day0=day0) if len(ifdbfiles) == 0: # Not necessarily an error print 'UDB_FB2PROCESS: No IFDB files:' sys.stdout.flush() #endif #read in the files fb = [] for j in range(len(fdbfiles)): fbj = fdb.fdb_read(fdbfiles[j]) if len(fbj) > 0: fb = fb + fbj #endif #endfor ifb = [] if len(ifdbfiles) > 0: for j in range(len(ifdbfiles)): ifbj = fdb.pfdb_read(ifdbfiles[j]) if len(ifbj) > 0: ifb = ifb + ifbj #endif #endfor #endif #check for non-empty fb's first if len(fb) == 0: print "no data to process" sys.stdout.flush() otp = [] return fb, otp #endif #if I am here, I have IDB files from the FDB. Do I have PFDB entries? if len(ifb) == 0: print "no IFDB entries, process all FDB entries" sys.stdout.flush() #For each FDB entry, create a IFDB entry, with pstatus = 0 for j in range(len(fb)): ifbj = fdb.pfiledb(fb[j].fileid, fb[j].scanid, fb[j].sourceid, fb[j].projectid, fb[j].st_ts, fb[j].en_ts, 0) ifb.append(ifbj) #endfor return fb, ifb #endif #if I am here, I have both FDB and IFDB entries, for each FDB, #check to see if there is a corresponding IFDB. If not, create one #with pstatus = 0 iflist = fdb.fdb_list_fileid(ifb) iidarray = array(iflist) ifb_out = [] for j in range(len(fb)): fileidj = fb[j].fileid ifbtemp = extract(iidarray == fileidj, ifb) if (len(ifbtemp) > 0): ifb_outj = ifbtemp[0] else: ifb_outj = fdb.pfiledb(fb[j].fileid, fb[j].scanid, fb[j].sourceid, fb[j].projectid, fb[j].st_ts, fb[j].en_ts, 0) #endif ifb_out.append(ifb_outj) #endfor return fb, ifb_out
def udb_process1scan(ifb_1): '''Given the set of ifb entries, process them into a single UDB Miriad dataset for this scan''' # defines a varplt class for miriad python: class # TaskVarplt(TaskBase): _keywords = # ['vis','device','log','xaxis','yaxis','nxy','xrange','yrange'] # _options = ['dtime','compress','overlay','unwrap'] # a call to the varplt task TaskVarplt(vis='temp_pytest', # xaxis='ut', yaxis='ytsys', device='/xs', # xrange='0.975,0.977',options='overlay').run() # Uvaver is the class we'll call here: vis is a set of files, out # is the output file, line='channel,50,1,10,10' selects 50 # channels, starting with channel 1, summing over 10 channel, then # skipping to every 10th channel, interval = '0.666667' specifies # 4 second intervals # TaskUVAver(vis='/dppdata1/IDB/IDB20131220232416,/dppdata1/IDB/IDB20131220232516', # out='temp_pytest1', # line='channel,50,1,10,10',interval='0.066667').run() # switched to 1 second intervals, jmm, 2014-06-17 #first get the list of input files n1 = len(ifb_1) filelist = fdb.fdb_list_fileid(ifb_1) #Next, name the output files, this should be based on the start #time of the first file, changed from using scan_id, 2014-06-17, #jmm scan_id = ifb_1[0].scanid source_id = ifb_1[0].sourceid st_ts = ifb_1[0].st_ts en_ts = ifb_1[n1 - 1].en_ts #Take care of two digit year here # ufileid = 'UDB'+'20'+scan_id ufileid = 'U' + ifb_1[0].fileid[1:] yyyy = ufileid[3:7] ufilename = udbdir + yyyy + '/' + ufileid ufb = fdb.pfiledb(ufileid, scan_id, source_id, st_ts, en_ts, 0) #call taskuvaver on the file list, which looks like it needs to be #a comma-separated string filelist_str = '' filelist_str = idbdir + filelist[0] if n1 > 1: #just for testing if n1 > 5: n1 = 5 #end of just for testing for j in range(n1 - 1): filelist_str = filelist_str + ',' + idbdir + filelist[j + 1] #If the file exists, you need to delete it before uvavering if os.path.isdir(ufilename) == True: print "dataset: ", ufilename, " will be deleted" shutil.rmtree(ufilename) #Dropped 4 second averaging, 2014-06-17, jmm TaskUVAver(vis=filelist_str, out=ufilename, line='channel,50,1,10,10').run() return [ufb]