def find_gaincal(t=None, scan_length=6, findwhat='FEATTNTEST'): # This will find the project on the day of "t" or, going backwards, the nearest # day to "t" # This routine returns a timerange that may be used as the timerange in any # of the above programs. It will return the appropriate timerange for a # FEATTNTEST scan on the date for which t is given to this function, or a # list of appropriate time range. If the length of the scan is not 6 minute, # please set scan_length to the appropriate length. ''' Makes an SQL query to find the FEATTNTEST or GAINCALTESTscans for the date given in the Time() object t. A list of timestamps is returned, along with the timestamp of the object provided. ''' loop_ = 1 if t is None: # Get today's date t = util.Time.now() timestamp = int(t.lv) while loop_ == 1: stimestamp = timestamp - (timestamp % 86400) # Start of day etimestamp = stimestamp + 86399 # End of day # Open handle to SQL database cursor = dbutil.get_cursor() # Try to find a scan header with project SOLPNTCAL (only works after 2014 Nov. 30) verstr = dbutil.find_table_version(cursor,timestamp,True) if verstr is None: print 'No scan_header table found for given time.' return [], timestamp # First retrieve the Project from all scan headers for the day cursor.execute('select timestamp,Project from hV'+verstr+'_vD1 where timestamp between '+str(stimestamp)+' and '+str(etimestamp)+' order by timestamp') data = np.transpose(np.array(cursor.fetchall())) names = stateframedef.numpy.array(cursor.description)[:,0] cursor.close() if len(data) == 0: # No FEATTNTEST found, so return empty list (and timestamp) return [], timestamp else: projdict = dict(zip(names,data)) projdict['timestamp'] = projdict['timestamp'].astype('float') # Convert timestamps from string to float good = np.where(projdict['Project'] == findwhat)[0] if len(good) != 0: if len(good) == 1: loop_ = 0 tgc = [projdict['timestamp'][good]], timestamp start_= Time(tgc[0][0], format = 'lv').iso end_ = Time(tgc[0][0]+60*scan_length, format = 'lv').iso trange = Time([start_[0] , end_[0]]) return trange else: loop_ = 0 tgc = [projdict['timestamp'][good]], timestamp start_trange = Time(tgc[0][0], format = 'lv').iso end_trange = Time(tgc[0][0]+60*scan_length, format = 'lv').iso tranges = [] for i in range(start_trange.shape[0]): trange = Time([start_trange[i], end_trange[i]]) tranges.append(trange) return tranges else: timestamp = timestamp - 60*60*24 loop_ = 1
def get_projects(t, nosql=False): ''' Read all projects from SQL for the current date and return a summary as a dictionary with keys Timestamp, Project, and EOS (another timestamp) ''' if nosql == True: return get_projects_nosql(t) import dbutil # timerange is 12 UT to 12 UT on next day, relative to the day in Time() object t trange = Time([int(t.mjd) + 12. / 24, int(t.mjd) + 36. / 24], format='mjd') tstart, tend = trange.lv.astype('str') cursor = dbutil.get_cursor() mjd = t.mjd # Get the project IDs for scans during the period verstrh = dbutil.find_table_version(cursor, trange[0].lv, True) if verstrh is None: print 'No scan_header table found for given time.' return {} query = 'select Timestamp,Project from hV' + verstrh + '_vD1 where Timestamp between ' + tstart + ' and ' + tend + ' order by Timestamp' projdict, msg = dbutil.do_query(cursor, query) if msg != 'Success': print msg return {} elif len(projdict) == 0: # No Project ID found, so return data and empty projdict dictionary print 'SQL Query was valid, but no Project data were found.' return {} projdict['Timestamp'] = projdict['Timestamp'].astype( 'float') # Convert timestamps from string to float for i in range(len(projdict['Project'])): projdict['Project'][i] = projdict['Project'][i].replace('\x00', '') projdict.update({'EOS': projdict['Timestamp'][1:]}) projdict.update({'Timestamp': projdict['Timestamp'][:-1]}) projdict.update({'Project': projdict['Project'][:-1]}) cursor.close() return projdict
def get_sql_info(trange): ''' Get all antenna information from the SQL database for a given timerange, including TrackFlag and Parallactic Angle Also determines if the RFSwitch state (i.e. which 27-m receiver is being used). ''' cursor = db.get_cursor() sqldict = db.get_dbrecs(cursor, dimension=15, timestamp=trange) azeldict = stateframe.azel_from_sqldict(sqldict) time = Time(sqldict['Timestamp'][:, 0].astype(int), format='lv') azeldict.update({'Time': time}) sqldict = db.get_dbrecs(cursor, dimension=1, timestamp=trange) azeldict.update({'RFSwitch': sqldict['FEMA_Powe_RFSwitchStatus']}) azeldict.update({'LF_Rcvr': sqldict['FEMA_Rece_LoFreqEnabled']}) if np.median(azeldict['RFSwitch']) == 0.0 and np.median( azeldict['LF_Rcvr']) == 1.0: azeldict.update({'Receiver': 'Low'}) elif np.median(azeldict['RFSwitch']) == 1.0 and np.median( azeldict['LF_Rcvr']) == 0.0: azeldict.update({'Receiver': 'High'}) else: azeldict.update({'Receiver': 'Unknown'}) cursor.close() return azeldict
def read_cal(type,t=None): ''' Read the calibration data of the given type, for the given time (as a Time() object), or for the current time if None. Returns a dictionary of look-up information and a binary buffer containing the calibration record. ''' import dbutil, read_xml2, sys if t is None: t = util.Time.now() timestamp = int(t.lv) # Given (or current) time as LabVIEW timestamp xmldict, ver = read_cal_xml(type, t) cursor = dbutil.get_cursor() if xmldict != {}: query = 'set textsize 2147483647 select top 1 * from abin where Version = '+str(type+ver/10.)+' and Timestamp <= '+str(timestamp)+' order by Timestamp desc' sqldict, msg = dbutil.do_query(cursor,query) cursor.close() if msg == 'Success': if sqldict == {}: print 'Error: Query returned no records.' print query return {}, None buf = sqldict['Bin'][0] # Binary representation of data return xmldict, str(buf) else: print 'Unknown error occurred reading',typdict[type][0] print sys.exc_info()[1] return {}, None else: return {}, None
def get_sql_info(trange): ''' Get all antenna information from the SQL database for a given timerange, including TrackFlag and Parallactic Angle ''' cursor = db.get_cursor() sqldict = db.get_dbrecs(cursor, dimension=15, timestamp=trange) azeldict = stateframe.azel_from_sqldict(sqldict) time = Time(sqldict['Timestamp'][:, 0].astype(int), format='lv') azeldict.update({'Time': time}) cursor.close() return azeldict
def get_sql_info(trange): ''' Get all antenna information from the SQL database for a given timerange, including TrackFlag and Parallactic Angle ''' cursor = db.get_cursor() sqldict = db.get_dbrecs(cursor,dimension=15,timestamp=trange) azeldict = stateframe.azel_from_sqldict(sqldict) time = Time(sqldict['Timestamp'][:,0].astype(int),format='lv') azeldict.update({'Time':time}) cursor.close() return azeldict
def write_cal(type,buf,t=None): ''' Write the calibration data provided in data buffer buf of the given type, for the given time (as a Time() object), or for the current time if None. Typcially, the time should refer to when the calibration data were taken, so the correct time object should be provided. The type keyword is a real number whose integer part indicates the type definition. The fractional part must not be 0 (since this would indicate a type definition record rather than a data record). The relevant type definition is read from the database, and its total size is determined and compared with the buffer size, as a sanity check. Returns True if success, or False if failure. ''' import dbutil, read_xml2, sys if t is None: t = util.Time.now() timestamp = int(t.lv) # Given (or current) time as LabVIEW timestamp typdict = cal_types() try: typinfo = typdict[int(type)] except: print 'Type',int(type),'not found in type definition dictionary.' return False cursor = dbutil.get_cursor() # Read type definition XML from abin table and do a sanity check query = 'select top 1 * from abin where Version = '+str(int(type))+'.0 and Timestamp <='+str(timestamp)+' order by Timestamp desc' outdict, msg = dbutil.do_query(cursor,query) if msg == 'Success': if len(outdict) == 0: # This type of xml file does not yet exist in the database, so indicate an error print 'Error: Type',type,'not defined in abin table.' cursor.close() return False else: # There is one, so read it and do a sanity check against binary data f = open('/tmp/tmp.xml','wb') f.write(outdict['Bin'][0]) f.close() keys, mydict, fmt, ver = read_xml2.xml_read('/tmp/tmp.xml') binsize = get_size(fmt) if len(buf) == binsize: cursor.execute('insert into aBin (Timestamp,Version,Description,Bin) values (?,?,?,?)', timestamp, type+ver/10., typinfo[0], dbutil.stateframedef.pyodbc.Binary(buf)) cursor.commit() cursor.close() return True else: print 'Error: Size of buffer',len(buf),'does not match this calibration type. Expecting',binsize cursor.close() return False
def flare_monitor(t): ''' Get all front-end power-detector voltages for the given day from the stateframe SQL database, and obtain the median of them, to use as a flare monitor. Returns ut times in plot_date format and median voltages. ''' import dbutil # timerange is 12 UT to 12 UT on next day, relative to the day in Time() object t trange = Time([int(t.mjd) + 12. / 24, int(t.mjd) + 36. / 24], format='mjd') tstart, tend = trange.lv.astype('str') cursor = dbutil.get_cursor() mjd = t.mjd verstr = dbutil.find_table_version(cursor, tstart) if verstr is None: print 'No stateframe table found for given time.' return tstart, [], {} query = 'select Timestamp,Ante_Fron_FEM_HPol_Voltage,Ante_Fron_FEM_VPol_Voltage from fV' + verstr + '_vD15 where timestamp between ' + tstart + ' and ' + tend + ' order by timestamp' data, msg = dbutil.do_query(cursor, query) if msg != 'Success': print msg return tstart, [], {} for k, v in data.items(): data[k].shape = (len(data[k]) / 15, 15) hv = [] try: ut = Time(data['Timestamp'][:, 0].astype('float'), format='lv').plot_date except: print 'Error for time', t.iso print 'Query:', query, ' returned msg:', msg print 'Keys:', data.keys() print data['Timestamp'][0, 0] hfac = np.median(data['Ante_Fron_FEM_HPol_Voltage'].astype('float'), 0) vfac = np.median(data['Ante_Fron_FEM_VPol_Voltage'].astype('float'), 0) for i in range(4): if hfac[i] > 0: hv.append(data['Ante_Fron_FEM_HPol_Voltage'][:, i] / hfac[i]) if vfac[i] > 0: hv.append(data['Ante_Fron_FEM_VPol_Voltage'][:, i] / vfac[i]) #import pdb; pdb.set_trace() flm = np.median(np.array(hv), 0) good = np.where(abs(flm[1:] - flm[:-1]) < 0.1)[0] projdict = get_projects(t) return ut[good], flm[good], projdict
def drop_deftable(version): ''' Drops ALL traces of a given version of a stateframe definition. Use with CAUTION!!! Requests confirmation from the keyboard. ''' import dbutil as db cursor = db.get_cursor() # Get all table and view names from this version query = 'select * from information_schema.tables where table_name like "fV'+str(int(version))+'%"' result, msg = db.do_query(cursor, query) if msg == 'Success': names = result['TABLE_NAME'] print 'You are about to permanently delete the following tables:' for name in names: print ' ',name ans = raw_input('Are you sure? [y/n]: ').upper() if ans != 'Y': print 'Action canceled by user' return False # Delete the version from the stateframedef table query = 'delete from StateFrameDef where Version='+str(int(version)) r, msg = db.do_query(cursor, query) if msg != 'Success': print 'Error, could not delete from stateframedef for version:',version print msg return False # Loop over table and view names, dropping each in turn for name in names: query = 'drop table '+name r, msg = db.do_query(cursor, query) if msg != 'Success': print 'Error dropping table',name print msg # Drop Bin Reader procedure query = 'drop proc ov_fBinReader_V'+str(int(version)) r, msg = db.do_query(cursor, query) if msg != 'Success': print 'Error, could not delete Bin Reader procedure for version:',version print msg return False else: return False print 'Successfully dropped all existing tables and table definition for version', version return True
def get_median_wind(wthr): ''' Temporary work-around for mis-behaving weather station. Given the weather dictionary, query the SQL database for the last 120-s of wind data, and calculate median rather than average. I hope this does not take too long! Returns the same dictionary, with median replacing average wind. ''' import dbutil as db cursor = db.get_cursor() query = 'select top 120 Timestamp,Sche_Data_Weat_Wind from fV66_vD1 order by Timestamp desc' data, msg = db.do_query(cursor,query) if msg == 'Success': try: medwind = np.median(data['Sche_Data_Weat_Wind']) wthr.update({'mt2MinRollAvgWindSpeed': medwind}) except: pass cursor.close() return wthr
def get_fseqfile(t=None): if t is None: # 10 s ago... tlv = Time.now().lv - 10 else: tlv = int(t.lv) cursor = db.get_cursor() ver = db.find_table_version(cursor,tlv) # Get front end attenuator states query = 'select Timestamp,LODM_LO1A_FSeqFile from fV'+ver+'_vD1 where Timestamp between '+str(tlv)+' and '+str(tlv+1)+' order by Timestamp' data, msg = db.do_query(cursor, query) if msg == 'Success': fseqfile = data['LODM_LO1A_FSeqFile'][0].replace('\x00','') if fseqfile == 'none': fseqfile = None else: print 'Error: ',msg fseqfile = None cursor.close() return fseqfile
def find_solpnt(t=None): ''' Makes an SQL query to find the SOLPNTCAL scans for the date given in the Time() object t. A list of timestamps is returned, along with the timestamp of the object provided. ''' import dbutil if t is None: # Get today's date t = util.Time.now() timestamp = int(t.lv) stimestamp = timestamp - (timestamp % 86400) # Start of day etimestamp = stimestamp + 86399 # End of day # Open handle to SQL database cursor = dbutil.get_cursor() # Try to find a scan header with project SOLPNTCAL (only works after 2014 Nov. 30) verstr = dbutil.find_table_version(cursor, timestamp, True) if verstr is None: print 'No scan_header table found for given time.' return [], timestamp # First retrieve the Project from all scan headers for the day cursor.execute('select timestamp,Project from hV' + verstr + '_vD1 where timestamp between ' + str(stimestamp) + ' and ' + str(etimestamp) + ' order by timestamp') data = np.transpose(np.array(cursor.fetchall())) names = stateframedef.numpy.array(cursor.description)[:, 0] cursor.close() if len(data) == 0: # No SOLPNTCAL found, so return empty list (and timestamp) return [], timestamp else: projdict = dict(zip(names, data)) projdict['timestamp'] = projdict['timestamp'].astype( 'float') # Convert timestamps from string to float good = np.where(projdict['Project'] == 'SOLPNTCAL')[0] if len(good) != 0: if len(good) == 1: return [projdict['timestamp'][good]], timestamp else: return projdict['timestamp'][good], timestamp else: return [], timestamp
def findscans(trange): '''Identify phasecal scans from UFDB files ''' import dbutil import dump_tsys tstart, tend = trange.lv.astype(int).astype(str) cursor = dbutil.get_cursor() verstr = dbutil.find_table_version(cursor, tstart, True) query = 'select Timestamp,Project,SourceID from hV'+verstr+'_vD1 where left(Project,8) = "PHASECAL" and Timestamp between '+tstart+' and '+tend+' order by Timestamp' projdict, msg = dbutil.do_query(cursor, query) if msg != 'Success': return {'msg':msg} if projdict == {}: return {'msg':'No PHASECAL scans for this day'} tsint = projdict['Timestamp'].astype(int) # Check UFDB file to get duration ufdb = dump_tsys.rd_ufdb(Time(int(tstart),format='lv')) mjd0 = int(Time(int(tstart),format='lv').mjd) mjdnow = int(Time.now().mjd) if mjd0 < mjdnow: # The date is a previous day, so read a second ufdb file # to ensure we have the whole local day try: ufdb2 = dump_tsys.rd_ufdb(Time(int(tstart)+86400.,format='lv')) for key in ufdb.keys(): ufdb.update({key: np.append(ufdb[key], ufdb2[key])}) except: # No previous day, so just skip it. pass ufdb_times = ufdb['ST_TS'].astype(float).astype(int) idx = nearest_val_idx(tsint,ufdb_times) fpath = '/data1/eovsa/fits/UDB/' + trange[0].iso[:4] + '/' dur = [] file = [] for i in idx: dur.append(((ufdb['EN_TS'].astype(float) - ufdb['ST_TS'].astype(float))[i])/60.) file.append(fpath+ufdb['FILE'][i]) # Fix source ID to remove nulls srclist = np.array([str(i.replace('\x00','')) for i in projdict['SourceID']]) return {'Timestamp': tsint, 'SourceID': srclist, 'duration': np.array(dur), 'filelist':np.array(file), 'msg': msg}
def find_solpnt(t=None): ''' Makes an SQL query to find the SOLPNTCAL scans for the date given in the Time() object t. A list of timestamps is returned, along with the timestamp of the object provided. ''' import dbutil if t is None: # Get today's date t = util.Time.now() timestamp = int(t.lv) stimestamp = timestamp - (timestamp % 86400) # Start of day etimestamp = stimestamp + 86399 # End of day # Open handle to SQL database cursor = dbutil.get_cursor() # Try to find a scan header with project SOLPNTCAL (only works after 2014 Nov. 30) verstr = dbutil.find_table_version(cursor,timestamp,True) if verstr is None: print 'No scan_header table found for given time.' return [], timestamp # First retrieve the Project from all scan headers for the day cursor.execute('select timestamp,Project from hV'+verstr+'_vD1 where timestamp between '+str(stimestamp)+' and '+str(etimestamp)+' order by timestamp') data = np.transpose(np.array(cursor.fetchall())) names = stateframedef.numpy.array(cursor.description)[:,0] cursor.close() if len(data) == 0: # No SOLPNTCAL found, so return empty list (and timestamp) return [], timestamp else: projdict = dict(zip(names,data)) projdict['timestamp'] = projdict['timestamp'].astype('float') # Convert timestamps from string to float good = np.where(projdict['Project'] == 'SOLPNTCAL')[0] if len(good) != 0: if len(good) == 1: return [projdict['timestamp'][good]], timestamp else: return projdict['timestamp'][good], timestamp else: return [], timestamp
def read_cal_xml(type,t=None): ''' Read the calibration type definition xml record of the given type, for the given time (as a Time() object), or for the current time if None. Returns a dictionary of look-up information and its internal version. A side-effect is that a file /tmp/type<n>.xml is created, where <n> is the type. ''' import dbutil, read_xml2, sys if t is None: t = util.Time.now() timestamp = int(t.lv) # Given (or current) time as LabVIEW timestamp typdict = cal_types() try: typinfo = typdict[type] except: print 'Type',type,'not found in type definition dictionary.' return {}, None cursor = dbutil.get_cursor() # Read type definition XML from abin table query = 'select top 1 * from abin where Version = '+str(type)+'.0 and Timestamp <='+str(timestamp)+' order by Timestamp desc' sqldict, msg = dbutil.do_query(cursor,query) if msg == 'Success': if len(sqldict) == 0: # This type of xml file does not yet exist in the database, so mark it for adding print 'Type',type,'not defined in abin table.' cursor.close() return {}, None else: # There is one, so read it and the corresponding binary data buf = sqldict['Bin'][0] # Binary representation of xml file xmlfile = '/tmp/type'+str(type)+'.xml' f = open(xmlfile,'wb') f.write(buf) f.close() xmldict, thisver = read_xml2.xml_ptrs(xmlfile) cursor.close() return xmldict, thisver
def get_solpnt(t=None): ''' Get the SOLPNT data from the SQL database, occurring after time given in the Time() object t. If omitted, the first SOLPNT scan for the current day is used (if it exists). ''' import dbutil tstamps, timestamp = find_solpnt(t) # Find first SOLPNTCAL occurring after timestamp (time given by Time() object) if tstamps != []: print 'SOLPNTCAL scans were found at ', for tstamp in tstamps: if type(tstamp) is np.ndarray: # Annoyingly necessary when only one time in tstamps tstamp = tstamp[0] t1 = util.Time(tstamp,format='lv') print t1.iso,';', print ' ' good = np.where(tstamps >= timestamp)[0] # This is the timestamp of the first SOLPNTCAL scan after given time if good.shape[0] == 0: stimestamp = tstamps[0] else: stimestamp = tstamps[good][0] else: print 'Warning: No SOLPNTCAL scan found, so interpreting given time as SOLPNTCAL time.' stimestamp = timestamp # Grab 300 records after the start time cursor = dbutil.get_cursor() # Now version independent! verstr = dbutil.find_table_version(cursor,timestamp) if verstr is None: print 'No stateframe table found for the given time.' return {} solpntdict = dbutil.get_dbrecs(cursor,version=int(verstr),dimension=15,timestamp=stimestamp,nrecs=300) # Need dimension-1 data to get antennas in subarray -- Note: sometimes the antenna list # is zero (an unlikely value!) around the time of the start of a scan, so keep searching # first 100 records until non-zero: for i in range(100): blah = dbutil.get_dbrecs(cursor,version=int(verstr),dimension=1,timestamp=stimestamp+i,nrecs=1) if blah['LODM_Subarray1'][0] != 0: break cursor.close() sub1 = blah['LODM_Subarray1'][0] subarray1 = [] antlist = [] for i in range(16): subarray1.append(sub1 & (1<<i) > 0) if subarray1[-1]: antlist.append(i) # print 'Antlist:',antlist # rao = np.zeros([15,300],dtype='int') # deco = np.zeros([15,300],dtype='int') # trk = np.zeros([15,300],dtype='bool') # hpol = np.zeros([15,300],dtype='float') # vpol = np.zeros([15,300],dtype='float') # ra = np.zeros(15,dtype='float') # dec = np.zeros(15,dtype='float') ra = (solpntdict['Ante_Cont_RAVirtualAxis'][:,antlist]*np.pi/10000./180.).astype('float') dec = (solpntdict['Ante_Cont_DecVirtualAxis'][:,antlist]*np.pi/10000./180.).astype('float') hpol = (solpntdict['Ante_Fron_FEM_HPol_Voltage'][:,antlist]).astype('float') vpol = (solpntdict['Ante_Fron_FEM_VPol_Voltage'][:,antlist]).astype('float') rao = (solpntdict['Ante_Cont_RAOffset'][:,antlist]).astype('float') deco = (solpntdict['Ante_Cont_DecOffset'][:,antlist]).astype('float') times = solpntdict['Timestamp'][:,0].astype('int64').astype('float') # Convert pointing info to track information outdict = stateframe.azel_from_sqldict(solpntdict) trk = np.logical_and(outdict['dAzimuth'][:,antlist]<0.0020,outdict['dElevation'][:,antlist]<0.0020) return {'Timestamp':stimestamp,'tstamps':times,'antlist':antlist,'trjfile':'SOLPNT.TRJ','ra':ra,'dec':dec, 'rao':rao,'deco':deco,'trk':trk,'hpol':hpol,'vpol':vpol}
def get_attncal(trange, do_plot=False, dataonly=False): ''' Finds GAINCALTEST scans from FDB files corresponding to the days present in trange Time() object (can be multiple days), calculates the attenuation differences for the various FEMATTN states 1-8 relative to FEMATTN state 0, and optionally plots the results for states 1 and 2 (the most commonly used). To analyze only a single day, trange Time() object can have the same time repeated, or can be a single time. Returns a list of dictionaries, each pertaining to one of the days in trange, with keys defined as follows: 'time': The start time of the GAINCALTEST scan, as a Time() object 'fghz': The list of frequencies [GHz] at which attenuations are measured 'attn': The array of attenuations [dB] of size (nattn, nant, npol, nf), where nattn = 8, nant = 13, npol = 2, and nf is variable 'rcvr': The array of receiver noise level (raw units) of size (nant, npol, nf), where nant = 13, npol = 2, and nf is variable 'rcvr_auto': Same as rcvr, but for auto-correlation (hence it is complex) N.B.: Ignores days with other than one GAINCALTEST measurement, e.g. 0 or 2, the first is obvious, while the second is because there is no way to tell which of the 2 are good. The dataonly parameter tells the routine to skip calculating the attenuation and only return the IDB data from the (first) gaincal. ''' from util import get_idbdir, fname2mjd, nearest_val_idx import socket import dbutil if type(trange.mjd) == np.float: # Interpret single time as both start and end time mjd1 = int(trange.mjd) mjd2 = mjd1 else: mjd1, mjd2 = trange.mjd.astype(int) if do_plot: import matplotlib.pylab as plt f, ax = plt.subplots(4, 13) f.set_size_inches((14, 5)) ax[0, 0].set_ylabel('Atn1X [dB]') ax[1, 0].set_ylabel('Atn1Y [dB]') ax[2, 0].set_ylabel('Atn2X [dB]') ax[3, 0].set_ylabel('Atn2Y [dB]') for i in range(13): ax[0, i].set_title('Ant ' + str(i + 1)) ax[3, i].set_xlabel('Freq [GHz]') for j in range(2): ax[j, i].set_ylim(1, 3) ax[j + 2, i].set_ylim(3, 5) outdict = [] for mjd in range(mjd1, mjd2 + 1): fdb = dt.rd_fdb(Time(mjd, format='mjd')) gcidx, = np.where(fdb['PROJECTID'] == 'GAINCALTEST') if len(gcidx) == 1: print fdb['FILE'][gcidx] gcidx = gcidx[0] else: for i, fname in enumerate(fdb['FILE'][gcidx]): print str(i) + ': GAINCALTEST File', fname idex = input('There is more than one GAINCALTEST. Select: ' + str(np.arange(len(gcidx))) + ':') gcidx = gcidx[idex] datadir = get_idbdir(Time(mjd, format='mjd')) # Add date path if on pipeline # if datadir.find('eovsa') != -1: datadir += fdb['FILE'][gcidx][3:11]+'/' host = socket.gethostname() if host == 'pipeline': datadir += fdb['FILE'][gcidx][3:11] + '/' file = datadir + fdb['FILE'][gcidx] out = ri.read_idb([file]) if dataonly: return out # Get time from filename and read 120 records of attn state from SQL database filemjd = fname2mjd(fdb['FILE'][gcidx]) cursor = dbutil.get_cursor() d15 = dbutil.get_dbrecs(cursor, dimension=15, timestamp=Time(filemjd, format='mjd'), nrecs=120) cursor.close() # Find time indexes of the 62 dB attn state # Uses only ant 1 assuming all are the same dtot = (d15['Ante_Fron_FEM_HPol_Atte_Second'] + d15['Ante_Fron_FEM_HPol_Atte_First'])[:, 0] # Use system clock day number to identify bad SQL entries and eliminate them good, = np.where(d15['Ante_Cont_SystemClockMJDay'][:, 0] != 0) #import pdb; pdb.set_trace() # Indexes into SQL records where a transition occurred. transitions, = np.where(dtot[good] - np.roll(dtot[good], 1) != 0) # Eliminate any zero-index transition (if it exists) if transitions[0] == 0: transitions = transitions[1:] # These now have to be translated into indexes into the data, using the times idx = nearest_val_idx(d15['Timestamp'][good, 0][transitions], Time(out['time'], format='jd').lv) #import pdb; pdb.set_trace() vx = np.nanmedian( out['p'][:13, :, :, np.arange(idx[0] + 1, idx[1] - 1)], 3) va = np.mean(out['a'][:13, :2, :, np.arange(idx[0] + 1, idx[1] - 1)], 3) vals = [] attn = [] for i in range(1, 10): vals.append( np.nanmedian( out['p'][:13, :, :, np.arange(idx[i] + 1, idx[i + 1] - 1)], 3) - vx) attn.append(np.log10(vals[0] / vals[-1]) * 10.) #vals = [] #attna = [] #for i in range(1,10): # vals.append(np.median(out['a'][:13,:2,:,np.arange(idx[i],idx[i+1])],3) - va) # attna.append(np.log10(vals[0]/vals[-1])*10.) if do_plot: for i in range(13): for j in range(2): ax[j, i].plot(out['fghz'], attn[1][i, j], '.', markersize=3) #ax[j,i].plot(out['fghz'],attna[1][i,j],'.',markersize=1) ax[j + 2, i].plot(out['fghz'], attn[2][i, j], '.', markersize=3) #ax[j+2,i].plot(out['fghz'],attna[2][i,j],'.',markersize=1) outdict.append({ 'time': Time(out['time'][0], format='jd'), 'fghz': out['fghz'], 'rcvr_auto': va, # 'attna': np.array(attna[1:]), 'rcvr': vx, 'attn': np.array(attn[1:]) }) return outdict
def send_xml2sql(): ''' Routine to send any changed calibration xml definitions to the SQL Server. The latest definition (if any) for a given type is checked to see if the version matches. If not, an update is stored. This routine will typically be run whenever a definition is added or changed. ''' import dbutil, read_xml2, sys t = util.Time.now() timestamp = t.lv # Current time as LabVIEW timestamp cursor = dbutil.get_cursor() typdict = cal_types() for key in typdict.keys(): #print 'Working on',typdict[key][0] # Execute the code to create the xml description for this key exec 'buf = '+typdict[key][1]+'()' # Resulting buf must be written to a temporary file and reread # by xml_ptrs() f = open('/tmp/tmp.xml','wb') f.write(buf) f.close() mydict, xmlver = read_xml2.xml_ptrs('/tmp/tmp.xml') defn_version = float(key)+xmlver/10. # Version number expected # Retrieve most recent key.0 record and check its version against the expected one query = 'select top 1 * from abin where Version = '+key+'.0 order by Timestamp desc' #print 'Executing query' outdict, msg = dbutil.do_query(cursor,query) #print msg if msg == 'Success': if len(outdict) == 0: # This type of xml file does not yet exist in the database, so mark it for adding add = True else: # There is one, so see if it agrees with the version buf = outdict['Bin'][0] # Binary representation of xml file f = open('/tmp/tmp.xml','wb') f.write(buf) f.close() mydict, thisver = read_xml2.xml_ptrs('/tmp/tmp.xml') #print 'Versions:',float(key)+thisver/10.,defn_version if (float(key)+thisver/10.) == defn_version: # This description is already there so we will skip it add = False else: add = True else: # Some kind of error occurred, so print the message and skip adding the xml description #print 'Query',query,'resulted in error message:',msg add = False if add: # This is either new or updated, so add the xml description # to the database #print 'Trying to add',typdict[key][0] try: cursor.execute('insert into aBin (Timestamp,Version,Description,Bin) values (?,?,?,?)', timestamp, float(key), typdict[key][0], dbutil.stateframedef.pyodbc.Binary(buf)) print typdict[key][0],'successfully added/updated to version',defn_version except: print 'Unknown error occurred in adding',typdict[key][0] print sys.exc_info()[1] else: print typdict[key][0],'version',defn_version,'already exists--not updated' cursor.commit() cursor.close()
def get_gain_state(trange, dt=None, relax=False): ''' Get all gain-state information for a given timerange. Returns a dictionary with keys as follows: times: A Time object containing the array of times, size (nt) h1: The first HPol attenuator value for 15 antennas, size (nt, 15) v1: The first VPol attenuator value for 15 antennas, size (nt, 15) h2: The second HPol attenuator value for 15 antennas, size (nt, 15) v2: The second VPol attenuator value for 15 antennas, size (nt, 15) dcmattn: The base DCM attenuations for nbands x 15 antennas x 2 Poln, size (34 or 52,30) The order is Ant1 H, Ant1 V, Ant2 H, Ant2 V, etc. dcmoff: If DPPoffset-on is 0, this is None (meaning there are no changes to the above base attenuations). If DPPoffset-on is 1, then dcmoff is a table of offsets to the base attenuation, size (nt, 50). The offset applies to all antennas/polarizations. Optional keywords: dt Seconds between entries to read from SQL stateframe database. If omitted, 1 s is assumed. relax Used for gain of reference time, in case there are no SQL data for the requested time. In that case it finds the data for the nearest later time. ''' if dt is None: tstart, tend = [str(i) for i in trange.lv] else: # Expand time by 1/2 of dt before and after tstart = str(np.round(trange[0].lv - dt / 2)) tend = str(np.round(trange[1].lv + dt / 2)) cursor = db.get_cursor() ver = db.find_table_version(cursor, trange[0].lv) # Get front end attenuator states # Attempt to solve the problem if there are no data if relax: # Special case of reference gain, where we want the first nt records after tstart, in case there # are no data at time tstart nt = int(float(tend) - float(tstart) - 1) * 15 query = 'select top '+str(nt)+' Timestamp,Ante_Fron_FEM_HPol_Atte_First,Ante_Fron_FEM_HPol_Atte_Second,' \ +'Ante_Fron_FEM_VPol_Atte_First,Ante_Fron_FEM_VPol_Atte_Second,Ante_Fron_FEM_Clockms from fV' \ +ver+'_vD15 where Timestamp >= '+tstart+' order by Timestamp' else: query = 'select Timestamp,Ante_Fron_FEM_HPol_Atte_First,Ante_Fron_FEM_HPol_Atte_Second,' \ +'Ante_Fron_FEM_VPol_Atte_First,Ante_Fron_FEM_VPol_Atte_Second,Ante_Fron_FEM_Clockms from fV' \ +ver+'_vD15 where Timestamp >= '+tstart+' and Timestamp < '+tend+' order by Timestamp' #if dt: # # If dt (seconds between measurements) is set, add appropriate SQL statement to query # query += ' and (cast(Timestamp as bigint) % '+str(dt)+') = 0 ' data, msg = db.do_query(cursor, query) if msg == 'Success': if dt: # If we want other than full cadence, get new array shapes and times n = len(data['Timestamp']) # Original number of times new_n = ( n / 15 / dt ) * 15 * dt # Truncated number of times equally divisible by dt new_shape = (n / 15 / dt, dt, 15) # New shape of truncated arrays times = Time(data['Timestamp'][:new_n].astype('int')[::15 * dt], format='lv') else: times = Time(data['Timestamp'].astype('int')[::15], format='lv') # Change tstart and tend to correspond to actual times from SQL tstart, tend = [str(i) for i in times[[0, -1]].lv] h1 = data['Ante_Fron_FEM_HPol_Atte_First'] h2 = data['Ante_Fron_FEM_HPol_Atte_Second'] v1 = data['Ante_Fron_FEM_VPol_Atte_First'] v2 = data['Ante_Fron_FEM_VPol_Atte_Second'] ms = data['Ante_Fron_FEM_Clockms'] nt = len(h1) / 15 h1.shape = (nt, 15) h2.shape = (nt, 15) v1.shape = (nt, 15) v2.shape = (nt, 15) ms.shape = (nt, 15) # Find any entries for which Clockms is zero, which indicates where no # gain-state measurement is available. for i in range(15): bad, = np.where(ms[:, i] == 0) if bad.size != 0 and bad.size != nt: # Find nearest adjacent good value good, = np.where(ms[:, i] != 0) idx = nearest_val_idx(bad, good) h1[bad, i] = h1[good[idx], i] h2[bad, i] = h2[good[idx], i] v1[bad, i] = v1[good[idx], i] v2[bad, i] = v2[good[idx], i] if dt: # If we want other than full cadence, find mean over dt measurements h1 = np.mean(h1[:new_n / 15].reshape(new_shape), 1) h2 = np.mean(h2[:new_n / 15].reshape(new_shape), 1) v1 = np.mean(v1[:new_n / 15].reshape(new_shape), 1) v2 = np.mean(v2[:new_n / 15].reshape(new_shape), 1) # Put results in canonical order [nant, nt] h1 = h1.T h2 = h2.T v1 = v1.T v2 = v2.T else: print 'Error reading FEM attenuations:', msg return {} # Get back end attenuator states xml, buf = ch.read_cal(2, t=trange[0]) dcmattn = stf.extract(buf, xml['Attenuation']) nbands = dcmattn.shape[0] dcmattn.shape = (nbands, 15, 2) # Put into canonical order [nant, npol, nband] dcmattn = np.moveaxis(dcmattn, 0, 2) # See if DPP offset is enabled query = 'select Timestamp,DPPoffsetattn_on from fV' \ +ver+'_vD1 where Timestamp >= '+tstart+' and Timestamp <= '+tend+'order by Timestamp' data, msg = db.do_query(cursor, query) if msg == 'Success': dppon = data['DPPoffsetattn_on'] if np.where(dppon > 0)[0].size == 0: dcm_off = None else: query = 'select Timestamp,DCMoffset_attn from fV' \ +ver+'_vD50 where Timestamp >= '+tstart+' and Timestamp <= '+tend #if dt: # # If dt (seconds between measurements) is set, add appropriate SQL statement to query # query += ' and (cast(Timestamp as bigint) % '+str(dt)+') = 0 ' query += ' order by Timestamp' data, msg = db.do_query(cursor, query) if msg == 'Success': otimes = Time(data['Timestamp'].astype('int')[::15], format='lv') dcmoff = data['DCMoffset_attn'] dcmoff.shape = (nt, 50) # We now have a time-history of offsets, at least some of which are non-zero. # Offsets by slot number do us no good, so we need to translate to band number. # Get fseqfile name at mean of timerange, from stateframe SQL database fseqfile = get_fseqfile( Time(int(np.mean(trange.lv)), format='lv')) if fseqfile is None: print 'Error: No active fseq file.' dcm_off = None else: # Get fseqfile from ACC and return bandlist bandlist = fseqfile2bandlist(fseqfile) nbands = len(bandlist) # Use bandlist to covert nt x 50 array to nt x nbands array of DCM attn offsets # Note that this assumes DCM offset is the same for any multiply-sampled bands # in the sequence. dcm_off = np.zeros((nt, nbands), float) dcm_off[:, bandlist - 1] = dcmoff # Put into canonical order [nband, nt] dcm_off = dcm_off.T if dt: # If we want other than full cadence, find mean over dt measurements new_nt = len(times) dcm_off = dcm_off[:, :new_nt * dt] dcm_off.shape = (nbands, dt, new_nt) dcm_off = np.mean(dcm_off, 1) else: print 'Error reading DCM attenuations:', msg dcm_off = None else: print 'Error reading DPPon state:', msg dcm_off = None cursor.close() return { 'times': times, 'h1': h1, 'v1': v1, 'h2': h2, 'v2': v2, 'dcmattn': dcmattn, 'dcmoff': dcm_off }
def flare_monitor(t): ''' Get all front-end power-detector voltages for the given day from the stateframe SQL database, and obtain the median of them, to use as a flare monitor. Returns ut times in plot_date format and median voltages. ''' import dbutil # timerange is 13 UT to 02 UT on next day, relative to the day in Time() object t trange = Time([int(t.mjd) + 13./24,int(t.mjd) + 26./24],format='mjd') tstart, tend = trange.lv.astype('str') cursor = dbutil.get_cursor() mjd = t.mjd try: verstr = dbutil.find_table_version(cursor,tstart) if verstr is None: print 'No stateframe table found for given time.' return tstart, [], {} cursor.execute('select * from fV'+verstr+'_vD15 where I15 < 4 and timestamp between '+tstart+' and '+tend+' order by timestamp') except: print 'Error with query of SQL database.' return tstart, [], {} data = np.transpose(np.array(cursor.fetchall(),'object')) if len(data) == 0: # No data found, so return timestamp and empty lists print 'SQL Query was valid, but no data for',t.iso[:10],'were found (yet).' return tstart, [], {} ncol,ntot = data.shape data.shape = (ncol,ntot/4,4) names = np.array(cursor.description)[:,0] mydict = dict(zip(names,data)) hv = [] ut = Time(mydict['Timestamp'][:,0].astype('float'),format='lv').plot_date hfac = np.median(mydict['Ante_Fron_FEM_HPol_Voltage'].astype('float'),0) vfac = np.median(mydict['Ante_Fron_FEM_VPol_Voltage'].astype('float'),0) for i in range(4): if hfac[i] > 0: hv.append(mydict['Ante_Fron_FEM_HPol_Voltage'][:,i]/hfac[i]) if vfac[i] > 0: hv.append(mydict['Ante_Fron_FEM_VPol_Voltage'][:,i]/vfac[i]) flm = np.median(np.array(hv),0) good = np.where(abs(flm[1:]-flm[:-1])<0.01)[0] # Get the project IDs for scans during the period verstrh = dbutil.find_table_version(cursor,trange[0].lv,True) if verstrh is None: print 'No scan_header table found for given time.' return ut[good], flm[good], {} cursor.execute('select Timestamp,Project from hV'+verstrh+'_vD1 where Timestamp between '+tstart+' and '+tend+' order by Timestamp') data = np.transpose(np.array(cursor.fetchall())) names = np.array(cursor.description)[:,0] if len(data) == 0: # No Project ID found, so return data and empty projdict dictionary print 'SQL Query was valid, but no Project data were found.' return ut[good], flm[good], {} projdict = dict(zip(names,data)) projdict['Timestamp'] = projdict['Timestamp'].astype('float') # Convert timestamps from string to float # Get the times when scanstate is -1 cursor.execute('select Timestamp,Sche_Data_ScanState from fV'+verstr+'_vD1 where Timestamp between '+tstart+' and '+tend+' and Sche_Data_ScanState = -1 order by Timestamp') scan_off_times = np.transpose(np.array(cursor.fetchall()))[0] #Just list of timestamps if len(scan_off_times) > 2: gaps = scan_off_times[1:] - scan_off_times[:-1] - 1 eos = np.where(gaps > 10)[0] if len(eos) > 1: if scan_off_times[eos[1]] < projdict['Timestamp'][0]: # Gaps are not lined up, so drop the first: eos = eos[1:] EOS = scan_off_times[eos] if scan_off_times[eos[0]] <= projdict['Timestamp'][0]: # First EOS is earlier than first Project ID, so make first Project ID None. projdict['Timestamp'] = np.append([scan_off_times[0]],projdict['Timestamp']) projdict['Project'] = np.append(['None'],projdict['Project']) if scan_off_times[eos[-1]+1] >= projdict['Timestamp'][-1]: # Last EOS is later than last Project ID, so make last Project ID None. projdict['Timestamp'] = np.append(projdict['Timestamp'],[scan_off_times[eos[-1]+1]]) projdict['Project'] = np.append(projdict['Project'],['None']) EOS = np.append(EOS,[scan_off_times[eos[-1]+1],scan_off_times[-1]]) projdict.update({'EOS': EOS}) else: # Not enough scan changes to determine EOS (end-of-scan) times projdict.update({'EOS': []}) cursor.close() return ut[good],flm[good],projdict
def flare_monitor(t): ''' Get all front-end power-detector voltages for the given day from the stateframe SQL database, and obtain the median of them, to use as a flare monitor. Returns ut times in plot_date format and median voltages. ''' import dbutil # timerange is 13 UT to 02 UT on next day, relative to the day in Time() object t trange = Time([int(t.mjd) + 13. / 24, int(t.mjd) + 26. / 24], format='mjd') tstart, tend = trange.lv.astype('str') cursor = dbutil.get_cursor() mjd = t.mjd try: verstr = dbutil.find_table_version(cursor, tstart) if verstr is None: print 'No stateframe table found for given time.' return tstart, [], {} cursor.execute('select * from fV' + verstr + '_vD15 where I15 < 4 and timestamp between ' + tstart + ' and ' + tend + ' order by timestamp') except: print 'Error with query of SQL database.' return tstart, [], {} data = np.transpose(np.array(cursor.fetchall(), 'object')) if len(data) == 0: # No data found, so return timestamp and empty lists print 'SQL Query was valid, but no data for', t.iso[: 10], 'were found (yet).' return tstart, [], {} ncol, ntot = data.shape data.shape = (ncol, ntot / 4, 4) names = np.array(cursor.description)[:, 0] mydict = dict(zip(names, data)) hv = [] ut = Time(mydict['Timestamp'][:, 0].astype('float'), format='lv').plot_date hfac = np.median(mydict['Ante_Fron_FEM_HPol_Voltage'].astype('float'), 0) vfac = np.median(mydict['Ante_Fron_FEM_VPol_Voltage'].astype('float'), 0) for i in range(4): if hfac[i] > 0: hv.append(mydict['Ante_Fron_FEM_HPol_Voltage'][:, i] / hfac[i]) if vfac[i] > 0: hv.append(mydict['Ante_Fron_FEM_VPol_Voltage'][:, i] / vfac[i]) flm = np.median(np.array(hv), 0) good = np.where(abs(flm[1:] - flm[:-1]) < 0.01)[0] # Get the project IDs for scans during the period verstrh = dbutil.find_table_version(cursor, trange[0].lv, True) if verstrh is None: print 'No scan_header table found for given time.' return ut[good], flm[good], {} cursor.execute('select Timestamp,Project from hV' + verstrh + '_vD1 where Timestamp between ' + tstart + ' and ' + tend + ' order by Timestamp') data = np.transpose(np.array(cursor.fetchall())) names = np.array(cursor.description)[:, 0] if len(data) == 0: # No Project ID found, so return data and empty projdict dictionary print 'SQL Query was valid, but no Project data were found.' return ut[good], flm[good], {} projdict = dict(zip(names, data)) projdict['Timestamp'] = projdict['Timestamp'].astype( 'float') # Convert timestamps from string to float # Get the times when scanstate is -1 cursor.execute('select Timestamp,Sche_Data_ScanState from fV' + verstr + '_vD1 where Timestamp between ' + tstart + ' and ' + tend + ' and Sche_Data_ScanState = -1 order by Timestamp') scan_off_times = np.transpose(np.array( cursor.fetchall()))[0] #Just list of timestamps if len(scan_off_times) > 2: gaps = scan_off_times[1:] - scan_off_times[:-1] - 1 eos = np.where(gaps > 10)[0] if len(eos) > 1: if scan_off_times[eos[1]] < projdict['Timestamp'][0]: # Gaps are not lined up, so drop the first: eos = eos[1:] EOS = scan_off_times[eos] if scan_off_times[eos[0]] <= projdict['Timestamp'][0]: # First EOS is earlier than first Project ID, so make first Project ID None. projdict['Timestamp'] = np.append([scan_off_times[0]], projdict['Timestamp']) projdict['Project'] = np.append(['None'], projdict['Project']) if scan_off_times[eos[-1] + 1] >= projdict['Timestamp'][-1]: # Last EOS is later than last Project ID, so make last Project ID None. projdict['Timestamp'] = np.append(projdict['Timestamp'], [scan_off_times[eos[-1] + 1]]) projdict['Project'] = np.append(projdict['Project'], ['None']) EOS = np.append(EOS, [scan_off_times[eos[-1] + 1], scan_off_times[-1]]) projdict.update({'EOS': EOS}) else: # Not enough scan changes to determine EOS (end-of-scan) times projdict.update({'EOS': []}) cursor.close() return ut[good], flm[good], projdict
def send_xml2sql(type=None,t=None,test=False,nant=None,nfrq=None): ''' Routine to send any changed calibration xml definitions to the SQL Server. The latest definition (if any) for a given type is checked to see if the version matches. If not, an update is stored. This routine will typically be run whenever a definition is added or changed. If type is provided (i.e. not None), only the given type will be updated (and only if its internal version number has changed). The timestamp of the new record will be set according to the Time() object t, if provided, or the current time if not. As a debugging tool, if test is True, this routine goes through the motions but does not write to the abin table. ''' import dbutil, read_xml2, sys if t is None: t = util.Time.now() timestamp = int(t.lv) # Current time as LabVIEW timestamp cursor = dbutil.get_cursor() typdict = cal_types() if type: # If a particular type is specified, limit the action to that type typdict = {type:typdict[type]} for key in typdict.keys(): #print 'Working on',typdict[key][0] # Execute the code to create the xml description for this key if key == 1: # Special case for TP calibration if nant is None or nfrq is None: print 'For',typdict[key][0],'values for both nant and nfrq are required.' cursor.close() return exec 'buf = '+typdict[key][1]+'(nant='+str(nant)+',nfrq='+str(nfrq)+')' else: exec 'buf = '+typdict[key][1]+'()' # Resulting buf must be written to a temporary file and reread # by xml_ptrs() f = open('/tmp/tmp.xml','wb') f.write(buf) f.close() mydict, xmlver = read_xml2.xml_ptrs('/tmp/tmp.xml') defn_version = float(key)+xmlver/10. # Version number expected # Retrieve most recent key.0 record and check its version against the expected one query = 'select top 1 * from abin where Version = '+str(key)+'.0 and Timestamp <= '+str(timestamp)+' order by Timestamp desc' #print 'Executing query' outdict, msg = dbutil.do_query(cursor,query) #print msg if msg == 'Success': if len(outdict) == 0: # This type of xml file does not yet exist in the database, so mark it for adding add = True else: # There is one, so see if they differ buf2 = outdict['Bin'][0] # Binary representation of xml file if buf == buf2: # This description is already there so we will skip it add = False else: add = True else: # Some kind of error occurred, so print the message and skip adding the xml description #print 'Query',query,'resulted in error message:',msg add = False if add: # This is either new or updated, so add the xml description # to the database #print 'Trying to add',typdict[key][0] try: if test: print 'Would have updated',typdict[key][0],'to version',defn_version else: cursor.execute('insert into aBin (Timestamp,Version,Description,Bin) values (?,?,?,?)', timestamp, key, typdict[key][0], dbutil.stateframedef.pyodbc.Binary(buf)) print 'Type definition for',typdict[key][0],'successfully added/updated to version',defn_version,'--OK' cursor.commit() except: print 'Unknown error occurred in adding',typdict[key][0] print sys.exc_info()[1] else: print 'Type definition for',typdict[key][0],'version',defn_version,'exists--OK' cursor.close()
def get_reverseattn(trange_gaincal, trange_other=None, first_attn_base=5, second_attn_base=3, corrected_attns=False): # This function finds and return the tsys, the noise corrected tsys, and the noise and attenuation # tsys. The first parameter it takes should be a GAINCALTEST trange, which may be located with find_gaincal, # The second parameter it takes should be a FEATTNTEST trange, which may be located with find_gaincal as # well. It may or may not take a third parameter. Any form of file that was recorded by the system from # from the antennas may be inserted here, whether a flare or just quiet sun data. # # PLEASE NOTE: ANY trange with data may be used as "trange_gaincal", use a trange from a GAINCALTEST and the other # file as "trange_other" if you want the noise to be calibrated from the GAINCALTEST file, which will most likely # be more recent than the FEATTNTEST file it would otherwise take the noise from. tsys1, res1, noise_level, idx1a, idx3a, marker1, trange_feattncal, trange_feattncal2 = attn_noises(trange_gaincal) if corrected_attns == True: all_attns_avg = get_all_attns(trange_feattncal, test='FEATTNTEST', from_corrected_attns=True) all_attns_avg1 = get_all_attns(trange_feattncal2, test='FEATTNTEST2', from_corrected_attns=True) else: if corrected_attns == False: all_attns_avg = get_all_avg_attns(trange_feattncal, test='FEATTNTEST') all_attns_avg1 = get_all_avg_attns(trange_feattncal2, test='FEATTNTEST2') if trange_other == None: tsys = tsys1 res = res1 idx1 = idx1a idx3 = idx3a marker = marker1 else: s = sp.Spectrogram(trange_other) s.docal = False s.dosub = False s.domedian = False tsys, std = s.get_data() cursor = dbutil.get_cursor() res, msg = dbutil.do_query(cursor,'select Timestamp,Ante_Fron_FEM_HPol_Atte_First,Ante_Fron_FEM_HPol_Atte_Second from fV54_vD15 where Timestamp between '+str(trange_other[0].lv)+' and '+str(trange_other[1].lv)) cursor.close() idx1, idx2 = util.common_val_idx(res['Timestamp'][0::15].astype('int'), (s.time.lv+0.5).astype('int')) marker = -1 while idx1[-1] > idx2[-1]: idx1 = np.delete(idx1, -1) marker += 1 tsys = tsys[:, :, :, idx1] idx3, idx4 = util.common_val_idx(res['Timestamp'].astype('int'), (s.time.lv+0.5).astype('int')) res['Timestamp'] = res['Timestamp'][idx3] if noise_level.shape[2] < tsys.shape[2]: freqs_for_range = noise_level.shape[2] else: freqs_for_range = tsys.shape[2] tsys_noise_corrected = [] for ant in range(tsys.shape[0]): pol_corrected = [] for pol in range(tsys.shape[1]): freq_corrected = [] for freq in range(freqs_for_range): index_corrected = [] for index in range(tsys.shape[3]): index_corrected.append(tsys[ant, pol, freq, index] - noise_level[ant, pol, freq]) freq_corrected.append(index_corrected) pol_corrected.append(freq_corrected) tsys_noise_corrected.append(pol_corrected) tsys_noise_corrected= np.array(tsys_noise_corrected) freqloopslist = [tsys_noise_corrected.shape[2], all_attns_avg.shape[3], all_attns_avg1.shape[3]] freqloops = min(freqloopslist) if tsys.shape[3] < len(res['Ante_Fron_FEM_HPol_Atte_Second'][0::15]): indexloops = tsys.shape[3] else: if tsys.shape[3] >= len(res['Ante_Fron_FEM_HPol_Atte_Second'][0::15]): indexloops = len(res['Ante_Fron_FEM_HPol_Atte_Second'][0::15])-1 if tsys.shape[3] < len(res['Ante_Fron_FEM_HPol_Atte_First'][0::15]): indexloops1 = tsys.shape[3] else: if tsys.shape[3] >= len(res['Ante_Fron_FEM_HPol_Atte_First'][0::15]): indexloops1 = len(res['Ante_Fron_FEM_HPol_Atte_First'][0::15])-1 idxstart = marker + (15-8) xory = ['x' , 'y'] ant_postcorrected = [] for ant in range(tsys.shape[0]): pol_postcorrected = [] for pol in range(tsys.shape[1]): freq_postcorrected = [] for freq in range(freqloops): indices_postcorrected = [] for indx in range(indexloops): testlevel = res['Ante_Fron_FEM_HPol_Atte_Second'][ant::15][indx+idxstart] if 0 <= testlevel <= 31: pass else: print 'Problem with the attenuation of antenna ' + str(ant) + xory[pol] + ' at frequency channel ' + str(freq) + ' and time index ' + str(indx) + '. The attenuation is showing: ' + str(testlevel) testlevel = 0 indices_postcorrected.append(10**((all_attns_avg[testlevel, ant, pol, freq]-all_attns_avg[second_attn_base, ant, pol, freq])/10)*tsys_noise_corrected[ant, pol, freq, indx]) indices_postcorrected1 = [] for indx in range(indexloops1): testlevel = res['Ante_Fron_FEM_HPol_Atte_First'][ant::15][indx+idxstart] if 0 <= testlevel <= 31: pass else: print 'Problem with the attenuation of antenna ' + str(ant) + xory[pol] + ' at frequency channel ' + str(freq) + ' and time index ' + str(indx) + '. The attenuation is showing: ' + str(testlevel) testlevel = 0 indices_postcorrected1.append(10**((all_attns_avg1[testlevel, ant, pol, freq]-all_attns_avg1[first_attn_base, ant, pol, freq])/10)*indices_postcorrected[indx]) freq_postcorrected.append(indices_postcorrected1) pol_postcorrected.append(freq_postcorrected) ant_postcorrected.append(pol_postcorrected) tsys_attn_noise_corrected = np.array(ant_postcorrected) return tsys_attn_noise_corrected, tsys_noise_corrected, tsys
def attn_noises(trange_gaincal): # This function is used with "get_reverseattn." It find and returns background noise, and returns the "res" # from dbutil.do_query and returns the "tsys" from spectrogram_fit and get_data(). # The first parameter it takes should be a GAINCALTEST trange, which may be located with find_gaincal, # The second parameter it takes should be a FEATTNTEST trange, which may be located with find_gaincal as # well. # # PLEASE NOTE: ANY trange with data may be used as "trange_gaincal", use a trange from a GAINCALTEST and the other # file as "trange_other" if you want the noise to be calibrated from the GAINCALTEST file, which will most likely # be more recent than the FEATTNTEST file it would otherwise take the noise from. s = sp.Spectrogram(trange_gaincal) s.docal = False s.dosub = False s.domedian = False tsys, std = s.get_data() trange_feattncal = find_gaincal() if type(trange_feattncal) == list: trange_feattncal = trange_feattncal[-1] else: pass trange_feattncal2 = find_gaincal(t = Time('2015-07-21 00:00'), scan_length=5, findwhat='FEATTNTEST2') if type(trange_feattncal2) == list: trange_feattncal2 = trange_feattncal2[-1] else: pass ratios, calfilenoise = show_dB_ratio(trange_feattncal) ratios1, calfilenoise1 = show_dB_ratio(trange_feattncal2, test='FEATTNTEST2') cursor = dbutil.get_cursor() res, msg = dbutil.do_query(cursor,'select Timestamp,Ante_Fron_FEM_HPol_Atte_First,Ante_Fron_FEM_HPol_Atte_Second from fV54_vD15 where Timestamp between '+str(trange_gaincal[0].lv)+' and '+str(trange_gaincal[1].lv)) cursor.close() idx1, idx2 = util.common_val_idx(res['Timestamp'][0::15].astype('int'), (s.time.lv+0.5).astype('int')) idx3, idx4 = util.common_val_idx(res['Timestamp'].astype('int'), (s.time.lv+0.5).astype('int')) marker = -1 while idx1[-1] > idx2[-1]: idx1 = np.delete(idx1, -1) marker += 1 tsys = tsys[:, :, :, idx1] calfilenoise_ = [] for ant in range(calfilenoise.shape[0]): calfilenoisepol = [] for pol in range(calfilenoise.shape[1]): calfilenoisefreq = [] for freq in range(calfilenoise.shape[2]): calfilenoisefreq.append(np.average(calfilenoise[ant, pol, freq, :])) calfilenoisepol.append(calfilenoisefreq) calfilenoise_.append(calfilenoisepol) calfilenoise = np.array(calfilenoise_) noise_level = [] for ant in range(tsys.shape[0]): pol_noise = [] for pol in range(tsys.shape[1]): freq_noise = [] state, = np.where(np.logical_and(res['Ante_Fron_FEM_HPol_Atte_Second'][ant::15].astype('int') == 31,res['Ante_Fron_FEM_HPol_Atte_First'][ant::15].astype('int') == 31)) for freq in range(tsys.shape[2]): avg_noise = [] for index in state: try: if np.logical_and(tsys[ant, pol, freq, index] <= 0.005, index < tsys.shape[3]): avg_noise.append(tsys[ant, pol, freq, index]) except: pass freq_noise.append(np.average(avg_noise)) pol_noise.append(freq_noise) noise_level.append(pol_noise) noise_level = np.array(noise_level) for ant in range(tsys.shape[0]): for pol in range(tsys.shape[1]): for freq in range(tsys.shape[2]): if np.isnan(noise_level[ant, pol, freq]) == False: pass else: if np.isnan(noise_level[ant, pol, freq]) == True: try: noise_level[ant, pol, freq] = calfilenoise[ant, pol, freq] except: pass return tsys, res, noise_level, idx1, idx3, marker, trange_feattncal, trange_feattncal2
def rd_calpnt(filename): ''' Read and return contents of output of CALPNT or CALPNT2M observation Note that the "x" offsets are dRA and dAZ. To apply these in subsequent routines, dRA is converted to dHA by inverting the sign, while dAZ is converted to dXEL (angle on the sky) by multiplying by cos(EL). ''' import dbutil f = open(filename, 'r') lines = f.readlines() f.close() ants = lines[0].split('Ant ')[1:] antlist = [] try: for ant in ants: antlist.append(int(ant[:2])) except: print 'Unrecognized format (header line) for', filename return None nants = len(ants) lines = lines[1:] nlines = len(lines) dra = np.zeros((nants, nlines), np.float) ddec = np.zeros((nants, nlines), np.float) ha = [] dec = [] source = [] timstr = [] for i, line in enumerate(lines): if len(line) < 9: dra = dra[:, :i] ddec = ddec[:, :i] break vals = line[9:].split() if len(vals) != nants * 2 + 4: print 'Error reading line', i + 2, 'of', filename dra = dra[:, :i] ddec = ddec[:, :i] break else: try: source.append(line[:8]) timstr.append(vals[0] + ' ' + vals[1]) ha.append(vals[2]) dec.append(vals[3]) dra[:, i] = np.array(vals[4::2]).astype(float) ddec[:, i] = np.array(vals[5::2]).astype(float) except: print 'Error parsing line', i + 2, 'of', filename # Convert HA, Dec from degrees to radians, and then convert HA to RA ha = np.array(ha).astype(float) * np.pi / 180. dec = np.array(dec).astype(float) * np.pi / 180. times = Time(timstr) ra = np.zeros_like(ha) for i in range(len(ha)): ra[i] = eovsa_lst(times[i]) - ha[i] # Read pointing parameters from SQL database at time of first observation params_old = np.zeros((9, 15), int) cursor = dbutil.get_cursor() timestamp = times[0].lv # Read stateframe data at time of first observation D15data = dbutil.get_dbrecs(cursor, dimension=15, timestamp=timestamp, nrecs=1) for p in range(9): params_old[p], = D15data['Ante_Cont_PointingCoefficient' + str(p + 1)] params_old = params_old[:, np.array(antlist) - 1] # Pare down to only antennas in antlist return { 'filename': filename, 'source': source, 'time': times, 'params_old': params_old, 'ra': ra, 'dec': dec, 'ha': ha, 'antlist': antlist, 'dra': dra, 'ddec': ddec }
def flare_monitor(t): ''' Get all front-end power-detector voltages for the given day from the stateframe SQL database, and obtain the median of them, to use as a flare monitor. Returns ut times in plot_date format and median voltages. ''' import dbutil # timerange is 12 UT to 12 UT on next day, relative to the day in Time() object t trange = Time([int(t.mjd) + 12. / 24, int(t.mjd) + 36. / 24], format='mjd') tstart, tend = trange.lv.astype('str') cursor = dbutil.get_cursor() mjd = t.mjd verstr = dbutil.find_table_version(cursor, tstart) if verstr is None: print 'No stateframe table found for given time.' return tstart, [], {} query = 'select Timestamp,Ante_Fron_FEM_HPol_Voltage,Ante_Fron_FEM_VPol_Voltage from fV' + verstr + '_vD15 where timestamp between ' + tstart + ' and ' + tend + ' order by timestamp' data, msg = dbutil.do_query(cursor, query) if msg != 'Success': print msg return tstart, [], {} for k, v in data.items(): data[k].shape = (len(data[k]) / 15, 15) hv = [] try: ut = Time(data['Timestamp'][:, 0].astype('float'), format='lv').plot_date except: print 'Error for time', t.iso print 'Query:', query, ' returned msg:', msg print 'Keys:', data.keys() print data['Timestamp'][0, 0] hfac = np.median(data['Ante_Fron_FEM_HPol_Voltage'].astype('float'), 0) vfac = np.median(data['Ante_Fron_FEM_VPol_Voltage'].astype('float'), 0) for i in range(4): if hfac[i] > 0: hv.append(data['Ante_Fron_FEM_HPol_Voltage'][:, i] / hfac[i]) if vfac[i] > 0: hv.append(data['Ante_Fron_FEM_VPol_Voltage'][:, i] / vfac[i]) flm = np.median(np.array(hv), 0) good = np.where(abs(flm[1:] - flm[:-1]) < 0.01)[0] # Get the project IDs for scans during the period verstrh = dbutil.find_table_version(cursor, trange[0].lv, True) if verstrh is None: print 'No scan_header table found for given time.' return ut[good], flm[good], {} query = 'select Timestamp,Project from hV' + verstrh + '_vD1 where Timestamp between ' + tstart + ' and ' + tend + ' order by Timestamp' projdict, msg = dbutil.do_query(cursor, query) if msg != 'Success': print msg return ut[good], flm[good], {} elif len(projdict) == 0: # No Project ID found, so return data and empty projdict dictionary print 'SQL Query was valid, but no Project data were found.' return ut[good], flm[good], {} projdict['Timestamp'] = projdict['Timestamp'].astype( 'float') # Convert timestamps from string to float for i in range(len(projdict['Project'])): projdict['Project'][i] = projdict['Project'][i].replace('\x00', '') # # Get the times when scanstate is -1 # cursor.execute('select Timestamp,Sche_Data_ScanState from fV'+verstr+'_vD1 where Timestamp between '+tstart+' and '+tend+' and Sche_Data_ScanState = -1 order by Timestamp') # scan_off_times = np.transpose(np.array(cursor.fetchall()))[0] #Just list of timestamps # if len(scan_off_times) > 2: # gaps = scan_off_times[1:] - scan_off_times[:-1] - 1 # eos = np.where(gaps > 10)[0] # if len(eos) > 1: # if scan_off_times[eos[1]] < projdict['Timestamp'][0]: # # Gaps are not lined up, so drop the first: # eos = eos[1:] # EOS = scan_off_times[eos] # if scan_off_times[eos[0]] <= projdict['Timestamp'][0]: # # First EOS is earlier than first Project ID, so make first Project ID None. # projdict['Timestamp'] = np.append([scan_off_times[0]],projdict['Timestamp']) # projdict['Project'] = np.append(['None'],projdict['Project']) # if scan_off_times[eos[-1]+1] >= projdict['Timestamp'][-1]: # # Last EOS is later than last Project ID, so make last Project ID None. # projdict['Timestamp'] = np.append(projdict['Timestamp'],[scan_off_times[eos[-1]+1]]) # projdict['Project'] = np.append(projdict['Project'],['None']) # EOS = np.append(EOS,[scan_off_times[eos[-1]+1],scan_off_times[-1]]) # projdict.update({'EOS': EOS}) # else: # # Not enough scan changes to determine EOS (end-of-scan) times # projdict.update({'EOS': []}) # This turns out to be a more rational, and "good-enough"" approach to the end of scan problem. # The last scan, though, will be ignored... projdict.update({'EOS': projdict['Timestamp'][1:]}) projdict.update({'Timestamp': projdict['Timestamp'][:-1]}) projdict.update({'Project': projdict['Project'][:-1]}) cursor.close() return ut[good], flm[good], projdict
def DCM_master_attn_cal(update=False): ''' New version of this command, which uses the power values in the 10gbe packet headers instead of the very slow measurement of the ADC levels themselves. This version only takes about 8 s! If update is True, it writes the results to the SQL database. Returns the DCM_master_table in the form of lines of text strings, with labels (handy for viewing). ''' pwr = np.zeros((50,8,4),'int') # Capture on eth2 interface command = 'tcpdump -i eth2 -c 155000 -w /home/user/Python/dcm2.pcap -s 1000' p.sendcmd(command) # Capture on eth3 interface command = 'tcpdump -i eth3 -c 155000 -w /home/user/Python/dcm3.pcap -s 1000' p.sendcmd(command) headers = p.list_header('/home/user/Python/dcm2.pcap') for line in headers: try: j, id, p1,p2,p3,p4 = np.array(map(int,line.split()))[[0,3,6,7,8,9]] pwr[j,id] = (p1, p2, p3, p4) except: # This is to skip the non-data header lines in the list pass headers = p.list_header('/home/user/Python/dcm3.pcap') for line in headers: try: j, id, p1,p2,p3,p4 = np.array(map(int,line.split()))[[0,3,6,7,8,9]] pwr[j,id] = (p1, p2, p3, p4) except: # This is to skip the non-data header lines in the list pass # Reshape to (slot, nant, npol) pwr.shape = (50,16,2) # Read current frequency sequence from database cursor = db.get_cursor() query = 'select top 50 FSeqList from hV37_vD50 order by Timestamp desc' fseq, msg = db.do_query(cursor, query) if msg == 'Success': fseqlist = fseq['FSeqList'][::-1] # Reverse the order bandlist = ((np.array(fseqlist)-0.44)*2).astype(int) cursor.close() # Read current DCM_master_table from database xml, buf = ch.read_cal(2) orig_table = stf.extract(buf,xml['Attenuation']) # Order pwr values according to bandlist, taking median of any repeated values new_pwr = np.zeros((34,16,2)) for i in range(34): idx, = np.where(bandlist-1 == i) if len(idx) > 0: new_pwr[i] = np.median(pwr[idx],0) new_pwr.shape = (34,32) # Now determine the change in attenuation needed to achieve a target # value of 1600. Eliminate last two entries, corresponding to Ant16 attn = np.log10(new_pwr[:,:-2]/1600.)*10. new_table = (np.clip(orig_table + attn,0,30)/2).astype(int)*2 DCMlines = [] DCMlines.append('# Ant1 Ant2 Ant3 Ant4 Ant5 Ant6 Ant7 Ant8 Ant9 Ant10 Ant11 Ant12 Ant13 Ant14 Ant15') DCMlines.append('# X Y X Y X Y X Y X Y X Y X Y X Y X Y X Y X Y X Y X Y X Y X Y') DCMlines.append('# ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- -----') for band in range(1,35): DCMlines.append('{:2} : {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2}'.format(band,*new_table[band-1])) if update: msg = ch.dcm_master_table2sql(DCMlines) if msg: print 'Success' else: print 'Error writing table to SQL database!' return DCMlines
def get_fem_level(trange, dt=None): ''' Get FEM attenuation levels for a given timerange. Returns a dictionary with keys as follows: times: A Time object containing the array of times, size (nt) hlev: The FEM attenuation level for HPol, size (nt, 15) vlev: The FEM attenuation level for VPol, size (nt, 15) dcmattn: The base DCM attenuations for 34 bands x 15 antennas x 2 Poln, size (34,30) The order is Ant1 H, Ant1 V, Ant2 H, Ant2 V, etc. dcmoff: If DPPoffset-on is 0, this is None (meaning there are no changes to the above base attenuations). If DPPoffset-on is 1, then dcmoff is a table of offsets to the base attenuation, size (nt, 50). The offset applies to all antennas/polarizations. Optional keywords: dt Seconds between entries to read from SQL stateframe database. If omitted, 1 s is assumed. ''' if dt is None: tstart, tend = [str(i) for i in trange.lv] else: # Expand time by 1/2 of dt before and after tstart = str(np.round(trange[0].lv - dt / 2)) tend = str(np.round(trange[1].lv + dt / 2)) cursor = db.get_cursor() ver = db.find_table_version(cursor, trange[0].lv) # Get front end attenuator states query = 'select Timestamp,Ante_Fron_FEM_Clockms,' \ +'Ante_Fron_FEM_HPol_Regi_Level,Ante_Fron_FEM_VPol_Regi_Level from fV' \ +ver+'_vD15 where Timestamp >= '+tstart+' and Timestamp <= '+tend+' order by Timestamp' data, msg = db.do_query(cursor, query) if msg == 'Success': if dt: # If we want other than full cadence, get new array shapes and times n = len(data['Timestamp']) # Original number of times new_n = ( n / 15 / dt ) * 15 * dt # Truncated number of times equally divisible by dt new_shape = (n / 15 / dt, dt, 15) # New shape of truncated arrays times = Time(data['Timestamp'][:new_n].astype('int')[::15 * dt], format='lv') else: times = Time(data['Timestamp'].astype('int')[::15], format='lv') hlev = data['Ante_Fron_FEM_HPol_Regi_Level'] vlev = data['Ante_Fron_FEM_VPol_Regi_Level'] ms = data['Ante_Fron_FEM_Clockms'] nt = len(hlev) / 15 hlev.shape = (nt, 15) vlev.shape = (nt, 15) ms.shape = (nt, 15) # Find any entries for which Clockms is zero, which indicates where no # gain-state measurement is available. for i in range(15): bad, = np.where(ms[:, i] == 0) if bad.size != 0 and bad.size != nt: # Find nearest adjacent good value good, = np.where(ms[:, i] != 0) idx = nearest_val_idx(bad, good) hlev[bad, i] = hlev[good[idx], i] vlev[bad, i] = vlev[good[idx], i] if dt: # If we want other than full cadence, find mean over dt measurements hlev = np.mean(hlev[:new_n / 15].reshape(new_shape), 1) vlev = np.mean(vlev[:new_n / 15].reshape(new_shape), 1) # Put results in canonical order [nant, nt] hlev = hlev.T vlev = vlev.T else: print 'Error reading FEM levels:', msg return {} # Get back end attenuator states xml, buf = ch.read_cal(2, t=trange[0]) dcmattn = stf.extract(buf, xml['Attenuation']) dcmattn.shape = (34, 15, 2) # Put into canonical order [nant, npol, nband] dcmattn = np.moveaxis(dcmattn, 0, 2) # See if DPP offset is enabled query = 'select Timestamp,DPPoffsetattn_on from fV' \ +ver+'_vD1 where Timestamp >= '+tstart+' and Timestamp <= '+tend+'order by Timestamp' data, msg = db.do_query(cursor, query) if msg == 'Success': dppon = data['DPPoffsetattn_on'] if np.where(dppon > 0)[0].size == 0: dcm_off = None else: query = 'select Timestamp,DCMoffset_attn from fV' \ +ver+'_vD50 where Timestamp >= '+tstart+' and Timestamp <= '+tend+' order by Timestamp' data, msg = db.do_query(cursor, query) if msg == 'Success': otimes = Time(data['Timestamp'].astype('int')[::15], format='lv') dcmoff = data['DCMoffset_attn'] dcmoff.shape = (nt, 50) # We now have a time-history of offsets, at least some of which are non-zero. # Offsets by slot number do us no good, so we need to translate to band number. # Get fseqfile name at mean of timerange, from stateframe SQL database fseqfile = get_fseqfile( Time(int(np.mean(trange.lv)), format='lv')) if fseqfile is None: print 'Error: No active fseq file.' dcm_off = None else: # Get fseqfile from ACC and return bandlist bandlist = fseqfile2bandlist(fseqfile) # Use bandlist to covert nt x 50 array to nt x 34 band array of DCM attn offsets # Note that this assumes DCM offset is the same for any multiply-sampled bands # in the sequence. dcm_off = np.zeros((nt, 34), float) dcm_off[:, bandlist - 1] = dcmoff # Put into canonical order [nband, nt] dcm_off = dcm_off.T if dt: # If we want other than full cadence, find mean over dt measurements new_nt = len(times) dcm_off = dcm_off[:, :new_nt * dt] dcm_off.shape = (34, dt, new_nt) dcm_off = np.mean(dcm_off, 1) else: print 'Error reading DCM attenuations:', msg dcm_off = None else: print 'Error reading DPPon state:', msg dcm_off = None cursor.close() return { 'times': times, 'hlev': hlev.astype(int), 'vlev': vlev.astype(int), 'dcmattn': dcmattn, 'dcmoff': dcm_off }
def TPcal(x, y, calfac, offsun): ''' Writes Total Power calibration factors and offsun IF level to SQL database (caltype = 1) ''' # ******* # Version has to be updated at same time as xml description in cal_header.py # ******* version = 1.0 fghz = x['fghz'] nf = len(fghz) tstamp = x['ut'][0] # Start time of SOLPNTCAL dims = calfac.shape buf = '' if nf == 448: buf = struct.pack('d', tstamp) buf += struct.pack('d', version) # Case of 448 frequencies only # Array dimension for frequency list buf += struct.pack('I', 448) # Frequency list buf += struct.pack('448f', *fghz) # Polarization array (dimension, then two states--XX, YY) buf += struct.pack('Iii', *[2, -5, -6]) # Array dimension for Antenna cluster (2.1 m ants only) buf += struct.pack('I', 13) # Empty array for filling in for missing antennas empty = np.zeros(448, 'float') for i in range(dims[2]): # Array dimensions for freq/poln for this antenna buf += struct.pack('2I', *[448, 2]) # Cal factors for the two polarizations buf += struct.pack('448f', *calfac[0, :, i]) buf += struct.pack('448f', *calfac[1, :, i]) # Array dimensions for freq/poln for this antenna buf += struct.pack('2I', *[448, 2]) # Offsun IF level for the two polarizations buf += struct.pack('448f', *offsun[0, :, i]) buf += struct.pack('448f', *offsun[1, :, i]) for i in range(dims[2], 13): # Same as above for missing antennas buf += struct.pack('2I', *[448, 2]) buf += struct.pack('448f', *empty) buf += struct.pack('448f', *empty) buf += struct.pack('2I', *[448, 2]) buf += struct.pack('448f', *empty) buf += struct.pack('448f', *empty) t = Time.now() timestamp = t.lv cursor = dbutil.get_cursor() cursor.execute( 'insert into aBin (Timestamp,Version,Description,Bin) values (?,?,?,?)', timestamp, 1.0 + version / 10., 'Total Power Calibration', dbutil.stateframedef.pyodbc.Binary(buf)) # ******* # NB! To retrieve these large binary data strings, one must declare text size on select, e.g. # cursor.execute('set textsize 100000 select * from aBin where version = 1.1 ') # where the given size is greater than the size desired. # ******* # Temporarily store in disk file for checking format... #f = open('/tmp/tpcal.dat','wb') #f.write(buf) #f.close() cursor.commit() cursor.close()
def offsets2ants(t,xoff,yoff,ant_str=None): ''' Given a start time (Time object) and a list of offsets output by sp_offsets() for 13 antennas, convert to pointing coefficients (multiply by 10000), add to coefficients listed in stateframe, and send to the relevant antennas. The antennas to update are specified with ant_str (defaults to no antennas, for safety). ''' def send_cmds(cmds,acc): ''' Sends a series of commands to ACC. The sequence of commands is not checked for validity! cmds a list of strings, each of which must be a valid command ''' import socket for cmd in cmds: #print 'Command:',cmd s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: s.connect((acc['host'],acc['scdport'])) s.send(cmd) time.sleep(0.01) s.close() except: print 'Error: Could not send command',cmd,' to ACC.' return oldant = [8,9,10,12] if ant_str is None: print 'No antenna list specified, so there is nothing to do!' return try: timestamp = int(Time(t,format='mjd').lv) except: print 'Error interpreting time as Time() object' return from util import ant_str2list import dbutil as db import stateframe as stf accini = stf.rd_ACCfile() acc = {'host': accini['host'], 'scdport':accini['scdport']} antlist = ant_str2list(ant_str) if antlist is None: return cursor = db.get_cursor() # Read current stateframe data (as of 10 s ago) D15data = db.get_dbrecs(cursor,dimension=15,timestamp=timestamp,nrecs=1) p1_cur, = D15data['Ante_Cont_PointingCoefficient1'] p7_cur, = D15data['Ante_Cont_PointingCoefficient7'] for i in antlist: if i in oldant: # Change sign of RA offset to be HA, for old antennas (9, 10, 11 or 13) p1_inc = int(-xoff[i]*10000) else: p1_inc = int(xoff[i]*10000) p7_inc = int(yoff[i]*10000) p1_new = p1_cur[i] + p1_inc p7_new = p7_cur[i] + p7_inc print 'Updating P1 for Ant',i+1,'P1_old =',p1_cur[i],'P1_inc =',p1_inc,'P1_new =',p1_new cmd1 = 'pointingcoefficient1 '+str(p1_new)+' ant'+str(i+1) print 'Updating P7 for Ant',i+1,'P7_old =',p7_cur[i],'P7_inc =',p7_inc,'P7_new =',p7_new cmd7 = 'pointingcoefficient7 '+str(p7_new)+' ant'+str(i+1) print 'Commands to be sent:' print cmd1 print cmd7 send_cmds([cmd1],acc) send_cmds([cmd7],acc)
def get_state_idx(trange, cycles=4, attenuator=1): #This program creates an array of shape (6, 4) which contains the # times in which each attenuator is in each state. 6 attenuators, # 4 cycles. firstorsecond = ['First', 'Second'] s = sp.Spectrogram(trange) s.docal = False s.dosub = False s.domedian = False cursor = dbutil.get_cursor() res, msg = dbutil.do_query(cursor,'select Timestamp,Ante_Fron_FEM_HPol_Atte_First,Ante_Fron_FEM_HPol_Atte_Second from fV54_vD15 where Timestamp between '+str(trange[0].lv)+' and '+str(trange[1].lv)) cursor.close() if msg == 'Success': antlist = [] for i in [0, 1, 2, 4, 8, 16]: statelist = [] for j in range(15): state, = np.where(np.logical_and(res['Ante_Fron_FEM_HPol_Atte_' + firstorsecond[attenuator]][j::15].astype('int') == i,res['Ante_Fron_FEM_HPol_Atte_' + firstorsecond[attenuator-1]][j::15].astype('int') != 0)) statelist.append(state) statelist = np.array(statelist) antlist.append(statelist) states = np.array(antlist) states = np.rollaxis(states, 1) for i in range(15): for j in range(6): states[i, j] = res['Timestamp'][i::15][states[i, j]] else: print 'failure' return None time_array = (s.time.lv+0.001).astype('int') time_list = list(time_array) attns = ['0', '1', '2', '4', '8', '16'] common_list = [] for j in range(6): # Antenna 1 is used as the reference antenna here. # Earlier versions had only indices which were shared # for attenuations AND antennas, but because of a # small timing error that occur between antennas # during the scan itself, this older version would # fail sometimes. i1, i2 = util.common_val_idx(time_array,states[0,j]) if i1.shape == i2.shape: common_ant_list = i2 else: print 'There is a problem with antenna '+str(i)+' at attenuation '+attns[j] common_list.append(common_ant_list) final_indices = [] final_indices1 = [] for i in range(6): index_list = [] for indxs in common_list[i]: try: index_list.append(time_list.index(states[0,i][indxs])) except: pass final_indices1.append(index_list) for i in range(6): indices_array = np.array(final_indices1[i]) final_indices.append(indices_array) final_indices = np.array(final_indices) rolled_indices = [] for i in range(6): rolled = np.roll(final_indices[i], -1) rolled_indices.append(rolled) subtracted_list = [] for j in range(6): subtracted_list.append(rolled_indices[j] - final_indices[j]) break_lists = [] for k in range(6): break_list = [] for indx in range(subtracted_list[k].shape[0]): if np.absolute(subtracted_list[k][indx]) <= 2: break_list.append(indx) else: break_list.append(-1) break_lists.append(break_list) for i in range(6): for indx in range(int(len(break_lists[i]))-1): try: if break_lists[i][indx] == break_lists[i][indx-1]: break_lists[i].pop(indx) except: pass break_list = [] for j in range(6): breaklist = np.array(break_lists[j]) break_list.append(breaklist) break_spots = [] for i in range(6): try: break_spot = [] for indx in range(len(break_list[i])): if break_list[i][indx] == -1: break_spot.append(indx) break_spots.append(break_spot) except: pass split_lists = [] for k in range(6): steps_list = [break_list[k][0:break_spots[k][0]]] for j in range(cycles-1): try: steps_list.append(break_list[k][1 + break_spots[k][j]:break_spots[k][j+1]]) except: pass split_lists.append(steps_list) split_lists = np.array(split_lists) final_grouped_indices = [] for i in range(6): grouped_indices = [] for j in range(cycles): try: indices_ = [] for indxs in split_lists[i][j]: indices_.append(rolled_indices[i][indxs]) grouped_indices.append(indices_) except: pass final_grouped_indices.append(grouped_indices) final_grouped_indices = np.array(final_grouped_indices) for i in range(6): for j in range(cycles): try: for k in range(1,int(len(final_grouped_indices[i][j]))-1): try: for m in range(len(final_ped_indices[i][j])): if (final_grouped_indices[i][j][k-1] + 3) <= final_grouped_indices[i][j][k]: final_grouped_indices[i][j].pop(k-1) if (final_grouped_indices[i][j][k+1]-3) >= final_grouped_indices[i][j][k]: final_grouped_indices[i][j].pop(k+1) except: pass except: pass return final_grouped_indices, res
def DCM_attn_anal(filename): ''' Analyze a DCMATTNTEST observation to determine the 2- and 4-bit attenuation values. Input is a Miriad file. Returns two arrays, at2 and at4 of size (nant,npol) = (13,2) representing the attenuation, in dB, of the 2- and 4-bit, resp. ''' import read_idb as ri import dbutil as db import cal_header as ch import stateframe as stf import copy from util import Time import matplotlib.pylab as plt out = ri.read_idb([filename]) ts = int(Time(out['time'][0], format='jd').lv + 0.5) te = int(Time(out['time'][-1], format='jd').lv + 0.5) query = 'select Timestamp,DCM_Offset_Attn from fV65_vD15 where Timestamp between ' + str( ts) + ' and ' + str(te) + ' order by Timestamp' cursor = db.get_cursor() data, msg = db.do_query(cursor, query) cursor.close() dcm_offset = data['DCM_Offset_Attn'].reshape( len(data['DCM_Offset_Attn']) / 15, 15) dcm_offset = dcm_offset[:, 0] # All antennas are the same t = Time(out['time'][0], format='jd') xml, buf = ch.read_cal(2, t) table = stf.extract(buf, xml['Attenuation']) bandlist = ((out['fghz'] - 0.5) * 2).astype(int) tbl = table[bandlist - 1] tbl.shape = (len(bandlist), 15, 2) tbl = np.swapaxes(np.swapaxes(tbl, 0, -1), 0, 1) tbl2 = np.broadcast_to(tbl, (out['time'].shape[0], 15, 2, 134)) tbl = copy.copy(np.rollaxis(tbl2, 0, 4)) # Shape (nant,npol,nf,nt) pwr = out['p'][:15] # Shape (nant,npol,nf,nt) # Add value of dcm_offset to table for i, offset in enumerate(dcm_offset): tbl[:, :, :, i] += offset # Clip to valid attenuations tbl = np.clip(tbl, 0, 30) # Isolate good times in various attn states goodm2, = np.where(dcm_offset == -2) goodm2 = goodm2[2:-3] good2, = np.where(dcm_offset == 2) good2 = good2[2:-3] good0, = np.where(dcm_offset[goodm2[-1]:good2[0]] == 0) good0 += goodm2[-1] good0 = good0[2:-3] good4, = np.where(dcm_offset == 4) good4 = good4[2:-3] good6, = np.where(dcm_offset == 6) good6 = good6[2:-3] goodbg = good6 + 30 # Assumes FEMATTN 15 follows good6 30 s later # Perform median over good times and create pwrmed with medians # The 5 indexes correspond to dcm_offsets -2, 0, 2, 4 and 6 nant, npol, nf, nt = pwr.shape pwrmed = np.zeros((nant, npol, nf, 5)) # Do not forget to subtract the background bg = np.median(pwr[:, :, :, goodbg], 3) pwrmed[:, :, :, 0] = np.median(pwr[:, :, :, goodm2], 3) - bg pwrmed[:, :, :, 1] = np.median(pwr[:, :, :, good0], 3) - bg pwrmed[:, :, :, 2] = np.median(pwr[:, :, :, good2], 3) - bg pwrmed[:, :, :, 3] = np.median(pwr[:, :, :, good4], 3) - bg pwrmed[:, :, :, 4] = np.median(pwr[:, :, :, good6], 3) - bg good = np.array([goodm2[0], good0[0], good2[0], good4[0], good6[0]]) tbl = tbl[:, :, :, good] at2 = np.zeros((13, 2), float) at4 = np.zeros((13, 2), float) at8 = np.zeros((13, 2), float) f1, ax1 = plt.subplots(2, 13) f2, ax2 = plt.subplots(2, 13) f3, ax3 = plt.subplots(2, 13) for ant in range(13): for pol in range(2): pts = [] for i in range(4): for v in [0, 4, 8, 12, 16, 20, 24, 28]: idx, = np.where(tbl[ant, pol, :, i] == v) if len(idx) != 0: good, = np.where((tbl[ant, pol, idx, i] + 2) == tbl[ant, pol, idx, i + 1]) if len(good) != 0: pts.append(pwrmed[ant, pol, idx[good], i] / pwrmed[ant, pol, idx[good], i + 1]) pts = np.concatenate(pts) ax1[pol, ant].plot(pts, '.') ax1[pol, ant].set_ylim(0, 2) at2[ant, pol] = np.log10(np.median(pts)) * 10. pts = [] for i in range(3): for v in [0, 2, 8, 10, 16, 18, 24, 26]: idx, = np.where(tbl[ant, pol, :, i] == v) if len(idx) != 0: good, = np.where((tbl[ant, pol, idx, i] + 4) == tbl[ant, pol, idx, i + 2]) if len(good) != 0: pts.append(pwrmed[ant, pol, idx[good], i] / pwrmed[ant, pol, idx[good], i + 2]) pts = np.concatenate(pts) ax2[pol, ant].plot(pts, '.') ax2[pol, ant].set_ylim(0, 3) at4[ant, pol] = np.log10(np.median(pts)) * 10. pts = [] i = 0 for v in [0, 2, 4, 6, 16, 18, 20, 22]: idx, = np.where(tbl[ant, pol, :, i] == v) if len(idx) != 0: good, = np.where((tbl[ant, pol, idx, i] + 8) == tbl[ant, pol, idx, i + 4]) if len(good) != 0: pts.append(pwrmed[ant, pol, idx[good], i] / pwrmed[ant, pol, idx[good], i + 4]) try: pts = np.concatenate(pts) except: # Looks like there were no points for this antenna/polarization, so set to nominal attn pts = [6.30957, 6.30957, 6.30957] ax3[pol, ant].plot(pts, '.') ax3[pol, ant].set_ylim(5, 8) at8[ant, pol] = np.log10(np.median(pts)) * 10. plt.show() # Generate output table, a complex array of size (nant,npol,nbits) attn = np.zeros((16, 2, 4), np.complex) # Set to nominal values, then overwrite with measured ones for i in range(16): for j in range(2): attn[i, j] = [2.0 + 0j, 4.0 + 0j, 8.0 + 0j, 16.0 + 0j] attn[:13, :, 0] = at2 + 0j attn[:13, :, 1] = at4 + 0j attn[:13, :, 2] = at8 + 0j return attn
def TPcal(x, y, calfac, offsun): ''' Writes Total Power calibration factors and offsun IF level to SQL database (caltype = 1) ''' # ******* # Version has to be updated at same time as xml description in cal_header.py # ******* version = 1.0 fghz = x['fghz'] nf = len(fghz) tstamp = x['ut'][0] # Start time of SOLPNTCAL dims = calfac.shape buf = '' if nf == 448: buf = struct.pack('d',tstamp) buf += struct.pack('d',version) # Case of 448 frequencies only # Array dimension for frequency list buf += struct.pack('I',448) # Frequency list buf += struct.pack('448f',*fghz) # Polarization array (dimension, then two states--XX, YY) buf += struct.pack('Iii',*[2,-5,-6]) # Array dimension for Antenna cluster (2.1 m ants only) buf += struct.pack('I',13) # Empty array for filling in for missing antennas empty = np.zeros(448,'float') for i in range(dims[2]): # Array dimensions for freq/poln for this antenna buf += struct.pack('2I',*[448,2]) # Cal factors for the two polarizations buf += struct.pack('448f',*calfac[0,:,i]) buf += struct.pack('448f',*calfac[1,:,i]) # Array dimensions for freq/poln for this antenna buf += struct.pack('2I',*[448,2]) # Offsun IF level for the two polarizations buf += struct.pack('448f',*offsun[0,:,i]) buf += struct.pack('448f',*offsun[1,:,i]) for i in range(dims[2],13): # Same as above for missing antennas buf += struct.pack('2I',*[448,2]) buf += struct.pack('448f',*empty) buf += struct.pack('448f',*empty) buf += struct.pack('2I',*[448,2]) buf += struct.pack('448f',*empty) buf += struct.pack('448f',*empty) t = Time.now() timestamp = t.lv cursor = dbutil.get_cursor() cursor.execute('insert into aBin (Timestamp,Version,Description,Bin) values (?,?,?,?)', timestamp,1.0+version/10.,'Total Power Calibration',dbutil.stateframedef.pyodbc.Binary(buf)) # ******* # NB! To retrieve these large binary data strings, one must declare text size on select, e.g. # cursor.execute('set textsize 100000 select * from aBin where version = 1.1 ') # where the given size is greater than the size desired. # ******* # Temporarily store in disk file for checking format... #f = open('/tmp/tpcal.dat','wb') #f.write(buf) #f.close() cursor.commit() cursor.close()
def gain_state(trange=None): ''' Read and assemble the gain state for the given timerange from the SQL database, or for the last 10 minutes if trange is None. Returns the complex attenuation of the FEM for the timerange as an array of size (nant, npol, ntimes) [not band dependent], and the complex attenuation of the DCM for the same timerange as an array of size (nant, npol, nbands, ntimes). Also returns the time as a Time() object array. ''' from util import Time import dbutil as db from fem_attn_calib import fem_attn_update import cal_header as ch if trange is None: t = Time.now() t2 = Time(t.jd - 600. / 86400., format='jd') trange = Time([t2.iso, t.iso]) ts = trange[0].lv # Start timestamp te = trange[1].lv # End timestamp cursor = db.get_cursor() # First get FEM attenuation for timerange D15dict = db.get_dbrecs(cursor, dimension=15, timestamp=trange) DCMoffdict = db.get_dbrecs(cursor, dimension=50, timestamp=trange) DCMoff_v_slot = DCMoffdict['DCMoffset_attn'] # DCMoff_0 = D15dict['DCM_Offset_Attn'][:,0] # All ants are the same fem_attn = {} fem_attn['timestamp'] = D15dict['Timestamp'][:, 0] nt = len(fem_attn['timestamp']) junk = np.zeros([nt, 1], dtype='int') #add the non-existing antenna 16 fem_attn['h1'] = np.append(D15dict['Ante_Fron_FEM_HPol_Atte_First'], junk, axis=1) #FEM hpol first attn value fem_attn['h2'] = np.append(D15dict['Ante_Fron_FEM_HPol_Atte_Second'], junk, axis=1) #FEM hpol second attn value fem_attn['v1'] = np.append(D15dict['Ante_Fron_FEM_VPol_Atte_First'], junk, axis=1) #FEM vpol first attn value fem_attn['v2'] = np.append(D15dict['Ante_Fron_FEM_VPol_Atte_Second'], junk, axis=1) #FEM vpol second attn value fem_attn['ants'] = np.append(D15dict['I15'][0, :], [15]) # Add corrections from SQL database for start time of timerange fem_attn_corr = fem_attn_update(fem_attn, trange[0]) # Next get DCM attenuation for timerange # Getting next earlier scan header ver = db.find_table_version(cursor, ts, True) query = 'select top 50 Timestamp,FSeqList from hV' + ver + '_vD50 where Timestamp <= ' + str( ts) + ' order by Timestamp desc' fseq, msg = db.do_query(cursor, query) if msg == 'Success': fseqlist = fseq['FSeqList'][::-1] # Reverse the order bandlist = ((np.array(fseqlist) - 0.44) * 2).astype(int) cursor.close() # Read current DCM_table from database xml, buf = ch.read_cal(3, trange[0]) orig_table = stf.extract(buf, xml['Attenuation']).astype('int') orig_table.shape = (50, 15, 2) xml, buf = ch.read_cal(6, trange[0]) dcm_attn_bitv = np.nan_to_num(stf.extract( buf, xml['DCM_Attn_Real'])) + np.nan_to_num( stf.extract(buf, xml['DCM_Attn_Imag'])) * 1j # # Add one more bit (all zeros) to take care of unit bit # dcm_attn_bitv = np.concatenate((np.zeros((16,2,1),'int'),dcm_attn_bitv),axis=2) # We now have: # orig_table the original DCM at start of scan, size (nslot, nant=15, npol) # DCMoff_0 the offset applied to all antennas and slots (ntimes) # DCMoff_v_slot the offest applied to all antennas but varies by slot (ntimes, nslot) # dcm_attn_bitv the measured (non-nominal) attenuations for each bit value (nant=16, npol, nbit) -- complex # Now I need to convert slot to band, add appropriately, and organize as (nant=16, npol, nband, ntimes) # Add one more antenna (all zeros) to orig_table orig_table = np.concatenate((orig_table, np.zeros((50, 1, 2), 'int')), axis=1) ntimes, nslot = DCMoff_v_slot.shape dcm_attn = np.zeros((16, 2, 34, ntimes), np.int) for i in range(ntimes): for j in range(50): idx = bandlist[j] - 1 # This adds attenuation for repeated bands--hopefully the same value for each repeat dcm_attn[:, :, idx, i] += orig_table[j, :, :] + DCMoff_v_slot[i, j] # Normalize repeated bands by finding number of repeats and dividing. for i in range(1, 35): n = len(np.where(bandlist == i)[0]) if n > 1: dcm_attn[:, :, i - 1, :] /= n # Make sure attenuation is in range dcm_attn = np.clip(dcm_attn, 0, 30) # Finally, correct for non-nominal (measured) bit values # Start with 0 attenuation as reference dcm_attn_corr = dcm_attn * (0 + 0j) att = np.zeros((16, 2, 34, ntimes, 5), np.complex) # Calculate resulting attenuation based on bit attn values (2,4,8,16) for i in range(4): # Need dcm_attn_bitv[...,i] to be same shape as dcm_attn bigger_bitv = np.broadcast_to(dcm_attn_bitv[..., i], (ntimes, 34, 16, 2)) bigger_bitv = np.swapaxes( np.swapaxes(np.swapaxes(bigger_bitv, 0, 3), 1, 2), 0, 1) att[..., i] = (np.bitwise_and(dcm_attn, 2**(i + 1)) >> (i + 1)) * bigger_bitv dcm_attn_corr = dcm_attn_corr + att[..., i] # Move ntimes column to next to last position, and then sum over last column (the two attenuators) fem_attn_corr = np.sum(np.rollaxis(fem_attn_corr, 0, 3), 3) # Output is FEM shape (nant, npol, ntimes) = (16, 2, ntimes) # DCM shape (nant, npol, nband, ntimes) = (16, 2, 34, ntimes) # Arrays are complex, in dB units tjd = Time(fem_attn['timestamp'].astype('int'), format='lv').jd return fem_attn_corr, dcm_attn_corr, tjd
def tp_bgnd_all(tpdata): ''' Create time-variable background from ROACH inlet temperature This version is far superior to the earlier, crude version, but beware that it works best for a long timerange of data, especially when there is a flare in the data. Inputs: tpdata dictionary returned by read_idb() NB: tpdata is not changed. Returns: bgnd The background fluctuation array of size (nf,nt) to be subtracted from any antenna's total power (or mean of antenna total powers) ''' import dbutil as db from util import Time, nearest_val_idx outfghz = tpdata['fghz'] try: outtime = tpdata['time'] trange = Time(outtime[[0, -1]], format='jd') except: outtime = tpdata['ut_mjd'] trange = Time(outtime[[0, -1]], format='mjd') nt = len(outtime) if nt < 1200: print 'TP_BGND: Error, timebase too small. Must have at least 1200 time samples.' return None nf = len(outfghz) outpd = Time(outtime, format='jd').plot_date cursor = db.get_cursor() data = db.get_dbrecs(cursor, dimension=8, timestamp=trange) pd = Time(data['Timestamp'][:, 0].astype(int), format='lv').plot_date inlet = data['Sche_Data_Roac_TempInlet'] # Inlet temperature variation sinlet = np.sum(inlet.astype(float), 1) # Eliminate 0 values in sinlet by replacing with nearest good value bad, = np.where(sinlet == 0) good, = np.where(sinlet != 0) idx = nearest_val_idx( bad, good) # Find locations of nearest good values to bad ones sinlet[bad] = sinlet[good[idx]] # Overwrite bad values with good ones sinlet -= np.mean( sinlet) # Remove offset, to provide zero-mean fluctuation sinlet = np.roll( sinlet, -110) # Shift phase of variation by 110 s earlier (seems to be needed) # Interpolate sinlet values to the times in the data sint = np.interp(outpd, pd, sinlet) # sint = np.roll(sint,-90) # Shift phase of variation by 90 s earlier # sint -= np.mean(sint) # Remove offset, to provide zero-mean fluctuation sdev = np.std(sint) sint_ok = np.abs(sint) < 2 * sdev bgnd = np.zeros((13, 2, nf, nt), float) for ant in range(13): for pol in range(2): for i in range(nf): # Subtract smooth trend from data nt = len(tpdata['p'][ant, pol, i]) wlen = min(nt, 2000) if wlen % 2 != 0: wlen -= 1 sig = tpdata['p'][ant, pol, i] - smooth( tpdata['p'][ant, pol, i], wlen, 'blackman')[wlen / 2:-(wlen / 2 - 1)] # Eliminate the worst outliers and repeat stdev = np.nanstd(sig) good, = np.where(np.abs(sig) < 2 * stdev) if len(good) > nt * 0.1: wlen = min(len(good), 2000) if wlen % 2 != 0: wlen -= 1 sig = tpdata['p'][ant, pol, i, good] - smooth( tpdata['p'][ant, pol, i, good], wlen, 'blackman')[wlen / 2:-(wlen / 2 - 1)] sint_i = sint[good] stdev = np.std(sig) # Final check for data quality good, = np.where( np.logical_and(sig < 2 * stdev, sint_ok[good])) if len(good) > nt * 0.1: p = np.polyfit(sint_i[good], sig[good], 1) else: p = [1., 0.] # Apply correction for this frequency bgnd[ant, pol, i] = sint * p[0] + p[1] return bgnd