def sql2rstn(t=None): """This function extracts the RSTN data from SQL with SQL_timestamp 0300 on the date supplied. If the values could be extracted then the data is returned in a list as follows: 0 - timestamp: Astropy Time which is the date on which the data was collected. This should match dt 1 - freq: A float32 numpy array containing the 9 frequencies in GHz 2 - data: The flux data which is a 9x7 int16 numpy array. SQL_timestamp is also returned.""" if t is None: t = Time.now() sqlt = Time(np.floor(t.mjd) + 0.125, format='mjd') xml, buf = ch.read_cal(12, sqlt) if buf is None: return None, None sqlt_read = Time(extract(buf, xml['SQL_timestamp']), format='lv') #if np.floor(sqlt.mjd) != np.floor(sqlt_read.mjd): return None, None data = [] data.append(Time(extract(buf, xml['Timestamp']), format='lv')) data.append(extract(buf, xml['FGHz'])) data.append(extract(buf, xml['Flux'])) return data, sqlt_read
def unrot_refcal(refcal_in): ''' Apply feed-rotation correction to data read with rd_refcal(), returning updated data in the same format for further processing. ''' import dbutil as db import copy import chan_util_bc as cu import cal_header as ch from stateframe import extract refcal = copy.deepcopy(refcal_in) xml, buf = ch.read_cal(11, Time(refcal['times'][0][0], format='jd')) dph = extract(buf, xml['XYphase']) xi_rot = extract(buf, xml['Xi_Rot']) freq = extract(buf, xml['FGHz']) freq = freq[np.where(freq != 0)] band = [] for f in freq: band.append(cu.freq2bdname(f)) bds, sidx = np.unique(band, return_index=True) nbd = len(bds) eidx = np.append(sidx[1:], len(band)) dxy = np.zeros((14, 34), dtype=np.float) xi = np.zeros(34, dtype=np.float) fghz = np.zeros(34) # average dph and xi_rot frequencies within each band, to convert to 34-band representation for b, bd in enumerate(bds): fghz[bd - 1] = np.nanmean(freq[sidx[b]:eidx[b]]) xi[bd - 1] = np.nanmean(xi_rot[sidx[b]:eidx[b]]) for a in range(14): dxy[a, bd - 1] = np.angle(np.sum(np.exp(1j * dph[a, sidx[b]:eidx[b]]))) nscans = len(refcal['scanlist']) for i in range(nscans): # Read parallactic angles for this scan trange = Time([refcal['tstlist'][i].iso, refcal['tedlist'][i].iso]) times, chi = db.get_chi(trange) tchi = times.jd t = refcal['times'][i] if len(t) > 0: vis = copy.deepcopy(refcal['vis'][i]) idx = nearest_val_idx(t, tchi) pa = chi[idx] # Parallactic angle for the times of this refcal. pa[:, [8, 9, 10, 12]] = 0.0 nt = len(idx) # Number of times in this refcal # Apply X-Y delay phase correction for a in range(13): a1 = lobe(dxy[a] - dxy[13]) a2 = -dxy[13] - xi a3 = dxy[a] - xi + np.pi for j in range(nt): vis[a, 1, :, j] *= np.exp(1j * a1) vis[a, 2, :, j] *= np.exp(1j * a2) vis[a, 3, :, j] *= np.exp(1j * a3) for j in range(nt): for a in range(13): refcal['vis'][i][a, 0, :, j] = vis[a, 0, :, j] * np.cos(pa[j, a]) + vis[a, 3, :, j] * np.sin(pa[j, a]) refcal['vis'][i][a, 2, :, j] = vis[a, 2, :, j] * np.cos(pa[j, a]) + vis[a, 1, :, j] * np.sin(pa[j, a]) refcal['vis'][i][a, 3, :, j] = vis[a, 3, :, j] * np.cos(pa[j, a]) - vis[a, 0, :, j] * np.sin(pa[j, a]) refcal['vis'][i][a, 1, :, j] = vis[a, 1, :, j] * np.cos(pa[j, a]) - vis[a, 2, :, j] * np.sin(pa[j, a]) return refcal
def get_calfac(t=None): ''' Read total power and auto-correlation calibration factors from the SQL database, for the time specified by Time() object t, or if None, at the next earlier calibration time to the current time. ''' tpcal_type = 10 # Calibration type specified in cal_header.py if t is None: t = Time.now() xml, buf = ch.read_cal(tpcal_type, t=t) fghz = stateframe.extract(buf, xml['FGHz']) nf = len(fghz) tpcalfac = np.zeros((13, 2, nf), np.float) tpoffsun = np.zeros((13, 2, nf), np.float) accalfac = np.zeros((13, 2, nf), np.float) acoffsun = np.zeros((13, 2, nf), np.float) nant = len(xml['Antenna']) for i in range(nant): iant = stateframe.extract(buf, xml['Antenna'][i]['Antnum']) - 1 tpcalfac[iant] = stateframe.extract(buf, xml['Antenna'][i]['TPCalfac']) accalfac[iant] = stateframe.extract(buf, xml['Antenna'][i]['ACCalfac']) tpoffsun[iant] = stateframe.extract(buf, xml['Antenna'][i]['TPOffsun']) acoffsun[iant] = stateframe.extract(buf, xml['Antenna'][i]['ACOffsun']) try: sqltime = stateframe.extract(buf, xml['SQL_timestamp']) except: sqltime = None return { 'fghz': fghz, 'timestamp': stateframe.extract(buf, xml['Timestamp']), 'sqltime': sqltime, 'tpcalfac': tpcalfac, 'accalfac': accalfac, 'tpoffsun': tpoffsun, 'acoffsun': acoffsun }
def sql2refcal(t, lohi=False): '''Supply a timestamp in Time format, return the closest refcal data''' import cal_header as ch import stateframe as stf if lohi: caltype = 12 else: caltype = 8 xml, buf = ch.read_cal(caltype, t=t) refcal = stf.extract( buf, xml['Refcal_Real']) + stf.extract(buf, xml['Refcal_Imag']) * 1j flag = stf.extract(buf, xml['Refcal_Flag']) fghz = stf.extract(buf, xml['Fghz']) sigma = stf.extract(buf, xml['Refcal_Sigma']) timestamp = Time(stf.extract(buf, xml['Timestamp']), format='lv') tbg = Time(stf.extract(buf, xml['T_beg']), format='lv') ted = Time(stf.extract(buf, xml['T_end']), format='lv') pha = np.angle(refcal) amp = np.absolute(refcal) return { 'pha': pha, 'amp': amp, 'flag': flag, 'fghz': fghz, 'sigma': sigma, 'timestamp': timestamp, 't_bg': tbg, 't_ed': ted }
def get_gain_corr(trange, tref=None, fghz=None): ''' Calls get_gain_state() for a timerange and a reference time, and returns the gain difference table to apply to data in the given timerange. If no reference time is provided, the gain state is referred to the nearest earlier REFCAL. Returns a dictionary containing: antgain Array of size (15, 2, 34, nt) = (nant, npol, nbands, nt) times A Time() object corresponding to the times in antgain ''' if tref is None: # No reference time specified, so get nearest earlier REFCAL xml, buf = ch.read_cal(8,t=trange[0]) tref = Time(stf.extract(buf,xml['Timestamp']),format='lv') # Get the gain state at the reference time (actually median over 1 minute) trefrange = Time([tref.iso,Time(tref.lv+61,format='lv').iso]) ref_gs = get_gain_state(trefrange) # refcal gain state for 60 s # Get median of refcal gain state (which should be constant anyway) ref_gs['h1'] = np.median(ref_gs['h1'],1) ref_gs['h2'] = np.median(ref_gs['h2'],1) ref_gs['v1'] = np.median(ref_gs['v1'],1) ref_gs['v2'] = np.median(ref_gs['v2'],1) # Get the gain state of the requested timerange src_gs = get_gain_state(trange) # solar gain state for timerange of file nt = len(src_gs['times']) antgain = np.zeros((15,2,34,nt),np.float32) # Antenna-based gains vs. band for i in range(15): for j in range(34): antgain[i,0,j] = src_gs['h1'][i] + src_gs['h2'][i] - ref_gs['h1'][i] - ref_gs['h2'][i] + src_gs['dcmattn'][i,0,j] - ref_gs['dcmattn'][i,0,j] antgain[i,1,j] = src_gs['v1'][i] + src_gs['v2'][i] - ref_gs['v1'][i] - ref_gs['v2'][i] + src_gs['dcmattn'][i,1,j] - ref_gs['dcmattn'][i,1,j] return {'antgain': antgain, 'times': src_gs['times']}
def acc2sql(): ''' This is just a test version to read the stateframe once a second from the ACC and send it to the SQL server. A more complete version of this is implemented in schedule.py for "production" work. ''' # Get stateframe structure and version accini = stateframe.rd_ACCfile() sf_version = accini['version'] brange, outlist = sfdef(accini['sf']) with pyodbc.connect("DRIVER={FreeTDS};SERVER=192.168.24.106,1433; \ DATABASE=eOVSA06;UID=aaa;PWD=I@bsbn2w;") as cnxn: cursor = cnxn.cursor() lineno = 0 while 1: lineno += 1 data, msg = stateframe.get_stateframe(accini) version = stateframe.extract(data,['d',8]) if version == sf_version: bufout = transmogrify(data, brange) try: cursor.execute('insert into fBin (Bin) values (?)',pyodbc.Binary(bufout)) print 'Record '+str(lineno)+' successfully written\r', cnxn.commit() except: # An exception could be an error, or just that the entry was already inserted pass else: print 'Error: Incompatible version in stateframe.' break time.sleep(1)
def fseq_is_running(fseqfile, accini=None): ''' Check current stateframe to see if the given sequence file is running. Returns True if so, False otherwise ''' import stateframe as stf if accini is None: accini = stf.rd_ACCfile() # Make sure this sequence is actually running, or start it if not buf, msg = stf.get_stateframe(accini) if msg != 'No Error': print 'Error reading stateframe.' return None fseq = stf.extract(buf, accini['sf']['LODM']['LO1A']['FSeqFile']) fseq = fseq.strip('\x00') # strip nulls from name result = fseq == fseqfile and stf.extract( buf, accini['sf']['LODM']['LO1A']['SweepStatus']) == 8 return result
def rstntext2sql(startdt, enddt, logfile=None): """This routine extracts data from the old archive text file and writes it to SQL. It will output a list of dates that were not archived. The SQL time will be the date of the data at 0300. The program checks to see if there is current data already for the date range. It will NOT overwrite a record if it is already present. startdt and enddt are the dates that will be written to SQL from startdt up to but not including enddt.""" data = rstnfluxfromtextarchive(startdt, enddt) if data is None: print "No RSTN data found in range ", startdt.iso, " to ", enddt.iso return print "Processing data from: ", startdt.iso, " to ", enddt.iso offset = int(np.floor(startdt.mjd)) days = int(np.floor(enddt.mjd)) - offset processed = np.zeros((days), dtype=bool) recordswritten = 0 existingrecords = 0 for d in data: print "Processing Date: ", d[0].iso i = int(np.floor(d[0].mjd)) - offset processed[i] = True sqltime = Time(np.floor(d[0].mjd) + 0.125, format='mjd') xml, buf = ch.read_cal(12, sqltime) if buf is not None: sqltime_read = Time(extract(buf, xml['SQL_timestamp']), format='lv') if np.floor(sqltime_read.mjd) != np.floor(sqltime.mjd): buf = None if buf is None: if ch.rstnflux2sql(d, sqltime): recordswritten += 1 print "Record Written" else: print "Record Write Failed!" else: print "Record Exists." existingrecords += 1 if logfile is None: logfile = "/tmp/missingrstn.txt" f = open(logfile, "w") for i in range(days): if not processed[i]: f.write(Time(float(i + offset) + 0.125, format='mjd').iso + "\n") f.close() print "Number of days searched: ", days print "Number of existing records: ", existingrecords print "Records Written: ", recordswritten print "Missing Records: ", days - (recordswritten + existingrecords)
def DCM_cal(filename=None,fseqfile='gainseq.fsq',dcmattn=None,missing='ant15',update=False): if filename is None: return 'Must specify ADC packet capture filename, e.g. "/dppdata1/PRT/PRT<yyyymmddhhmmss>adc.dat"' userpass = '******' fseq_handle = urllib2.urlopen('ftp://'+userpass+'acc.solar.pvt/parm/'+fseqfile,timeout=0.5) lines = fseq_handle.readlines() fseq_handle.close() for line in lines: if line.find('LIST:SEQUENCE') != -1: line = line[14:] bandlist = np.array(map(int,line.split(','))) if len(np.unique(bandlist)) != 34: print 'Frequency sequence must contain all bands [1-34]' return None # Read packet capture file adc = p.rd_jspec(filename) pwr = np.rollaxis(adc['phdr'],2)[:,:,:2] # Put measured power into uniform array arranged by band new_pwr = np.zeros((34,16,2)) for i in range(34): idx, = np.where(bandlist-1 == i) if len(idx) > 0: new_pwr[i] = np.median(pwr[idx],0) new_pwr.shape = (34,32) # Read table from the database. import cal_header import stateframe xml, buf = cal_header.read_cal(2) cur_table = stateframe.extract(buf,xml['Attenuation']) if dcmattn: # A DCM attenuation value was given, which presumes a constant value # so use it as the "original table." orig_table = np.zeros((34,30)) + dcmattn # orig_table[:,26:28] = 24 orig_table[:,28:] = 0 else: # No DCM attenuation value was given, so use current DCM master # table from the database. orig_table = cur_table attn = np.log10(new_pwr[:,:-2]/1600.)*10. # Zero any changes for missing antennas, and override orig_table with cur_table for those antennas if missing: idx = p.ant_str2list(missing) bad = np.sort(np.concatenate((idx*2,idx*2+1))) attn[:,bad] = 0 orig_table[:,bad] = cur_table[:,bad] new_table = (np.clip(orig_table + attn,0,30)/2).astype(int)*2 DCMlines = [] DCMlines.append('# Ant1 Ant2 Ant3 Ant4 Ant5 Ant6 Ant7 Ant8 Ant9 Ant10 Ant11 Ant12 Ant13 Ant14 Ant15') DCMlines.append('# X Y X Y X Y X Y X Y X Y X Y X Y X Y X Y X Y X Y X Y X Y X Y') DCMlines.append('# ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- -----') for band in range(1,35): DCMlines.append('{:2} : {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2}'.format(band,*new_table[band-1])) return DCMlines
def read_dbcalfac(t): ''' Read the contents of a SOLPNT calibration file from the SQL database, and return fghz, calfac, offsun arrays ''' import stateframe import cal_header as ch try: tp, buf = ch.read_cal(1, t) except: print 'READ_DBCALFAC: Error reading calibration factors' return None, None, None fghz = stateframe.extract(buf, tp['FGHz']) poln = stateframe.extract(buf, tp['Poln']) nf = len(fghz) npol = len(poln) nant = len(tp['Antenna']) calfac = np.zeros((npol, nf, nant), 'float') offsun = np.zeros((npol, nf, nant), 'float') for i, ant in enumerate(tp['Antenna']): calfac[:, :, i] = stateframe.extract(buf, ant['Calfac']) offsun[:, :, i] = stateframe.extract(buf, ant['Offsun']) idx = np.isfinite(calfac[0, :, 0]) return fghz[idx], calfac[:, idx, :], offsun[:, idx, :]
def sql2refcal(t): '''Supply a timestamp in Time format, return the closest refcal data''' import cal_header as ch import stateframe as stf xml, buf = ch.read_cal(8, t=t) refcal = stf.extract(buf, xml['Refcal_Real']) + stf.extract(buf, xml['Refcal_Imag']) * 1j flag = stf.extract(buf, xml['Refcal_Flag']) fghz = stf.extract(buf, xml['Fghz']) sigma = stf.extract(buf, xml['Refcal_Sigma']) timestamp = Time(stf.extract(buf, xml['Timestamp']), format='lv') tbg = Time(stf.extract(buf, xml['T_beg']), format='lv') ted = Time(stf.extract(buf, xml['T_end']), format='lv') pha = np.angle(refcal) amp = np.absolute(refcal) return {'pha': pha, 'amp': amp, 'flag': flag, 'fghz': fghz, 'sigma': sigma, 'timestamp': timestamp, 't_bg': tbg, 't_ed': ted}
def read_dbcalfac(t): ''' Read the contents of a SOLPNT calibration file from the SQL database, and return fghz, calfac, offsun arrays ''' import stateframe import cal_header as ch try: tp, buf = ch.read_cal(1,t) except: print 'READ_DBCALFAC: Error reading calibration factors' return None, None, None fghz = stateframe.extract(buf,tp['FGHz']) poln = stateframe.extract(buf,tp['Poln']) nf = len(fghz) npol = len(poln) nant = len(tp['Antenna']) calfac = np.zeros((npol,nf,nant),'float') offsun = np.zeros((npol,nf,nant),'float') for i,ant in enumerate(tp['Antenna']): calfac[:,:,i] = stateframe.extract(buf,ant['Calfac']) offsun[:,:,i] = stateframe.extract(buf,ant['Offsun']) idx = np.isfinite(calfac[0,:,0]) return fghz[idx], calfac[:,idx,:], offsun[:,idx,:]
def get_val_from_stflog(filename,loc): arr = False if len(loc) == 3: print 'Cannot plot multiple values--will plot only index 0' arr = True with open(filename,'rb') as f: data = f.read(100) recsize = struct.unpack_from('i',data,16)[0] tstamp = stateframe.extract(data,['d',0]) t = Time(tstamp,format='lv') f.seek(0) data = f.read(recsize) i = 0 nrec = os.stat(filename).st_size/recsize tm = np.zeros(nrec,'float') val = np.zeros(nrec,'float') while len(data) == recsize: # Read timestamps tm[i] = stateframe.extract(data,['d',0]) if arr: val[i] = stateframe.extract(data,loc)[0] else: val[i] = stateframe.extract(data,loc) data = f.read(recsize) i += 1 # Convert timestamps to Time() array times = Time(tm,format='lv') bad = np.where(val == 0.0)[0] val[bad] = np.NaN tm[bad] = np.NaN fig = plt.figure() sub_plot = fig.add_subplot(111) sub_plot.plot_date(times.plot_date,val,'+') fmt = plt.DateFormatter('%H:%M') sub_plot.xaxis.set_major_formatter(fmt) plt.title(t.iso[0:10]) return times,val
def chk_lo1a(accini, band, iteration=1): data, msg = stf.get_stateframe(accini) errstr = stf.extract(data,accini['sf']['LODM']['LO1A']['ERR']).split('"')[1] if iteration == 1 and errstr != 'No error': # Looks like a reboot of LO1A is needed! print '10-s delay while attempting to reboot LO1A' send_cmds(['LO1A-REBOOT'],acc) time.sleep(10) acc = {'host': accini['host'], 'scdport':accini['scdport']} acc_tune(band+1,acc) time.sleep(5) errstr = chk_lo1a(accini,band,iteration=2) if errstr != 'No error': errstr = 'Reboot attempt failed.' return errstr
def read_attncal(trange=None): ''' Given a timerange as a Time() object, read FEM attenuation records for each date from the SQL database, and return then as a list of attn dictionaries. To get values for only a single day, the trange Time() object can have the same time repeated, or can be a single time. Returns a list of dictionaries, each pertaining to one of the days in trange, with keys defined as follows: 'time': The start time of the GAINCALTEST scan, as a Time() object 'fghz': The list of frequencies [GHz] at which attenuations are measured 'attn': The array of attenuations [dB] of size (nattn, nant, npol, nf), where nattn = 8, nant = 13, npol = 2, and nf is variable ''' import cal_header as ch import stateframe as stf if trange is None: trange = Time.now() if type(trange.mjd) == np.float: # Interpret single time as both start and end time mjd1 = int(trange.mjd) mjd2 = mjd1 else: mjd1, mjd2 = trange.mjd.astype(int) attn = [] for mjd in range(mjd1, mjd2 + 1): # Read next earlier SQL entry from end of given UT day (mjd+0.999) xml, buf = ch.read_cal(7, t=Time(mjd + 0.999, format='mjd')) t = Time(stf.extract(buf, xml['Timestamp']), format='lv') fghz = stf.extract(buf, xml['FGHz']) nf = len(np.where(fghz != 0.0)[0]) fghz = fghz[:nf] attnvals = stf.extract(buf, xml['FEM_Attn_Real'])[:, :, :, :nf] attn.append({'time': t, 'fghz': fghz, 'attn': attnvals}) return attn
def set_fem_attn(level=3, ant_str='ant1-15'): ''' Read current power and attenuation values in FEMs, and set the attenuation such that the power level will be as close to "level" as possible. ''' accini = stf.rd_ACCfile() acc = {'host': accini['host'], 'scdport': accini['scdport']} ant_list = ant_str2list(ant_str) if ant_list is None: return 'Bad antenna list' accini = stf.rd_ACCfile() hatn1 = np.zeros((10, 15), dtype='int') vatn1 = np.zeros((10, 15), dtype='int') hatn2 = np.zeros((10, 15), dtype='int') vatn2 = np.zeros((10, 15), dtype='int') hpwr = np.zeros((10, 15), dtype='float') vpwr = np.zeros((10, 15), dtype='float') # Read 10 instances of attenuation and power, and take the median to avoid # glitches for i in range(10): # Read the frontend attenuations and powers for each antenna data, msg = stf.get_stateframe(accini) for iant in range(15): fem = accini['sf']['Antenna'][iant]['Frontend']['FEM'] hatn1[i, iant] = stf.extract(data, fem['HPol']['Attenuation']['First']) vatn1[i, iant] = stf.extract(data, fem['VPol']['Attenuation']['First']) hatn2[i, iant] = stf.extract(data, fem['HPol']['Attenuation']['Second']) vatn2[i, iant] = stf.extract(data, fem['VPol']['Attenuation']['Second']) hpwr[i, iant] = stf.extract(data, fem['HPol']['Power']) vpwr[i, iant] = stf.extract(data, fem['VPol']['Power']) time.sleep(1) hatn1 = np.median(hatn1, 0).astype('int') vatn1 = np.median(vatn1, 0).astype('int') hatn2 = np.median(hatn2, 0).astype('int') vatn2 = np.median(vatn2, 0).astype('int') hpwr = np.median(hpwr, 0) vpwr = np.median(vpwr, 0) hatn2 = np.clip(hatn2 - (level - hpwr).astype('int'), 0, 31) vatn2 = np.clip(vatn2 - (level - vpwr).astype('int'), 0, 31) # Send attenuation commands to each antenna in ant_list for iant in ant_list: hatn = str(hatn1[iant]) + ' ' + str( hatn2[iant]) + ' ant' + str(iant + 1) vatn = str(vatn1[iant]) + ' ' + str( vatn2[iant]) + ' ant' + str(iant + 1) send_cmds(['HATTN ' + hatn, 'VATTN ' + vatn], acc) return 'Success'
def compare_tbl(tbl, t=None): ''' Compare the given table with that from the SQL database. Inputs: tbl A text string version of a table as returned by DCM_calnew() t An optional Time() object giving a date/time of the table to compare to. If omitted or None, the currently active table is used. Note, the exact date is not required. Any date after the desired SQL record will return the active table for that date. ''' import cal_header import stateframe from copy import deepcopy xml, buf = cal_header.read_cal(2, t=t) cur_table = stateframe.extract(buf,xml['Attenuation']) diftbl = deepcopy(tbl) # Make a copy to update, to preserve structure for i in range(52): dif = map(int,tbl[i+3][4:].split()) - cur_table[i] diftbl[3+i] = '{:2} : {:3}{:3}{:3}{:3}{:3}{:3}{:3}{:3}{:3}{:3}{:3}{:3}{:3}{:3}{:3}{:3}{:3}{:3}{:3}{:3}{:3}{:3}{:3}{:3}{:3}{:3}{:3}{:3}{:3}{:3}'.format(i,*dif) return diftbl
def bandlist2dcmtable(bandlist, toACC=False): '''Use list of bands representing a frequency sequence, to set dcmtable.txt from the DCM_master_table, and return the lines of the table. Optionally, the table is sent to ACC. Input: bandlist numpy 50-element integer array of band numbers, 1-34 toACC optional boolean. If True, sends results to ACC and the SQL database. Default is False (does not send) ''' import stateframe as stf import cal_header as ch from ftplib import FTP # Convert from 1-based bandlist to zero-based band numbers bands = bandlist - 1 # Read master table from SQL server dcm, buf = ch.read_cal(2) dcm_m_attn = stf.extract(buf, dcm['Attenuation']) dcm_attn = dcm_m_attn[bands] lines = [] g = open('/tmp/DCM_table.txt', 'w') for line in dcm_attn: l = ' '.join(map(str, line)) lines.append(l) g.write(l + '\n') g.close() if toACC: ch.dcm_table2sql(lines) # Connect to ACC /parm directory and transfer scan_header files try: g = open('/tmp/DCM_table.txt', 'r') acc = FTP('acc.solar.pvt') acc.login('admin', 'observer') acc.cwd('parm') # Send DCM table lines to ACC print acc.storlines('STOR dcm.txt', g) g.close() print 'Successfully wrote dcm.txt to ACC' except: print 'Cannot FTP dcm.txt to ACC' return lines
def override(tbl, bandlist, t=None): ''' When a table has bad attenuation for a given band, this command will replace the new values with the ones in the current DCM_master_table. Inputs: tbl A text string version of a table as returned by DCM_calnew() bandlist A simple list of band numbers (1 - 52) that will be replaced. t An optional Time() object giving a date/time of the table to override from. If omitted or None, the currently active table is used. Note, the exact date is not required. Any date after the desired SQL record will return the active table for that date. ''' import cal_header import stateframe xml, buf = cal_header.read_cal(2, t=t) cur_table = stateframe.extract(buf,xml['Attenuation']) if type(bandlist) != list: # If bandlist is not a list, it may be just a single band, so make it a list (of one). bandlist = [bandlist] for band in bandlist: # Format cur_table[band-1] as text line, and use it to replace tbl[band+2] line tbl[band+2] = tbl[band+2][:5]+('{:3d}'*30).format(*cur_table[band-1]) return tbl
def set_fem_attn(level=3,ant_str='ant1-15'): ''' Read current power and attenuation values in FEMs, and set the attenuation such that the power level will be as close to "level" as possible. ''' accini = stf.rd_ACCfile() acc = {'host': accini['host'], 'scdport':accini['scdport']} ant_list = ant_str2list(ant_str) if ant_list is None: return 'Bad antenna list' accini = stf.rd_ACCfile() hatn1 = np.zeros((10,15),dtype='int') vatn1 = np.zeros((10,15),dtype='int') hatn2 = np.zeros((10,15),dtype='int') vatn2 = np.zeros((10,15),dtype='int') hpwr = np.zeros((10,15),dtype='float') vpwr = np.zeros((10,15),dtype='float') # Read 10 instances of attenuation and power, and take the median to avoid # glitches for i in range(10): # Read the frontend attenuations and powers for each antenna data, msg = stf.get_stateframe(accini) for iant in range(15): fem = accini['sf']['Antenna'][iant]['Frontend']['FEM'] hatn1[i,iant] = stf.extract(data,fem['HPol']['Attenuation']['First']) vatn1[i,iant] = stf.extract(data,fem['VPol']['Attenuation']['First']) hatn2[i,iant] = stf.extract(data,fem['HPol']['Attenuation']['Second']) vatn2[i,iant] = stf.extract(data,fem['VPol']['Attenuation']['Second']) hpwr[i,iant] = stf.extract(data,fem['HPol']['Power']) vpwr[i,iant] = stf.extract(data,fem['VPol']['Power']) time.sleep(1) hatn1 = np.median(hatn1,0).astype('int') vatn1 = np.median(vatn1,0).astype('int') hatn2 = np.median(hatn2,0).astype('int') vatn2 = np.median(vatn2,0).astype('int') hpwr = np.median(hpwr,0) vpwr = np.median(vpwr,0) hatn2 = np.clip(hatn2 - (level - hpwr).astype('int'),0,31) vatn2 = np.clip(vatn2 - (level - vpwr).astype('int'),0,31) # Send attenuation commands to each antenna in ant_list for iant in ant_list: hatn = str(hatn1[iant])+' '+str(hatn2[iant])+' ant'+str(iant+1) vatn = str(vatn1[iant])+' '+str(vatn2[iant])+' ant'+str(iant+1) send_cmds(['HATTN '+hatn,'VATTN '+vatn],acc) return 'Success'
def fem_attn_update(fem_attn, t=None, rdfromsql=True): '''Given a record of the frontend attenuation levels from the stateframe, recalculate the corrected attenuations levels. fem_attn_in: recorded attn levels in a 10-min duration fem_attn_bitv: complex corrections to be applied to the data. Read from the stateframe or provided as a (16, 2, 2, 5) array''' import cal_header as ch import stateframe as stf if rdfromsql: xml, buf = ch.read_cal(7,t) fem_attn_bitv=np.nan_to_num(stf.extract(buf, xml['FEM_Attn_Real'])) + np.nan_to_num(stf.extract(buf, xml['FEM_Attn_Imag'])) * 1j h1=fem_attn['h1'] h2=fem_attn['h2'] v1=fem_attn['v1'] v2=fem_attn['v2'] attn=np.concatenate((np.concatenate((h1[...,None],v1[...,None]),axis=2)[...,None], np.concatenate((h2[...,None],v2[...,None]),axis=2)[...,None]),axis=3) # Start with 0 attenuation as reference fem_attn_out=attn*0 # Calculate resulting attenuation based on bit attn values (1,2,4,8,16) for i in range(5): fem_attn_out = fem_attn_out + (np.bitwise_and(attn,2**i)>>i)*fem_attn_bitv[...,i] #fem_gain=10.**(-fem_gain_db/10.) return fem_attn_out
def find_calibrations(year, month): import calendar from util import Time import cal_header as ch import stateframe hc = calendar.HTMLCalendar(calendar.SUNDAY) html_table = hc.formatmonth(year, month) lines = html_table.split('\n') lines[2] = lines[2].replace('th class', 'th width="100" class') c = calendar.TextCalendar(calendar.SUNDAY) for i in c.itermonthdays(year, month): if i != 0: t = Time(str(year) + '-' + str(month) + '-' + str(i) + ' 20:00') cals = [] for caltype in [8, 9, 10]: xml, buf = ch.read_cal(caltype, t) if buf is None: cals.append(0) else: tout = Time(stateframe.extract(buf, xml['Timestamp']), format='lv') if (t - tout).value < 1. / 3: cals.append(1) else: cals.append(0) for k, line in enumerate(lines[3:]): idx = line.find(str(i)) ns = len(str(i)) if idx != -1: line = line[:idx + ns] + '<br>-r- -p- -tp- <br> {} {} {}'.format( *cals) + line[idx + ns:] break lines[k + 3] = line #print ''.join(line+'\n' for line in lines) return ''.join(line + '\n' for line in lines)
def get_calfac(t=None): ''' Read total power and auto-correlation calibration factors from the SQL database, for the time specified by Time() object t, or if None, at the next earlier calibration time to the current time. ''' tpcal_type = 10 # Calibration type specified in cal_header.py if t is None: t = Time.now() xml, buf = ch.read_cal(tpcal_type,t=t) fghz = stateframe.extract(buf,xml['FGHz']) nf = len(fghz) tpcalfac = np.zeros((13,2,nf),np.float) tpoffsun = np.zeros((13,2,nf),np.float) accalfac = np.zeros((13,2,nf),np.float) acoffsun = np.zeros((13,2,nf),np.float) nant = len(xml['Antenna']) for i in range(nant): iant = stateframe.extract(buf,xml['Antenna'][i]['Antnum'])-1 tpcalfac[iant] = stateframe.extract(buf,xml['Antenna'][i]['TPCalfac']) accalfac[iant] = stateframe.extract(buf,xml['Antenna'][i]['ACCalfac']) tpoffsun[iant] = stateframe.extract(buf,xml['Antenna'][i]['TPOffsun']) acoffsun[iant] = stateframe.extract(buf,xml['Antenna'][i]['ACOffsun']) return {'fghz':fghz,'timestamp':stateframe.extract(buf,xml['Timestamp']), 'tpcalfac':tpcalfac,'accalfac':accalfac,'tpoffsun':tpoffsun,'acoffsun':acoffsun}
def get_fem_level(trange, dt=None): ''' Get FEM attenuation levels for a given timerange. Returns a dictionary with keys as follows: times: A Time object containing the array of times, size (nt) hlev: The FEM attenuation level for HPol, size (nt, 15) vlev: The FEM attenuation level for VPol, size (nt, 15) dcmattn: The base DCM attenuations for 34 bands x 15 antennas x 2 Poln, size (34,30) The order is Ant1 H, Ant1 V, Ant2 H, Ant2 V, etc. dcmoff: If DPPoffset-on is 0, this is None (meaning there are no changes to the above base attenuations). If DPPoffset-on is 1, then dcmoff is a table of offsets to the base attenuation, size (nt, 50). The offset applies to all antennas/polarizations. Optional keywords: dt Seconds between entries to read from SQL stateframe database. If omitted, 1 s is assumed. ''' if dt is None: tstart, tend = [str(i) for i in trange.lv] else: # Expand time by 1/2 of dt before and after tstart = str(np.round(trange[0].lv - dt / 2)) tend = str(np.round(trange[1].lv + dt / 2)) cursor = db.get_cursor() ver = db.find_table_version(cursor, trange[0].lv) # Get front end attenuator states query = 'select Timestamp,Ante_Fron_FEM_Clockms,' \ +'Ante_Fron_FEM_HPol_Regi_Level,Ante_Fron_FEM_VPol_Regi_Level from fV' \ +ver+'_vD15 where Timestamp >= '+tstart+' and Timestamp <= '+tend+' order by Timestamp' data, msg = db.do_query(cursor, query) if msg == 'Success': if dt: # If we want other than full cadence, get new array shapes and times n = len(data['Timestamp']) # Original number of times new_n = ( n / 15 / dt ) * 15 * dt # Truncated number of times equally divisible by dt new_shape = (n / 15 / dt, dt, 15) # New shape of truncated arrays times = Time(data['Timestamp'][:new_n].astype('int')[::15 * dt], format='lv') else: times = Time(data['Timestamp'].astype('int')[::15], format='lv') hlev = data['Ante_Fron_FEM_HPol_Regi_Level'] vlev = data['Ante_Fron_FEM_VPol_Regi_Level'] ms = data['Ante_Fron_FEM_Clockms'] nt = len(hlev) / 15 hlev.shape = (nt, 15) vlev.shape = (nt, 15) ms.shape = (nt, 15) # Find any entries for which Clockms is zero, which indicates where no # gain-state measurement is available. for i in range(15): bad, = np.where(ms[:, i] == 0) if bad.size != 0 and bad.size != nt: # Find nearest adjacent good value good, = np.where(ms[:, i] != 0) idx = nearest_val_idx(bad, good) hlev[bad, i] = hlev[good[idx], i] vlev[bad, i] = vlev[good[idx], i] if dt: # If we want other than full cadence, find mean over dt measurements hlev = np.mean(hlev[:new_n / 15].reshape(new_shape), 1) vlev = np.mean(vlev[:new_n / 15].reshape(new_shape), 1) # Put results in canonical order [nant, nt] hlev = hlev.T vlev = vlev.T else: print 'Error reading FEM levels:', msg return {} # Get back end attenuator states xml, buf = ch.read_cal(2, t=trange[0]) dcmattn = stf.extract(buf, xml['Attenuation']) dcmattn.shape = (34, 15, 2) # Put into canonical order [nant, npol, nband] dcmattn = np.moveaxis(dcmattn, 0, 2) # See if DPP offset is enabled query = 'select Timestamp,DPPoffsetattn_on from fV' \ +ver+'_vD1 where Timestamp >= '+tstart+' and Timestamp <= '+tend+'order by Timestamp' data, msg = db.do_query(cursor, query) if msg == 'Success': dppon = data['DPPoffsetattn_on'] if np.where(dppon > 0)[0].size == 0: dcm_off = None else: query = 'select Timestamp,DCMoffset_attn from fV' \ +ver+'_vD50 where Timestamp >= '+tstart+' and Timestamp <= '+tend+' order by Timestamp' data, msg = db.do_query(cursor, query) if msg == 'Success': otimes = Time(data['Timestamp'].astype('int')[::15], format='lv') dcmoff = data['DCMoffset_attn'] dcmoff.shape = (nt, 50) # We now have a time-history of offsets, at least some of which are non-zero. # Offsets by slot number do us no good, so we need to translate to band number. # Get fseqfile name at mean of timerange, from stateframe SQL database fseqfile = get_fseqfile( Time(int(np.mean(trange.lv)), format='lv')) if fseqfile is None: print 'Error: No active fseq file.' dcm_off = None else: # Get fseqfile from ACC and return bandlist bandlist = fseqfile2bandlist(fseqfile) # Use bandlist to covert nt x 50 array to nt x 34 band array of DCM attn offsets # Note that this assumes DCM offset is the same for any multiply-sampled bands # in the sequence. dcm_off = np.zeros((nt, 34), float) dcm_off[:, bandlist - 1] = dcmoff # Put into canonical order [nband, nt] dcm_off = dcm_off.T if dt: # If we want other than full cadence, find mean over dt measurements new_nt = len(times) dcm_off = dcm_off[:, :new_nt * dt] dcm_off.shape = (34, dt, new_nt) dcm_off = np.mean(dcm_off, 1) else: print 'Error reading DCM attenuations:', msg dcm_off = None else: print 'Error reading DPPon state:', msg dcm_off = None cursor.close() return { 'times': times, 'hlev': hlev.astype(int), 'vlev': vlev.astype(int), 'dcmattn': dcmattn, 'dcmoff': dcm_off }
def old_version_test(sflog=None,sfxml=None,outbinfile=None,outtabfile=None): ''' Read stateframe log files of older versions and create output file of rearranged binary data, and corresponding stateframedef table as text file. sflog = file name of stateframe log to read sfxml = file name of corresponding XML file outbinfile = file name of output binary data file outtabfile = file name of output table text file ''' if sfxml: sf, version = rxml.xml_ptrs(sfxml) else: sf = None version = 0.0 if sflog: try: f = open(sflog,'rb') except: print 'Could not open file',sflog,'-- Exiting.' return # Get binary size and check version number data = f.read(100) if stateframe.extract(data,['d',8]) != version: print 'Stateframe log file version does not match XML version. -- Exiting' return recsize = stateframe.extract(data,sf['Binsize']) f.close() else: # No log file specified, so we will try to read directly from ACC once per second # Read one as a test and get its version number # Read from ACC accini = stateframe.rd_ACCfile() data, msg = stateframe.get_stateframe(accini) version = stateframe.extract(data,['d',8]) # Parse the stateframe dictionary and generate the brange and outlist dicts brange, outlist = sfdef(sf) # Calculate the startbytes in the list -- modifies outlist in place startbyte(outlist) stdout = sys.stdout # Save current stdout if outtabfile: # Write the table info to the given file name -- just sets stdout to the file, # writes it, and resets stdout try: sys.stdout = open(outtabfile,'w') except: print 'Could not redirect STDOUT to',outtabfile,' -- Will print to screen' sys.stdout = stdout outlist2table(outlist,version) sys.stdout = stdout # Reset to standard stdout if outbinfile: try: g = open(outbinfile,'wb') except: print 'Could not open file',outbinfile,'for writing. -- Exiting.' return if sflog: # Read from log file f = open(sflog,'rb') while 1: # Read and rearrange 1000 records try: indata = f.read(recsize) outdata = transmogrify(indata,brange) g.write(outdata) except: f.close() g.close() return else: # Read from ACC accini = stateframe.rd_ACCfile() for i in range(60): # Read from ACC and rearrange 60 records -- takes 1 min indata, msg = stateframe.get_stateframe(accini) outdata = transmogrify(indata,brange) g.write(outdata) time.sleep(1) g.close() return
def sql2phacalX(trange, *args, **kwargs): '''Supply a timestamp in Time format, return the closest phacal data. If a time range is provided, return records within the time range.''' import cal_header as ch import stateframe as stf xml, bufs = ch.read_calX(9, t=trange, *args, **kwargs) if isinstance(bufs, list): phacals = [] for i, buf in enumerate(bufs): try: phacal_flag = stf.extract(buf, xml['Phacal_Flag']) fghz = stf.extract(buf, xml['Fghz']) sigma = stf.extract(buf, xml['Phacal_Sigma']) timestamp = Time(stf.extract(buf, xml['Timestamp']), format='lv') tbg = Time(stf.extract(buf, xml['T_beg']), format='lv') ted = Time(stf.extract(buf, xml['T_end']), format='lv') pha = stf.extract(buf, xml['Phacal_Pha']) amp = stf.extract(buf, xml['Phacal_Amp']) tmp = stf.extract(buf, xml['MBD']) poff, pslope = tmp[:, :, 0], tmp[:, :, 1] flag = stf.extract(buf, xml['Flag'])[:, :, 0] t_ref = Time(stf.extract(buf, xml['T_refcal']), format='lv') phacals.append({ 'pslope': pslope, 't_pha': timestamp, 'flag': flag, 'poff': poff, 't_ref': t_ref, 'phacal': { 'pha': pha, 'amp': amp, 'flag': phacal_flag, 'fghz': fghz, 'sigma': sigma, 'timestamp': timestamp, 't_bg': tbg, 't_ed': ted } }) except: print 'failed to load record {} ---> {}'.format( i + 1, Time(stf.extract(buf, xml['Timestamp']), format='lv').iso) return phacals elif isinstance(bufs, str): phacal_flag = stf.extract(bufs, xml['Phacal_Flag']) fghz = stf.extract(bufs, xml['Fghz']) sigma = stf.extract(bufs, xml['Phacal_Sigma']) timestamp = Time(stf.extract(bufs, xml['Timestamp']), format='lv') tbg = Time(stf.extract(bufs, xml['T_beg']), format='lv') ted = Time(stf.extract(bufs, xml['T_end']), format='lv') pha = stf.extract(bufs, xml['Phacal_Pha']) amp = stf.extract(bufs, xml['Phacal_Amp']) tmp = stf.extract(bufs, xml['MBD']) poff, pslope = tmp[:, :, 0], tmp[:, :, 1] flag = stf.extract(bufs, xml['Flag'])[:, :, 0] t_ref = Time(stf.extract(bufs, xml['T_refcal']), format='lv') return { 'pslope': pslope, 't_pha': timestamp, 'flag': flag, 'poff': poff, 't_ref': t_ref, 'phacal': { 'pha': pha, 'amp': amp, 'flag': phacal_flag, 'fghz': fghz, 'sigma': sigma, 'timestamp': timestamp, 't_bg': tbg, 't_ed': ted } }
def apply_attn_corr(data, tref=None): ''' Applys the attenuator state corrections to the given data dictionary, corrected to the gain-state at time given by Time() object tref. Inputs: data A dictionary returned by udb_util.py's readXdata(). tref A Time() object with the reference time, or if None, the gain state of the nearest earlier REFCAL is used. Output: cdata A dictionary with the gain-corrected data. The keys px, py, and x, are updated. NB: This is the same routine as in gaincal2.py, but modified to handle the different ordering/format of data from udb_util.py's readXdata() routine. ''' from gaincal2 import get_gain_state from util import common_val_idx, nearest_val_idx import copy if tref is None: # No reference time specified, so get nearest earlier REFCAL trange = Time(data['time'][[0,-1]],format='jd') xml, buf = ch.read_cal(8,t=trange[0]) tref = Time(stateframe.extract(buf,xml['Timestamp']),format='lv') # Get the gain state at the reference time (actually median over 1 minute) trefrange = Time([tref.iso,Time(tref.lv+60,format='lv').iso]) ref_gs = get_gain_state(trefrange) # refcal gain state for 60 s # Get median of refcal gain state (which should be constant anyway) ref_gs['h1'] = np.median(ref_gs['h1'],1) ref_gs['h2'] = np.median(ref_gs['h2'],1) ref_gs['v1'] = np.median(ref_gs['v1'],1) ref_gs['v2'] = np.median(ref_gs['v2'],1) # Get timerange from data trange = Time([data['time'][0],data['time'][-1]],format='jd') # Get time cadence dt = np.int(np.round(np.median(data['time'][1:] - data['time'][:-1]) * 86400)) if dt == 1: dt = None # Get the gain state of the requested timerange src_gs = get_gain_state(trange,dt) # solar gain state for timerange of file nt = len(src_gs['times']) antgain = np.zeros((15,2,34,nt),np.float32) # Antenna-based gains vs. band for i in range(15): for j in range(34): antgain[i,0,j] = src_gs['h1'][i] + src_gs['h2'][i] - ref_gs['h1'][i] - ref_gs['h2'][i] + src_gs['dcmattn'][i,0,j] - ref_gs['dcmattn'][i,0,j] antgain[i,1,j] = src_gs['v1'][i] + src_gs['v2'][i] - ref_gs['v1'][i] - ref_gs['v2'][i] + src_gs['dcmattn'][i,1,j] - ref_gs['dcmattn'][i,1,j] cdata = copy.deepcopy(data) # Create giant array of baseline-based gains, translated to baselines and frequencies fghz = data['fghz'] nf = len(fghz) blist = (fghz*2 - 1).astype(int) - 1 # Band list corresponding to frequencies in data blgain = np.zeros((nf,136,4,nt),float) # Baseline-based gains vs. frequency for k,bl in enumerate(get_bl_order()): i, j = bl if i < 15 and j < 15: blgain[:,k,0] = 10**((antgain[i,0,blist] + antgain[j,0,blist])/20.) blgain[:,k,1] = 10**((antgain[i,1,blist] + antgain[j,1,blist])/20.) blgain[:,k,2] = 10**((antgain[i,0,blist] + antgain[j,1,blist])/20.) blgain[:,k,3] = 10**((antgain[i,1,blist] + antgain[j,0,blist])/20.) # Reorder antgain axes to put frequencies in first slot, to match data antgain = np.swapaxes(np.swapaxes(antgain,1,2),0,1) antgainf = 10**(antgain[blist]/10.) idx = nearest_val_idx(data['time'],src_gs['times'].jd) # Correct the auto- and cross-correlation data cdata['x'] *= blgain[:,:,:,idx] # Reshape px and py arrays cdata['px'].shape = (134,16,3,nt) cdata['py'].shape = (134,16,3,nt) # Correct the power cdata['px'][:,:15,0] *= antgainf[:,:,0,idx] cdata['py'][:,:15,0] *= antgainf[:,:,1,idx] # Correct the power-squared cdata['px'][:,:15,1] *= antgainf[:,:,0,idx]**2 cdata['py'][:,:15,1] *= antgainf[:,:,1,idx]**2 # Reshape px and py arrays back to original cdata['px'].shape = (134*16*3,nt) cdata['py'].shape = (134*16*3,nt) return cdata
def unrot(data, azeldict=None): ''' Apply the correction to differential feed rotation to data, and return the corrected data. Inputs: data A dictionary returned by udb_util.py's readXdata(). azeldict The dictionary returned from get_sql_info(), or if None, the appropriate get_sql_info() call is done internally. Output: cdata A dictionary with the phase-corrected data. Only the key x is updated. ''' import copy from util import lobe trange = Time(data['time'][[0,-1]],format='jd') if azeldict is None: azeldict = get_sql_info(trange) chi = azeldict['ParallacticAngle'] # (nt, nant) # Correct parallactic angle for equatorial mounts, relative to Ant14 for i in [8,9,10,12,13]: chi[:,i] -= chi[:,13] # Ensure that nearest valid parallactic angle is used for times in the data good, = np.where(azeldict['ActualAzimuth'][0] != 0) tidx = nearest_val_idx(data['time'],azeldict['Time'][good].jd) # Read X-Y Delay phase from SQL database and get common frequencies xml, buf = ch.read_cal(11,t=trange[0]) fghz = stateframe.extract(buf,xml['FGHz']) good, = np.where(fghz != 0.) fghz = fghz[good] dph = stateframe.extract(buf,xml['XYphase']) dph = dph[:,good] fidx1, fidx2 = common_val_idx(data['fghz'],fghz,precision=4) missing = np.setdiff1d(np.arange(len(data['fghz'])),fidx1) nf, nbl, npol, nt = data['x'].shape nf = len(fidx1) # Correct data for X-Y delay phase for k,bl in enumerate(get_bl_order()): i, j = bl if i < 14 and j < 14 and i != j: a1 = lobe(dph[i,fidx2] - dph[j,fidx2]) a2 = -dph[j,fidx2] + np.pi/2 a3 = dph[i,fidx2] - np.pi/2 data['x'][fidx1,k,1] *= np.repeat(np.exp(1j*a1),nt).reshape(nf,nt) data['x'][fidx1,k,2] *= np.repeat(np.exp(1j*a2),nt).reshape(nf,nt) data['x'][fidx1,k,3] *= np.repeat(np.exp(1j*a3),nt).reshape(nf,nt) # Correct data for differential feed rotation cdata = copy.deepcopy(data) for n in range(nt): for k,bl in enumerate(get_bl_order()): i, j = bl if i < 14 and j < 14 and i != j: dchi = chi[n,i] - chi[n,j] cchi = np.cos(dchi) schi = np.sin(dchi) cdata['x'][:,k,0,n] = data['x'][:,k,0,n]*cchi + data['x'][:,k,3,n]*schi cdata['x'][:,k,2,n] = data['x'][:,k,2,n]*cchi + data['x'][:,k,1,n]*schi cdata['x'][:,k,3,n] = data['x'][:,k,3,n]*cchi - data['x'][:,k,0,n]*schi cdata['x'][:,k,1,n] = data['x'][:,k,1,n]*cchi - data['x'][:,k,2,n]*schi # Set flags for any missing frequencies (hopefully this also works when missing is np.array([])) cdata[missing] = np.ma.masked return cdata
def DCM_attn_anal(filename): ''' Analyze a DCMATTNTEST observation to determine the 2- and 4-bit attenuation values. Input is a Miriad file. Returns two arrays, at2 and at4 of size (nant,npol) = (13,2) representing the attenuation, in dB, of the 2- and 4-bit, resp. ''' import read_idb as ri import dbutil as db import cal_header as ch import stateframe as stf import copy from util import Time import matplotlib.pylab as plt out = ri.read_idb([filename]) ts = int(Time(out['time'][0], format='jd').lv + 0.5) te = int(Time(out['time'][-1], format='jd').lv + 0.5) query = 'select Timestamp,DCM_Offset_Attn from fV65_vD15 where Timestamp between ' + str( ts) + ' and ' + str(te) + ' order by Timestamp' cursor = db.get_cursor() data, msg = db.do_query(cursor, query) cursor.close() dcm_offset = data['DCM_Offset_Attn'].reshape( len(data['DCM_Offset_Attn']) / 15, 15) dcm_offset = dcm_offset[:, 0] # All antennas are the same t = Time(out['time'][0], format='jd') xml, buf = ch.read_cal(2, t) table = stf.extract(buf, xml['Attenuation']) bandlist = ((out['fghz'] - 0.5) * 2).astype(int) tbl = table[bandlist - 1] tbl.shape = (len(bandlist), 15, 2) tbl = np.swapaxes(np.swapaxes(tbl, 0, -1), 0, 1) tbl2 = np.broadcast_to(tbl, (out['time'].shape[0], 15, 2, 134)) tbl = copy.copy(np.rollaxis(tbl2, 0, 4)) # Shape (nant,npol,nf,nt) pwr = out['p'][:15] # Shape (nant,npol,nf,nt) # Add value of dcm_offset to table for i, offset in enumerate(dcm_offset): tbl[:, :, :, i] += offset # Clip to valid attenuations tbl = np.clip(tbl, 0, 30) # Isolate good times in various attn states goodm2, = np.where(dcm_offset == -2) goodm2 = goodm2[2:-3] good2, = np.where(dcm_offset == 2) good2 = good2[2:-3] good0, = np.where(dcm_offset[goodm2[-1]:good2[0]] == 0) good0 += goodm2[-1] good0 = good0[2:-3] good4, = np.where(dcm_offset == 4) good4 = good4[2:-3] good6, = np.where(dcm_offset == 6) good6 = good6[2:-3] goodbg = good6 + 30 # Assumes FEMATTN 15 follows good6 30 s later # Perform median over good times and create pwrmed with medians # The 5 indexes correspond to dcm_offsets -2, 0, 2, 4 and 6 nant, npol, nf, nt = pwr.shape pwrmed = np.zeros((nant, npol, nf, 5)) # Do not forget to subtract the background bg = np.median(pwr[:, :, :, goodbg], 3) pwrmed[:, :, :, 0] = np.median(pwr[:, :, :, goodm2], 3) - bg pwrmed[:, :, :, 1] = np.median(pwr[:, :, :, good0], 3) - bg pwrmed[:, :, :, 2] = np.median(pwr[:, :, :, good2], 3) - bg pwrmed[:, :, :, 3] = np.median(pwr[:, :, :, good4], 3) - bg pwrmed[:, :, :, 4] = np.median(pwr[:, :, :, good6], 3) - bg good = np.array([goodm2[0], good0[0], good2[0], good4[0], good6[0]]) tbl = tbl[:, :, :, good] at2 = np.zeros((13, 2), float) at4 = np.zeros((13, 2), float) at8 = np.zeros((13, 2), float) f1, ax1 = plt.subplots(2, 13) f2, ax2 = plt.subplots(2, 13) f3, ax3 = plt.subplots(2, 13) for ant in range(13): for pol in range(2): pts = [] for i in range(4): for v in [0, 4, 8, 12, 16, 20, 24, 28]: idx, = np.where(tbl[ant, pol, :, i] == v) if len(idx) != 0: good, = np.where((tbl[ant, pol, idx, i] + 2) == tbl[ant, pol, idx, i + 1]) if len(good) != 0: pts.append(pwrmed[ant, pol, idx[good], i] / pwrmed[ant, pol, idx[good], i + 1]) pts = np.concatenate(pts) ax1[pol, ant].plot(pts, '.') ax1[pol, ant].set_ylim(0, 2) at2[ant, pol] = np.log10(np.median(pts)) * 10. pts = [] for i in range(3): for v in [0, 2, 8, 10, 16, 18, 24, 26]: idx, = np.where(tbl[ant, pol, :, i] == v) if len(idx) != 0: good, = np.where((tbl[ant, pol, idx, i] + 4) == tbl[ant, pol, idx, i + 2]) if len(good) != 0: pts.append(pwrmed[ant, pol, idx[good], i] / pwrmed[ant, pol, idx[good], i + 2]) pts = np.concatenate(pts) ax2[pol, ant].plot(pts, '.') ax2[pol, ant].set_ylim(0, 3) at4[ant, pol] = np.log10(np.median(pts)) * 10. pts = [] i = 0 for v in [0, 2, 4, 6, 16, 18, 20, 22]: idx, = np.where(tbl[ant, pol, :, i] == v) if len(idx) != 0: good, = np.where((tbl[ant, pol, idx, i] + 8) == tbl[ant, pol, idx, i + 4]) if len(good) != 0: pts.append(pwrmed[ant, pol, idx[good], i] / pwrmed[ant, pol, idx[good], i + 4]) try: pts = np.concatenate(pts) except: # Looks like there were no points for this antenna/polarization, so set to nominal attn pts = [6.30957, 6.30957, 6.30957] ax3[pol, ant].plot(pts, '.') ax3[pol, ant].set_ylim(5, 8) at8[ant, pol] = np.log10(np.median(pts)) * 10. plt.show() # Generate output table, a complex array of size (nant,npol,nbits) attn = np.zeros((16, 2, 4), np.complex) # Set to nominal values, then overwrite with measured ones for i in range(16): for j in range(2): attn[i, j] = [2.0 + 0j, 4.0 + 0j, 8.0 + 0j, 16.0 + 0j] attn[:13, :, 0] = at2 + 0j attn[:13, :, 1] = at4 + 0j attn[:13, :, 2] = at8 + 0j return attn
def DCM_master_attn_cal(update=False): ''' New version of this command, which uses the power values in the 10gbe packet headers instead of the very slow measurement of the ADC levels themselves. This version only takes about 8 s! If update is True, it writes the results to the SQL database. Returns the DCM_master_table in the form of lines of text strings, with labels (handy for viewing). ''' pwr = np.zeros((50,8,4),'int') # Capture on eth2 interface command = 'tcpdump -i eth2 -c 155000 -w /home/user/Python/dcm2.pcap -s 1000' p.sendcmd(command) # Capture on eth3 interface command = 'tcpdump -i eth3 -c 155000 -w /home/user/Python/dcm3.pcap -s 1000' p.sendcmd(command) headers = p.list_header('/home/user/Python/dcm2.pcap') for line in headers: try: j, id, p1,p2,p3,p4 = np.array(map(int,line.split()))[[0,3,6,7,8,9]] pwr[j,id] = (p1, p2, p3, p4) except: # This is to skip the non-data header lines in the list pass headers = p.list_header('/home/user/Python/dcm3.pcap') for line in headers: try: j, id, p1,p2,p3,p4 = np.array(map(int,line.split()))[[0,3,6,7,8,9]] pwr[j,id] = (p1, p2, p3, p4) except: # This is to skip the non-data header lines in the list pass # Reshape to (slot, nant, npol) pwr.shape = (50,16,2) # Read current frequency sequence from database cursor = db.get_cursor() query = 'select top 50 FSeqList from hV37_vD50 order by Timestamp desc' fseq, msg = db.do_query(cursor, query) if msg == 'Success': fseqlist = fseq['FSeqList'][::-1] # Reverse the order bandlist = ((np.array(fseqlist)-0.44)*2).astype(int) cursor.close() # Read current DCM_master_table from database xml, buf = ch.read_cal(2) orig_table = stf.extract(buf,xml['Attenuation']) # Order pwr values according to bandlist, taking median of any repeated values new_pwr = np.zeros((34,16,2)) for i in range(34): idx, = np.where(bandlist-1 == i) if len(idx) > 0: new_pwr[i] = np.median(pwr[idx],0) new_pwr.shape = (34,32) # Now determine the change in attenuation needed to achieve a target # value of 1600. Eliminate last two entries, corresponding to Ant16 attn = np.log10(new_pwr[:,:-2]/1600.)*10. new_table = (np.clip(orig_table + attn,0,30)/2).astype(int)*2 DCMlines = [] DCMlines.append('# Ant1 Ant2 Ant3 Ant4 Ant5 Ant6 Ant7 Ant8 Ant9 Ant10 Ant11 Ant12 Ant13 Ant14 Ant15') DCMlines.append('# X Y X Y X Y X Y X Y X Y X Y X Y X Y X Y X Y X Y X Y X Y X Y') DCMlines.append('# ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- -----') for band in range(1,35): DCMlines.append('{:2} : {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2}'.format(band,*new_table[band-1])) if update: msg = ch.dcm_master_table2sql(DCMlines) if msg: print 'Success' else: print 'Error writing table to SQL database!' return DCMlines
def sql2phacalX(trange, *args, **kwargs): '''Supply a timestamp in Time format, return the closest phacal data. If a time range is provided, return records within the time range.''' import cal_header as ch import stateframe as stf xml, bufs = ch.read_calX(9, t=trange, *args, **kwargs) if isinstance(bufs, list): phacals = [] for i, buf in enumerate(bufs): try: phacal_flag = stf.extract(buf, xml['Phacal_Flag']) fghz = stf.extract(buf, xml['Fghz']) sigma = stf.extract(buf, xml['Phacal_Sigma']) timestamp = Time(stf.extract(buf, xml['Timestamp']), format='lv') tbg = Time(stf.extract(buf, xml['T_beg']), format='lv') ted = Time(stf.extract(buf, xml['T_end']), format='lv') pha = stf.extract(buf, xml['Phacal_Pha']) amp = stf.extract(buf, xml['Phacal_Amp']) tmp = stf.extract(buf, xml['MBD']) poff, pslope = tmp[:, :, 0], tmp[:, :, 1] flag = stf.extract(buf, xml['Flag'])[:, :, 0] t_ref = Time(stf.extract(buf, xml['T_refcal']), format='lv') phacals.append( {'pslope': pslope, 't_pha': timestamp, 'flag': flag, 'poff': poff, 't_ref': t_ref, 'phacal': {'pha': pha, 'amp': amp, 'flag': phacal_flag, 'fghz': fghz, 'sigma': sigma, 'timestamp': timestamp, 't_bg': tbg, 't_ed': ted}}) except: print 'failed to load record {} ---> {}'.format(i + 1, Time(stf.extract(buf, xml['Timestamp']), format='lv').iso) return phacals elif isinstance(bufs, str): phacal_flag = stf.extract(bufs, xml['Phacal_Flag']) fghz = stf.extract(bufs, xml['Fghz']) sigma = stf.extract(bufs, xml['Phacal_Sigma']) timestamp = Time(stf.extract(bufs, xml['Timestamp']), format='lv') tbg = Time(stf.extract(bufs, xml['T_beg']), format='lv') ted = Time(stf.extract(bufs, xml['T_end']), format='lv') pha = stf.extract(bufs, xml['Phacal_Pha']) amp = stf.extract(bufs, xml['Phacal_Amp']) tmp = stf.extract(bufs, xml['MBD']) poff, pslope = tmp[:, :, 0], tmp[:, :, 1] flag = stf.extract(bufs, xml['Flag'])[:, :, 0] t_ref = Time(stf.extract(bufs, xml['T_refcal']), format='lv') return {'pslope': pslope, 't_pha': timestamp, 'flag': flag, 'poff': poff, 't_ref': t_ref, 'phacal': {'pha': pha, 'amp': amp, 'flag': phacal_flag, 'fghz': fghz, 'sigma': sigma, 'timestamp': timestamp, 't_bg': tbg, 't_ed': ted}}
def apply_gain_corr(data, tref=None): ''' Applys the gain_state() corrections to the given data dictionary, corrected to the gain-state at time given by Time() object tref. Inputs: data A dictionary such as that returned by read_idb(). tref A Time() object with the reference time, or if None, the gain state of the nearest earlier REFCAL is used. Output: cdata A dictionary with the gain-corrected data. The keys p, x, p2, and a are all updated. ''' from util import common_val_idx, nearest_val_idx import copy if tref is None: # No reference time specified, so get nearest earlier REFCAL trange = Time(data['time'][[0, -1]], format='jd') xml, buf = ch.read_cal(8, t=trange[0]) if xml == {}: # No refcal for this date, so just use an early time as reference tref = Time(trange[0].iso[:10] + ' 13:30') else: tref = Time(stf.extract(buf, xml['Timestamp']), format='lv') # Get the gain state at the reference time (actually median over 1 minute) trefrange = Time([tref.iso, Time(tref.lv + 61, format='lv').iso]) ref_gs = get_gain_state(trefrange) # refcal gain state for 60 s # Get median of refcal gain state (which should be constant anyway) ref_gs['h1'] = np.median(ref_gs['h1'], 1) ref_gs['h2'] = np.median(ref_gs['h2'], 1) ref_gs['v1'] = np.median(ref_gs['v1'], 1) ref_gs['v2'] = np.median(ref_gs['v2'], 1) # Get timerange from data trange = Time([data['time'][0], data['time'][-1]], format='jd') # Get time cadence dt = np.int( np.round(np.median(data['time'][1:] - data['time'][:-1]) * 86400)) if dt == 1: dt = None # Get the gain state of the requested timerange src_gs = get_gain_state(trange, dt) # solar gain state for timerange of file nt = len(src_gs['times']) antgain = np.zeros((15, 2, 34, nt), np.float32) # Antenna-based gains vs. band for i in range(15): for j in range(34): antgain[i, 0, j] = src_gs['h1'][i] + src_gs['h2'][i] - ref_gs[ 'h1'][i] - ref_gs['h2'][i] + src_gs['dcmattn'][ i, 0, j] - ref_gs['dcmattn'][i, 0, j] antgain[i, 1, j] = src_gs['v1'][i] + src_gs['v2'][i] - ref_gs[ 'v1'][i] - ref_gs['v2'][i] + src_gs['dcmattn'][ i, 1, j] - ref_gs['dcmattn'][i, 1, j] cdata = copy.deepcopy(data) # Frequency list is provided, so produce baseline-based gain table as well # Create giant array of gains, translated to baselines and frequencies fghz = data['fghz'] nf = len(fghz) blist = (fghz * 2 - 1).astype(int) - 1 blgain = np.zeros((120, 4, nf, nt), float) # Baseline-based gains vs. frequency for i in range(14): for j in range(i + 1, 15): blgain[ri.bl2ord[i, j], 0] = 10**((antgain[i, 0, blist] + antgain[j, 0, blist]) / 20.) blgain[ri.bl2ord[i, j], 1] = 10**((antgain[i, 1, blist] + antgain[j, 1, blist]) / 20.) blgain[ri.bl2ord[i, j], 2] = 10**((antgain[i, 0, blist] + antgain[j, 1, blist]) / 20.) blgain[ri.bl2ord[i, j], 3] = 10**((antgain[i, 1, blist] + antgain[j, 0, blist]) / 20.) antgainf = 10**(antgain[:, :, blist] / 10.) #idx1, idx2 = common_val_idx(data['time'],src_gs['times'].jd) idx = nearest_val_idx(data['time'], src_gs['times'].jd) # Apply corrections (some times may be eliminated from the data) # Correct the cross-correlation data cdata['x'] *= blgain[:, :, :, idx] # Correct the power cdata['p'][:15] *= antgainf[:, :, :, idx] # Correct the autocorrelation cdata['a'][:15, :2] *= antgainf[:, :, :, idx] cross_fac = np.sqrt(antgainf[:, 0] * antgainf[:, 1]) cdata['a'][:15, 2] *= cross_fac[:, :, idx] cdata['a'][:15, 3] *= cross_fac[:, :, idx] # Correct the power-squared -- this should preserve SK cdata['p2'][:15] *= antgainf[:, :, :, idx]**2 # Remove any uncorrected times before returning #cdata['time'] = cdata['time'][idx1] #cdata['p'] = cdata['p'][:,:,:,idx1] #cdata['a'] = cdata['a'][:,:,:,idx1] #cdata['p2'] = cdata['p2'][:,:,:,idx1] #cdata['ha'] = cdata['ha'][idx1] #cdata['m'] = cdata['m'][:,:,:,idx1] return cdata
def sql2refcalX(trange, lohi=False, *args, **kwargs): '''same as sql2refcal. trange can be either a timestamp or a timerange.''' import cal_header as ch import stateframe as stf if lohi: caltype = 12 else: caltype = 8 xml, bufs = ch.read_calX(caltype, t=trange, *args, **kwargs) if isinstance(bufs, list): refcals = [] for i, buf in enumerate(bufs): try: ref = stf.extract(buf, xml['Refcal_Real']) + stf.extract( buf, xml['Refcal_Imag']) * 1j flag = stf.extract(buf, xml['Refcal_Flag']) fghz = stf.extract(buf, xml['Fghz']) sigma = stf.extract(buf, xml['Refcal_Sigma']) timestamp = Time(stf.extract(buf, xml['Timestamp']), format='lv') tbg = Time(stf.extract(buf, xml['T_beg']), format='lv') ted = Time(stf.extract(buf, xml['T_end']), format='lv') pha = np.angle(ref) amp = np.absolute(ref) refcals.append({ 'pha': pha, 'amp': amp, 'flag': flag, 'fghz': fghz, 'sigma': sigma, 'timestamp': timestamp, 't_bg': tbg, 't_ed': ted }) except: print 'failed to load record {} ---> {}'.format( i + 1, Time(stf.extract(buf, xml['Timestamp']), format='lv').iso) return refcals elif isinstance(bufs, str): refcal = stf.extract(bufs, xml['Refcal_Real']) + stf.extract( bufs, xml['Refcal_Imag']) * 1j flag = stf.extract(bufs, xml['Refcal_Flag']) fghz = stf.extract(bufs, xml['Fghz']) sigma = stf.extract(bufs, xml['Refcal_Sigma']) timestamp = Time(stf.extract(bufs, xml['Timestamp']), format='lv') tbg = Time(stf.extract(bufs, xml['T_beg']), format='lv') ted = Time(stf.extract(bufs, xml['T_end']), format='lv') pha = np.angle(refcal) amp = np.absolute(refcal) return { 'pha': pha, 'amp': amp, 'flag': flag, 'fghz': fghz, 'sigma': sigma, 'timestamp': timestamp, 't_bg': tbg, 't_ed': ted }
def apply_attn_corr(data, tref=None): ''' Applys the attenuator state corrections to the given data dictionary, corrected to the gain-state at time given by Time() object tref. Inputs: data A dictionary returned by udb_util.py's readXdata(). tref A Time() object with the reference time, or if None, the gain state of the nearest earlier REFCAL is used. Output: cdata A dictionary with the gain-corrected data. The keys px, py, and x, are updated. NB: This is the same routine as in gaincal2.py, but modified to handle the different ordering/format of data from udb_util.py's readXdata() routine. ''' from gaincal2 import get_gain_state from util import common_val_idx, nearest_val_idx import copy if tref is None: # No reference time specified, so get nearest earlier REFCAL trange = Time(data['time'][[0, -1]], format='jd') xml, buf = ch.read_cal(8, t=trange[0]) tref = Time(stateframe.extract(buf, xml['Timestamp']), format='lv') # Get the gain state at the reference time (actually median over 1 minute) trefrange = Time([tref.iso, Time(tref.lv + 60, format='lv').iso]) ref_gs = get_gain_state(trefrange) # refcal gain state for 60 s # Get median of refcal gain state (which should be constant anyway) ref_gs['h1'] = np.median(ref_gs['h1'], 1) ref_gs['h2'] = np.median(ref_gs['h2'], 1) ref_gs['v1'] = np.median(ref_gs['v1'], 1) ref_gs['v2'] = np.median(ref_gs['v2'], 1) # Get timerange from data trange = Time([data['time'][0], data['time'][-1]], format='jd') # Get time cadence dt = np.int( np.round(np.median(data['time'][1:] - data['time'][:-1]) * 86400)) if dt == 1: dt = None # Get the gain state of the requested timerange src_gs = get_gain_state(trange, dt) # solar gain state for timerange of file nt = len(src_gs['times']) antgain = np.zeros((15, 2, 34, nt), np.float32) # Antenna-based gains vs. band for i in range(15): for j in range(34): antgain[i, 0, j] = src_gs['h1'][i] + src_gs['h2'][i] - ref_gs[ 'h1'][i] - ref_gs['h2'][i] + src_gs['dcmattn'][ i, 0, j] - ref_gs['dcmattn'][i, 0, j] antgain[i, 1, j] = src_gs['v1'][i] + src_gs['v2'][i] - ref_gs[ 'v1'][i] - ref_gs['v2'][i] + src_gs['dcmattn'][ i, 1, j] - ref_gs['dcmattn'][i, 1, j] cdata = copy.deepcopy(data) # Create giant array of baseline-based gains, translated to baselines and frequencies fghz = data['fghz'] nf = len(fghz) blist = (fghz * 2 - 1).astype( int) - 1 # Band list corresponding to frequencies in data blgain = np.zeros((nf, 136, 4, nt), float) # Baseline-based gains vs. frequency for k, bl in enumerate(get_bl_order()): i, j = bl if i < 15 and j < 15: blgain[:, k, 0] = 10**((antgain[i, 0, blist] + antgain[j, 0, blist]) / 20.) blgain[:, k, 1] = 10**((antgain[i, 1, blist] + antgain[j, 1, blist]) / 20.) blgain[:, k, 2] = 10**((antgain[i, 0, blist] + antgain[j, 1, blist]) / 20.) blgain[:, k, 3] = 10**((antgain[i, 1, blist] + antgain[j, 0, blist]) / 20.) # Reorder antgain axes to put frequencies in first slot, to match data antgain = np.swapaxes(np.swapaxes(antgain, 1, 2), 0, 1) antgainf = 10**(antgain[blist] / 10.) idx = nearest_val_idx(data['time'], src_gs['times'].jd) # Correct the auto- and cross-correlation data cdata['x'] *= blgain[:, :, :, idx] # Reshape px and py arrays cdata['px'].shape = (134, 16, 3, nt) cdata['py'].shape = (134, 16, 3, nt) # Correct the power cdata['px'][:, :15, 0] *= antgainf[:, :, 0, idx] cdata['py'][:, :15, 0] *= antgainf[:, :, 1, idx] # Correct the power-squared cdata['px'][:, :15, 1] *= antgainf[:, :, 0, idx]**2 cdata['py'][:, :15, 1] *= antgainf[:, :, 1, idx]**2 # Reshape px and py arrays back to original cdata['px'].shape = (134 * 16 * 3, nt) cdata['py'].shape = (134 * 16 * 3, nt) return cdata
def unrot(data, azeldict=None): ''' Apply the correction to differential feed rotation to data, and return the corrected data. Inputs: data A dictionary returned by udb_util.py's readXdata(). azeldict The dictionary returned from get_sql_info(), or if None, the appropriate get_sql_info() call is done internally. Output: cdata A dictionary with the phase-corrected data. Only the key x is updated. ''' import copy from util import lobe trange = Time(data['time'][[0, -1]], format='jd') if azeldict is None: azeldict = get_sql_info(trange) chi = azeldict['ParallacticAngle'] # (nt, nant) # Correct parallactic angle for equatorial mounts, relative to Ant14 for i in [8, 9, 10, 12, 13]: chi[:, i] -= chi[:, 13] # Ensure that nearest valid parallactic angle is used for times in the data good, = np.where(azeldict['ActualAzimuth'][0] != 0) tidx = nearest_val_idx(data['time'], azeldict['Time'][good].jd) # Read X-Y Delay phase from SQL database and get common frequencies xml, buf = ch.read_cal(11, t=trange[0]) fghz = stateframe.extract(buf, xml['FGHz']) good, = np.where(fghz != 0.) fghz = fghz[good] dph = stateframe.extract(buf, xml['XYphase']) dph = dph[:, good] fidx1, fidx2 = common_val_idx(data['fghz'], fghz, precision=4) missing = np.setdiff1d(np.arange(len(data['fghz'])), fidx1) nf, nbl, npol, nt = data['x'].shape nf = len(fidx1) # Correct data for X-Y delay phase for k, bl in enumerate(get_bl_order()): i, j = bl if i < 14 and j < 14 and i != j: a1 = lobe(dph[i, fidx2] - dph[j, fidx2]) a2 = -dph[j, fidx2] + np.pi / 2 a3 = dph[i, fidx2] - np.pi / 2 data['x'][fidx1, k, 1] *= np.repeat(np.exp(1j * a1), nt).reshape(nf, nt) data['x'][fidx1, k, 2] *= np.repeat(np.exp(1j * a2), nt).reshape(nf, nt) data['x'][fidx1, k, 3] *= np.repeat(np.exp(1j * a3), nt).reshape(nf, nt) # Correct data for differential feed rotation cdata = copy.deepcopy(data) for n in range(nt): for k, bl in enumerate(get_bl_order()): i, j = bl if i < 14 and j < 14 and i != j: dchi = chi[n, i] - chi[n, j] cchi = np.cos(dchi) schi = np.sin(dchi) cdata['x'][:, k, 0, n] = data['x'][:, k, 0, n] * cchi + data['x'][:, k, 3, n] * schi cdata['x'][:, k, 2, n] = data['x'][:, k, 2, n] * cchi + data['x'][:, k, 1, n] * schi cdata['x'][:, k, 3, n] = data['x'][:, k, 3, n] * cchi - data['x'][:, k, 0, n] * schi cdata['x'][:, k, 1, n] = data['x'][:, k, 1, n] * cchi - data['x'][:, k, 2, n] * schi # Set flags for any missing frequencies (hopefully this also works when missing is np.array([])) cdata[missing] = np.ma.masked return cdata
def DCM_master_attn_cal(fseqfile=None, dcmattn=None, update=False): ''' New version of this command, which uses the power values in the 10gbe packet headers instead of the very slow measurement of the ADC levels themselves. This version only takes about 8 s! If update is True, it writes the results to the SQL database. Returns the DCM_master_table in the form of lines of text strings, with labels (handy for viewing). ''' import pcapture2 as p import dbutil as db import cal_header as ch import stateframe as stf bandlist = fseqfile2bandlist(fseqfile) if bandlist is None: print 'Must specify a frequency sequence.' return None # Make sure this sequence is actually running, or start it if not accini = stf.rd_ACCfile() if not fseq_is_running(fseqfile, accini): # Sequence is not running, so send ACC commands to start it send_cmds(['FSEQ-OFF'], accini) send_cmds(['FSEQ-INIT'], accini) send_cmds(['FSEQ-FILE ' + fseqfile], accini) send_cmds(['FSEQ-ON'], accini) bandlist2dcmtable(bandlist, toACC=True) time.sleep(3) if not fseq_is_running(fseqfile, accini): print 'Frequency sequence could not be started.' return None else: print 'Successfully started frequency sequence.' send_cmds(['dcmtable dcm.txt'], accini) send_cmds(['dcmauto-on'], accini) pwr = np.zeros((50, 8, 4), 'int') # Capture on eth2 interface command = 'tcpdump -i eth2 -c 155000 -w /home/user/Python/dcm2.pcap -s 1000' p.sendcmd(command) # Capture on eth3 interface command = 'tcpdump -i eth3 -c 155000 -w /home/user/Python/dcm3.pcap -s 1000' p.sendcmd(command) headers = p.list_header('/home/user/Python/dcm2.pcap') for line in headers: try: j, id, p1, p2, p3, p4 = np.array(map( int, line.split()))[[0, 3, 6, 7, 8, 9]] pwr[j, id] = (p1, p2, p3, p4) except: # This is to skip the non-data header lines in the list pass headers = p.list_header('/home/user/Python/dcm3.pcap') for line in headers: try: j, id, p1, p2, p3, p4 = np.array(map( int, line.split()))[[0, 3, 6, 7, 8, 9]] pwr[j, id] = (p1, p2, p3, p4) except: # This is to skip the non-data header lines in the list pass # Reshape to (slot, nant, npol) pwr.shape = (50, 16, 2) # # Read current frequency sequence from database # cursor = db.get_cursor() # query = 'select top 50 FSeqList from hV37_vD50 order by Timestamp desc' # fseq, msg = db.do_query(cursor, query) # if msg == 'Success': # fseqlist = fseq['FSeqList'][::-1] # Reverse the order # bandlist = ((np.array(fseqlist)-0.44)*2).astype(int) # cursor.close() if dcmattn is None: # Read current DCM_master_table from database xml, buf = ch.read_cal(2) orig_table = stf.extract(buf, xml['Attenuation']) else: # DCM attenuation is set to a constant value so create a table of such values. orig_table = np.zeros((34, 30)) + dcmattn orig_table[:, 26:] = 0 # Order pwr values according to bandlist, taking median of any repeated values new_pwr = np.zeros((34, 16, 2)) for i in range(34): idx, = np.where(bandlist - 1 == i) if len(idx) > 0: new_pwr[i] = np.median(pwr[idx], 0) new_pwr.shape = (34, 32) # Now determine the change in attenuation needed to achieve a target # value of 1600. Eliminate last two entries, corresponding to Ant16 attn = np.log10(new_pwr[:, :-2] / 1600.) * 10. new_table = (np.clip(orig_table + attn, 0, 30) / 2).astype(int) * 2 DCMlines = [] DCMlines.append( '# Ant1 Ant2 Ant3 Ant4 Ant5 Ant6 Ant7 Ant8 Ant9 Ant10 Ant11 Ant12 Ant13 Ant14 Ant15' ) DCMlines.append( '# X Y X Y X Y X Y X Y X Y X Y X Y X Y X Y X Y X Y X Y X Y X Y' ) DCMlines.append( '# ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- -----' ) for band in range(1, 35): DCMlines.append( '{:2} : {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2} {:2}' .format(band, *new_table[band - 1])) if update: msg = ch.dcm_master_table2sql(DCMlines) if msg: print 'Success' else: print 'Error writing table to SQL database!' return DCMlines
def sql2refcalX(trange, *args, **kwargs): '''same as sql2refcal. trange can be either a timestamp or a timerange.''' import cal_header as ch import stateframe as stf xml, bufs = ch.read_calX(8, t=trange, *args, **kwargs) if isinstance(bufs, list): refcals = [] for i, buf in enumerate(bufs): try: ref = stf.extract(buf, xml['Refcal_Real']) + stf.extract(buf, xml['Refcal_Imag']) * 1j flag = stf.extract(buf, xml['Refcal_Flag']) fghz = stf.extract(buf, xml['Fghz']) sigma = stf.extract(buf, xml['Refcal_Sigma']) timestamp = Time(stf.extract(buf, xml['Timestamp']), format='lv') tbg = Time(stf.extract(buf, xml['T_beg']), format='lv') ted = Time(stf.extract(buf, xml['T_end']), format='lv') pha = np.angle(ref) amp = np.absolute(ref) refcals.append( {'pha': pha, 'amp': amp, 'flag': flag, 'fghz': fghz, 'sigma': sigma, 'timestamp': timestamp, 't_bg': tbg, 't_ed': ted}) except: print 'failed to load record {} ---> {}'.format(i + 1, Time(stf.extract(buf, xml['Timestamp']), format='lv').iso) return refcals elif isinstance(bufs, str): refcal = stf.extract(bufs, xml['Refcal_Real']) + stf.extract(bufs, xml['Refcal_Imag']) * 1j flag = stf.extract(bufs, xml['Refcal_Flag']) fghz = stf.extract(bufs, xml['Fghz']) sigma = stf.extract(bufs, xml['Refcal_Sigma']) timestamp = Time(stf.extract(bufs, xml['Timestamp']), format='lv') tbg = Time(stf.extract(bufs, xml['T_beg']), format='lv') ted = Time(stf.extract(bufs, xml['T_end']), format='lv') pha = np.angle(refcal) amp = np.absolute(refcal) return {'pha': pha, 'amp': amp, 'flag': flag, 'fghz': fghz, 'sigma': sigma, 'timestamp': timestamp, 't_bg': tbg, 't_ed': ted}
def cal_qual(t=None, savfig=True): ''' Check the quality of the total power and gain calibrations for a given date ''' import cal_header as ch from stateframe import extract import dump_tsys as dt import pipeline_cal as pc import matplotlib.pylab as plt import rstn from util import get_idbdir import socket if t is None: t = Time.now() mjd = t.mjd # First check whether the total power calibration is current caltype = 10 xml, buf = ch.read_cal(caltype, t=t) tp_mjd = Time(extract(buf, xml['SQL_timestamp']), format='lv').mjd if mjd - tp_mjd > 0.5: print 'CAL_QUAL: Warning, TP Calibration not (yet) available for this date.' # Find GCAL scan for this date fdb = dt.rd_fdb(Time(mjd, format='mjd')) gcidx, = np.where(fdb['PROJECTID'] == 'GAINCALTEST') if len(gcidx) == 1: datadir = get_idbdir(t) + fdb['FILE'][gcidx][0][3:11] + '/' # List of GCAL files gcalfile = [datadir + i for i in fdb['FILE'][gcidx]] else: print 'CAL_QUAL: Warning, no GAINCALTEST scan for this date. Will try using the GAINCALTEST from previous day.' fdb = dt.rd_fdb(Time(mjd - 1, format='mjd')) gcidx, = np.where(fdb['PROJECTID'] == 'GAINCALTEST') if len(gcidx) == 1: datadir = get_idbdir(t) # Add date path if on pipeline # if datadir.find('eovsa') != -1: datadir += fdb['FILE'][gcidx][0][3:11]+'/' host = socket.gethostname() if host == 'pipeline': datadir += fdb['FILE'][gcidx][0][3:11] + '/' # List of GCAL files gcalfile = [datadir + i for i in fdb['FILE'][gcidx]] else: print 'CAL_QUAL: Error, no GAINCALTEST scan for previous day.' return # Find SOLPNTCAL scan for this date fdb = dt.rd_fdb(Time(mjd, format='mjd')) gcidx, = np.where(fdb['PROJECTID'] == 'SOLPNTCAL') if len(gcidx) > 0: datadir = get_idbdir(t) # Add date path if on pipeline # if datadir.find('eovsa') != -1: datadir += fdb['FILE'][gcidx][0][3:11]+'/' host = socket.gethostname() if host == 'pipeline': datadir += fdb['FILE'][gcidx][0][3:11] + '/' # List of SOLPNTCAL files solpntfile = [datadir + i for i in fdb['FILE'][gcidx]] else: print 'CAL_QUAL: Error, no SOLPNTCAL scan(s) for this date.' return files = gcalfile + solpntfile outnames = [] for file in files: outnames.append( pc.udb_corr(file, calibrate=True, attncal=True, desat=True)) out = ri.read_idb(outnames, srcchk=False) nt = len(out['time']) nf = len(out['fghz']) tpfac = 500. / nf frq, flux = rstn.rd_rstnflux(t) s = rstn.rstn2ant(frq, flux, out['fghz'] * 1000., t) fluximg = s.repeat(nt).reshape(nf, nt) f, ax = plt.subplots(4, 7) f.set_size_inches(16, 7, forward=True) f.tight_layout(rect=[0.0, 0.0, 1, 0.95]) ax.shape = (2, 14) for i in range(13): for j in range(2): ax[j, i].imshow(out['p'][i, j], aspect='auto', origin='lower', vmax=np.max(s), vmin=0) ax[j, i].plot(np.clip(out['p'][i, j, int(nf / 3.)] / tpfac, 0, nf), linewidth=1) ax[j, i].plot(np.clip(out['p'][i, j, int(2 * nf / 3.)] / tpfac, 0, nf), linewidth=1) ax[j, i].set_title('Ant ' + str(i + 1) + [' X Pol', ' Y Pol'][j], fontsize=10) for j in range(2): ax[j, 13].imshow(fluximg, aspect='auto', origin='lower', vmax=np.max(s), vmin=0) ax[j, 13].set_title('RSTN Flux', fontsize=10) for i in range(13): for j in range(2): ax[j, i].plot(np.clip(fluximg[int(nf / 3.)] / tpfac, 0, nf), '--', linewidth=1, color='C0') ax[j, i].plot(np.clip(fluximg[int(2 * nf / 3.)] / tpfac, 0, nf), '--', linewidth=1, color='C1') f.suptitle('Total Power Calibration Quality for ' + t.iso[:10]) date = t.iso[:10].replace('-', '') if savfig: try: plt.savefig('/common/webplots/flaremon/daily/' + date[:4] + '/QUAL_' + date + 'TP.png') except: plt.savefig('/tmp/' + date[:4] + '/QUAL_' + date + 'TP.png') print 'The .png file could not be created in the /common/webplots/flaremon/daily/ folder.' print 'A copy was created in /tmp/.' f, ax = plt.subplots(4, 7) f.set_size_inches(16, 7, forward=True) f.tight_layout(rect=[0.0, 0.0, 1, 0.95]) ax.shape = (2, 14) for i in range(13): for j in range(2): ax[j, i].imshow(np.real(out['a'][i, j]), aspect='auto', origin='lower', vmax=np.max(s), vmin=0) ax[j, i].plot(np.clip(np.real(out['a'][i, j, int(nf / 3.)] / tpfac), 0, nf), linewidth=1) ax[j, i].plot(np.clip( np.real(out['a'][i, j, int(2 * nf / 3.)] / tpfac), 0, nf), linewidth=1) ax[j, i].set_title('Ant ' + str(i + 1) + [' X Pol', ' Y Pol'][j], fontsize=10) for j in range(2): ax[j, 13].imshow(fluximg, aspect='auto', origin='lower', vmax=np.max(s), vmin=0) ax[j, 13].set_title('RSTN Flux', fontsize=10) for i in range(13): for j in range(2): ax[j, i].plot(np.clip(fluximg[int(nf / 3.)] / tpfac, 0, nf), '--', linewidth=1, color='C0') ax[j, i].plot(np.clip(fluximg[int(2 * nf / 3.)] / tpfac, 0, nf), '--', linewidth=1, color='C1') f.suptitle('Cross-Power Calibration Quality for ' + t.iso[:10]) date = t.iso[:10].replace('-', '') if savfig: try: plt.savefig('/common/webplots/flaremon/daily/' + date[:4] + '/QUAL_' + date + 'XP.png') except: plt.savefig('/tmp/' + date[:4] + '/QUAL_' + date + 'XP.png') print 'The .png file could not be created in the /common/webplots/flaremon/daily/ folder.' print 'A copy was created in /tmp/.'
def unrot(data, azeldict=None): ''' Apply the correction to differential feed rotation to data, and return the corrected data. This also applies flags to data whose antennas are not tracking. Inputs: data A dictionary returned by udb_util.py's readXdata(). azeldict The dictionary returned from get_sql_info(), or if None, the appropriate get_sql_info() call is done internally. Output: cdata A dictionary with the phase-corrected data. Only the key x is updated. ''' import copy from util import lobe, bl2ord trange = Time(data['time'][[0, -1]], format='jd') if azeldict is None: azeldict = get_sql_info(trange) chi = azeldict['ParallacticAngle'] * np.pi / 180. # (nt, nant) # Correct parallactic angle for equatorial mounts, relative to Ant14 chi[:, [8, 9, 10, 12, 13]] = 0 # Currently 0, but can be measured and updated # Which antennas are tracking track = azeldict['TrackFlag'] # True if tracking # Ensure that nearest valid parallactic angle is used for times in the data good = np.where(azeldict['ActualAzimuth'] != 0) tidx = [] # List of arrays of indexes for each antenna for i in range(14): gd = good[0][np.where(good[1] == i)] tidx.append(nearest_val_idx(data['time'], azeldict['Time'][gd].jd)) # Read X-Y Delay phase from SQL database and get common frequencies xml, buf = ch.read_cal(11, t=trange[0]) fghz = stateframe.extract(buf, xml['FGHz']) good, = np.where(fghz != 0.) fghz = fghz[good] dph = stateframe.extract(buf, xml['XYphase']) dph = dph[:, good] xi_rot = stateframe.extract(buf, xml['Xi_Rot']) xi_rot = xi_rot[good] fidx1, fidx2 = common_val_idx(data['fghz'], fghz, precision=4) missing = np.setdiff1d(np.arange(len(data['fghz'])), fidx1) nf, nbl, npol, nt = data['x'].shape nf = len(fidx1) # Correct data for X-Y delay phase for i in range(13): for j in range(i + 1, 14): k = bl2ord[i, j] a1 = lobe(dph[i, fidx2] - dph[j, fidx2]) a2 = -dph[j, fidx2] - xi_rot[fidx2] a3 = dph[i, fidx2] - xi_rot[fidx2] + np.pi data['x'][fidx1, k, 1] *= np.repeat(np.exp(1j * a1), nt).reshape(nf, nt) data['x'][fidx1, k, 2] *= np.repeat(np.exp(1j * a2), nt).reshape(nf, nt) data['x'][fidx1, k, 3] *= np.repeat(np.exp(1j * a3), nt).reshape(nf, nt) # Correct data for differential feed rotation cdata = copy.deepcopy(data) for n in range(nt): for i in range(13): for j in range(i + 1, 14): k = bl2ord[i, j] ti = tidx[i][n] tj = tidx[j][n] if track[ti, i] and track[tj, j]: dchi = chi[ti, i] - chi[tj, j] cchi = np.cos(dchi) schi = np.sin(dchi) cdata['x'][:, k, 0, n] = data['x'][:, k, 0, n] * cchi + data['x'][:, k, 3, n] * schi cdata['x'][:, k, 2, n] = data['x'][:, k, 2, n] * cchi + data['x'][:, k, 1, n] * schi cdata['x'][:, k, 3, n] = data['x'][:, k, 3, n] * cchi - data['x'][:, k, 0, n] * schi cdata['x'][:, k, 1, n] = data['x'][:, k, 1, n] * cchi - data['x'][:, k, 2, n] * schi else: cdata['x'][:, k, :, n] = np.ma.masked # Set flags for any missing frequencies (hopefully this also works when "missing" is np.array([])) cdata['x'][missing] = np.ma.masked return cdata
def gain_state(trange=None): ''' Read and assemble the gain state for the given timerange from the SQL database, or for the last 10 minutes if trange is None. Returns the complex attenuation of the FEM for the timerange as an array of size (nant, npol, ntimes) [not band dependent], and the complex attenuation of the DCM for the same timerange as an array of size (nant, npol, nbands, ntimes). Also returns the time as a Time() object array. ''' from util import Time import dbutil as db from fem_attn_calib import fem_attn_update import cal_header as ch if trange is None: t = Time.now() t2 = Time(t.jd - 600. / 86400., format='jd') trange = Time([t2.iso, t.iso]) ts = trange[0].lv # Start timestamp te = trange[1].lv # End timestamp cursor = db.get_cursor() # First get FEM attenuation for timerange D15dict = db.get_dbrecs(cursor, dimension=15, timestamp=trange) DCMoffdict = db.get_dbrecs(cursor, dimension=50, timestamp=trange) DCMoff_v_slot = DCMoffdict['DCMoffset_attn'] # DCMoff_0 = D15dict['DCM_Offset_Attn'][:,0] # All ants are the same fem_attn = {} fem_attn['timestamp'] = D15dict['Timestamp'][:, 0] nt = len(fem_attn['timestamp']) junk = np.zeros([nt, 1], dtype='int') #add the non-existing antenna 16 fem_attn['h1'] = np.append(D15dict['Ante_Fron_FEM_HPol_Atte_First'], junk, axis=1) #FEM hpol first attn value fem_attn['h2'] = np.append(D15dict['Ante_Fron_FEM_HPol_Atte_Second'], junk, axis=1) #FEM hpol second attn value fem_attn['v1'] = np.append(D15dict['Ante_Fron_FEM_VPol_Atte_First'], junk, axis=1) #FEM vpol first attn value fem_attn['v2'] = np.append(D15dict['Ante_Fron_FEM_VPol_Atte_Second'], junk, axis=1) #FEM vpol second attn value fem_attn['ants'] = np.append(D15dict['I15'][0, :], [15]) # Add corrections from SQL database for start time of timerange fem_attn_corr = fem_attn_update(fem_attn, trange[0]) # Next get DCM attenuation for timerange # Getting next earlier scan header ver = db.find_table_version(cursor, ts, True) query = 'select top 50 Timestamp,FSeqList from hV' + ver + '_vD50 where Timestamp <= ' + str( ts) + ' order by Timestamp desc' fseq, msg = db.do_query(cursor, query) if msg == 'Success': fseqlist = fseq['FSeqList'][::-1] # Reverse the order bandlist = ((np.array(fseqlist) - 0.44) * 2).astype(int) cursor.close() # Read current DCM_table from database xml, buf = ch.read_cal(3, trange[0]) orig_table = stf.extract(buf, xml['Attenuation']).astype('int') orig_table.shape = (50, 15, 2) xml, buf = ch.read_cal(6, trange[0]) dcm_attn_bitv = np.nan_to_num(stf.extract( buf, xml['DCM_Attn_Real'])) + np.nan_to_num( stf.extract(buf, xml['DCM_Attn_Imag'])) * 1j # # Add one more bit (all zeros) to take care of unit bit # dcm_attn_bitv = np.concatenate((np.zeros((16,2,1),'int'),dcm_attn_bitv),axis=2) # We now have: # orig_table the original DCM at start of scan, size (nslot, nant=15, npol) # DCMoff_0 the offset applied to all antennas and slots (ntimes) # DCMoff_v_slot the offest applied to all antennas but varies by slot (ntimes, nslot) # dcm_attn_bitv the measured (non-nominal) attenuations for each bit value (nant=16, npol, nbit) -- complex # Now I need to convert slot to band, add appropriately, and organize as (nant=16, npol, nband, ntimes) # Add one more antenna (all zeros) to orig_table orig_table = np.concatenate((orig_table, np.zeros((50, 1, 2), 'int')), axis=1) ntimes, nslot = DCMoff_v_slot.shape dcm_attn = np.zeros((16, 2, 34, ntimes), np.int) for i in range(ntimes): for j in range(50): idx = bandlist[j] - 1 # This adds attenuation for repeated bands--hopefully the same value for each repeat dcm_attn[:, :, idx, i] += orig_table[j, :, :] + DCMoff_v_slot[i, j] # Normalize repeated bands by finding number of repeats and dividing. for i in range(1, 35): n = len(np.where(bandlist == i)[0]) if n > 1: dcm_attn[:, :, i - 1, :] /= n # Make sure attenuation is in range dcm_attn = np.clip(dcm_attn, 0, 30) # Finally, correct for non-nominal (measured) bit values # Start with 0 attenuation as reference dcm_attn_corr = dcm_attn * (0 + 0j) att = np.zeros((16, 2, 34, ntimes, 5), np.complex) # Calculate resulting attenuation based on bit attn values (2,4,8,16) for i in range(4): # Need dcm_attn_bitv[...,i] to be same shape as dcm_attn bigger_bitv = np.broadcast_to(dcm_attn_bitv[..., i], (ntimes, 34, 16, 2)) bigger_bitv = np.swapaxes( np.swapaxes(np.swapaxes(bigger_bitv, 0, 3), 1, 2), 0, 1) att[..., i] = (np.bitwise_and(dcm_attn, 2**(i + 1)) >> (i + 1)) * bigger_bitv dcm_attn_corr = dcm_attn_corr + att[..., i] # Move ntimes column to next to last position, and then sum over last column (the two attenuators) fem_attn_corr = np.sum(np.rollaxis(fem_attn_corr, 0, 3), 3) # Output is FEM shape (nant, npol, ntimes) = (16, 2, ntimes) # DCM shape (nant, npol, nband, ntimes) = (16, 2, 34, ntimes) # Arrays are complex, in dB units tjd = Time(fem_attn['timestamp'].astype('int'), format='lv').jd return fem_attn_corr, dcm_attn_corr, tjd