def rd_miriad_tsys_16(trange,udb=False):
''' Read total power data (TSYS) directly from Miriad files for time range
given by trange. This version works only for 16-ant correlator
Simply calls read_idb and returns a subset of the data with new dictionary keys.
'''
import read_idb
out = read_idb.read_idb(trange)
return {'source':out['source'], 'fghz':out['fghz'], 'ut_mjd':out['time']-2400000.5, 'tsys':out['p']}
def rd_refcal(trange, projid='PHASECAL', srcid=None, quackint=180., navg=3):
'''take a time range from the Time object, e.g., trange=Time(['2017-04-08T05:00','2017-04-08T15:30']),
a projectid, and source id, return visibility data for all baselines correlated with ant 14.
***Optional keywords***
projid: string -- predefined PROJECTID when setting up the observations. Default is 'PHASECAL'
srcid: string -- if provided, then only use the specified source. E.g., '1229+020' is often used for
reference calibration. Default is to use all scans.
quackint: interval in seconds to skip at the beginning of each scan
navg: number of data points to average
***Output dictionary***
'''
import struct, time, glob, sys, socket
import dbutil as db
sclist = findfiles(trange, projid, srcid)
scanlist = sclist['scanlist']
srclist = sclist['srclist']
# read scans one by one
nscan = len(scanlist)
vis = []
bandnames = []
times = []
fghzs = []
has = []
decs = []
for n, scan in enumerate(scanlist):
print 'Reading scan: ' + scan
out = ri.read_idb([scan], navg=navg, quackint=quackint)
times.append(out['time'])
has.append(out['ha'])
decs.append(out['dec'])
nt = len(out['time'])
if nt == 0:
continue
bds, sidx = np.unique(out['band'], return_index=True)
nbd = len(bds)
eidx = np.append(sidx[1:], len(out['band']))
vs = np.zeros((15, 4, 34, nt), dtype=complex)
fghz = np.zeros(34)
# average over channels within each band
for b, bd in enumerate(bds):
fghz[bd - 1] = np.nanmean(out['fghz'][sidx[b]:eidx[b]])
for a in range(13):
for pl in range(4):
vs[a, pl, bd - 1] = np.mean(out['x'][bl2ord[a, 13], pl, sidx[b]:eidx[b]], axis=0)
vis.append(vs)
fghzs.append(fghz)
bandnames.append(bds)
return {'scanlist': scanlist, 'srclist': srclist, 'tstlist': sclist['tstlist'], 'tedlist': sclist['tedlist'],
'vis': vis, 'bandnames': bandnames, 'fghzs': fghzs, 'times': times, 'has':has, 'decs':decs}
def allday_process(path=None):
''' Process an all day list of corrected data files to create total power
and baseline amplitude FITS spectrograms (planned for submission to
NASA SDAC for support of the Parker Solar Probe).
Fixed a problem when nans appear in the data--use nanmean() and nanmedian()
'''
import glob
import read_idb as ri
from xspfits2 import tp_writefits
if path is None:
path = './'
files = glob.glob(path + 'IDB*')
files.sort()
for file in files:
out = ri.read_idb([file])
nant, npol, nf, nt = out['p'].shape
nant = 13
# Use only data from tracking antennas
azeldict = get_sql_info(Time(out['time'], format='jd')[[0, -1]])
idx = nearest_val_idx(out['time'], azeldict['Time'].jd)
tracking = azeldict['TrackFlag'].T
# Flag any data where the antennas are not tracking
for i in range(nant):
out['p'][i, :, :, ~tracking[i, idx]] = np.nan
# Determine best 8 antennas
med = np.nanmean(np.nanmedian(out['p'][:nant], 3), 1) # size nant,nf
medspec = np.nanmedian(med, 0) # size nf
p = np.polyfit(out['fghz'], medspec, 2)
spec = np.polyval(p, out['fghz']).repeat(nant).reshape(
nf, nant) # size nf, nant
stdev = np.std(med - np.transpose(spec), 1) # size nant
idx = stdev.argsort()[:8] # List of 8 best-fitting antennas
# Use list of antennas to get final median total power dynamic spectrum
med = np.nanmean(np.nanmedian(out['p'][idx], 0), 0)
# Write the total power spectrum to a FITS file
tp_writefits(out, med.astype(np.float32), filestem='TP_', outpath='./')
# Form sum of intermediate baselines
baseidx = np.array([
29, 30, 31, 32, 33, 34, 42, 43, 44, 45, 46, 54, 55, 56, 57, 65, 66,
67, 75, 76, 84
])
# Get uv distance for mid-time
#uvdist = np.sqrt(out['uvw'][:,nt/2,0]**2 + out['uvw'][:,nt/2,1]**2 + out['uvw'][:,nt/2,2]**2)
# Sort from low to high uv distance
#bah = uvdist.argsort()
# Use "intermediate" lengths, i.e. 20th to 39th in list, and sum amplitudes
med = np.abs(np.nansum(np.nansum(out['x'][baseidx], 0), 0))
# Write the baseline amplitude spectrum to a FITS file
tp_writefits(out, med.astype(np.float32), filestem='XP_', outpath='./')
def inspect(files, vmin=0.1, vmax=10, ant_str='ant1-13', srcchk=True):
''' Given the list of filenames output by calIDB(), reads and displays a log-scaled
median (over baselines) spectrogram for quick check. Input parameters allow the
displayed spectrogram to be scaled (vmin & vmax, which both should be positive
since the spectrogram is log-scaled), and the list of antennas to use for the
median. The output is the original data (out) and the median spectrogram (not
log scaled or clipped) obtained for baselines between 150 and 1000 nsec involving
the antennas in ant_str.
Note, the display for this routine is just for a quick sanity check to see if
the entire timerange for the flare looks okay. The nicely formatted plot with
background subtraction will be done using make_plot().
Inputs:
files The list of calibrated IDB files (output of calIDB). No default.
vmin, vmax The min and max values to use for the scaling of the quick look plot.
Both should be positive since the plot is log-scaled. Default 0.1, 10
ant_str The standard string of antennas to use (see util.ant_str2list()).
Default is all antennas 'ant1-13'
srcchk Not often needed, if True this forces all of the files to have the same
source name, which is generally desired. Files in the file list that
have different source names are skipped. It can be set to False
to override this behavior. Default is True.
Outputs:
out Standard output dictionary of read_idb, containing all of the data
read from the files. This includes the list of times and frequencies,
but also all of the other data from the files for convenience.
spec The actual spectrogram data obtained from forming the median over
the given antenna list, for baseline lengths 150-1000 nsec. This
is not log-scaled or clipped.
'''
out = ri.read_idb(files, srcchk=srcchk)
times = Time(out['time'],format='jd')
nt, = out['time'].shape
blen = np.sqrt(out['uvw'][:,int(nt/2),0]**2 + out['uvw'][:,int(nt/2),1]**2)
ants = ant_str2list(ant_str)
idx = []
for k,i in enumerate(ants[:-1]):
for j in ants[k+1:]:
idx.append(ri.bl2ord[i,j])
idx = np.array(idx)
good, = np.where(np.logical_and(blen[idx] > 150.,blen[idx] < 1000.))
spec = np.nanmedian(np.abs(out['x'][idx[good],0]),0)
nf, nt = spec.shape
plt.figure()
plt.imshow(np.log10(np.clip(spec,vmin,vmax)))
return out,spec
def rd_miriad_tsys_16(trange, udb=False, auto=False, tref=None):
''' Read total power data (TSYS) directly from Miriad files for time range
given by trange. This version works only for 16-ant correlator
Simply calls read_idb and returns a subset of the data with new dictionary keys.
2018-01-26 Changed to call gc.apply_fem_level() instead of gc.apply_gain_corr().
'''
import gaincal2 as gc
import read_idb
out = read_idb.read_idb(trange)
#cout = gc.apply_gain_corr(out, tref=tref)
try:
cout = gc.apply_fem_level(out)
except:
cout = out
if auto:
return {'source':out['source'], 'fghz':out['fghz'], 'ut_mjd':out['time']-2400000.5, 'tsys':np.real(cout['a'][:,:2])}
else:
return {'source':out['source'], 'fghz':out['fghz'], 'ut_mjd':out['time']-2400000.5, 'tsys':cout['p']}
def make_bl_table(trange,antstr='ant1-13'):
files = ri.get_trange_files(trange)
bl2ord = ri.p.bl_list()
antidx = ant_str2list(antstr)
fxx = open(trange[0].iso[:10].replace('-','')+'_baseline_xx.txt','w')
fyy = open(trange[0].iso[:10].replace('-','')+'_baseline_yy.txt','w')
fxy = open(trange[0].iso[:10].replace('-','')+'_baseline_xy.txt','w')
fyx = open(trange[0].iso[:10].replace('-','')+'_baseline_yx.txt','w')
for i,file in enumerate(files):
print 'Reading',file,'('+str(i+1)+' of '+str(len(files))+')'
out = ri.read_idb([file],navg=60)
data = out['x'][bl2ord[antidx,13],:,:,1:]
phase = np.angle(np.mean(data,3))*180./np.pi
amp = np.abs(np.mean(data,3))
x = np.mean(np.real(data),3)
y = np.mean(np.imag(data),3)
dx = np.std(np.real(data),3)
dy = np.std(np.imag(data),3)
dphase = (np.sqrt(x**2 * dy**2 + y**2 * dx**2)/amp**2)*180./np.pi
xx = phase[:,0,0]
yy = phase[:,1,0]
xy = phase[:,2,0]
yx = phase[:,3,0]
dxx = dphase[:,0,0]
dyy = dphase[:,1,0]
dxy = dphase[:,2,0]
dyx = dphase[:,3,0]
outdict = src2dict(out)
h = np.mean(outdict['ha'])
fxx.write(outdict['source'].name+' {:6.2f} {:6.2f} '.format(h*180./np.pi,outdict['dec']*180/np.pi)+('{:6.1f} '*10).format(*xx)+'\n')
fyy.write(outdict['source'].name+' {:6.2f} {:6.2f} '.format(h*180./np.pi,outdict['dec']*180/np.pi)+('{:6.1f} '*10).format(*yy)+'\n')
fxy.write(outdict['source'].name+' {:6.2f} {:6.2f} '.format(h*180./np.pi,outdict['dec']*180/np.pi)+('{:6.1f} '*10).format(*xy)+'\n')
fyx.write(outdict['source'].name+' {:6.2f} {:6.2f} '.format(h*180./np.pi,outdict['dec']*180/np.pi)+('{:6.1f} '*10).format(*yx)+'\n')
fxx.close()
fyy.close()
fxy.close()
fyx.close()
def rd_miriad_tsys_16(trange,
udb=False,
auto=False,
tref=None,
skycal=None,
desat=False):
''' Read total power data (TSYS) directly from Miriad files for time range
given by trange. This version works only for 16-ant correlator
Simply calls read_idb and returns a subset of the data with new dictionary keys.
2018-01-26 Changed to call gc.apply_fem_level() instead of gc.apply_gain_corr().
2019-05-21 Added skycal keyword, simply for passing through to apply_fem_level().
'''
import gaincal2 as gc
import read_idb
import calibration as cal
out = read_idb.read_idb(trange, desat=desat)
#cout = gc.apply_gain_corr(out, tref=tref)
try:
cout = gc.apply_fem_level(out, skycal=skycal)
except:
print 'RD_MIRIAD_TSYS_16: Error applying FEM level correction. No correction applied'
cout = out
if auto:
return {
'source': out['source'],
'fghz': out['fghz'],
'ut_mjd': out['time'] - 2400000.5,
'tsys': np.real(cout['a'][:, :2])
}
else:
return {
'source': out['source'],
'fghz': out['fghz'],
'ut_mjd': out['time'] - 2400000.5,
'tsys': cout['p']
}
def xdata_display(t, ax=None):
''' Given the time as a Time object, search the FDB file for files
associated with the scan for that time and create a dynamic spectrogram
on the axis specified by ax, or on a new plot if no ax. If the requested
time is more than 10 minutes after the last file of that scan, returns
None to indicate no plot.
'''
import time
import dump_tsys
#import get_X_data2 as gd
import read_idb as ri
import spectrogram_fit as sp
fdb = dump_tsys.rd_fdb(t)
# Get files from next day, in case scan extends past current day
t1 = Time(t.mjd + 1, format='mjd')
fdb1 = dump_tsys.rd_fdb(t1)
# Concatenate the two days (if the second day exists)
if fdb1 != {}:
for key in fdb.keys():
fdb[key] = np.concatenate((fdb[key], fdb1[key]))
# Find unique scan IDs
scans, idx = np.unique(fdb['SCANID'], return_index=True)
# Limit to scans in 'NormalObserving' mode
good, = np.where(fdb['PROJECTID'][idx] == 'NormalObserving')
if len(good) > 0:
scans = scans[good]
else:
print 'No NormalObserving scans found.'
return None, None, None
# Find scanID that starts earlier than, but closest to, the current time
for i, scan in enumerate(scans):
dt = t - Time(
time.strftime('%Y-%m-%d %H:%M:%S',
time.strptime(scan, '%y%m%d%H%M%S')))
if dt.sec > 0.:
iout = i
scan = scans[iout]
# Find files for this scan
fidx, = np.where(fdb['SCANID'] == scan)
tlevel = None
bflag = None
if len(fidx) > 0:
files = fdb['FILE'][fidx]
# Find out how old last file of this scan is, and proceed only if less than 20 minutes
# earlier than the time given in t.
try:
dt = t - Time(
time.strftime('%Y-%m-%d %H:%M:%S',
time.strptime(files[-1], 'IDB%Y%m%d%H%M%S')))
except:
dt = 10000. # Forces skip of plot creation
print 'Unexpected FDB file format.'
scan = None
if dt.sec < 1200.:
# This is a currently active scan, so create the figure
for i in range(len(files)):
files[i] = '/data1/IDB/' + files[i]
# data, uvw, fghz, times = gd.get_X_data(files)
out = ri.read_idb(files)
out = ri.flag_sk(out)
fghz = out['fghz']
times = Time(out['time'], format='jd')
data = out['x']
if ax is not None:
datstr = times[0].iso[:10]
ax.set_xlabel('Time [UT on ' + datstr + ']')
ax.set_ylabel('Frequency [GHz]')
ax.set_title('EOVSA Summed Cross-Correlation Amplitude for ' +
datstr)
sp.plot_spectrogram(fghz,
times,
sum(sum(abs(data[0:11, :]), 1), 0),
ax=ax,
logsample=None,
xdata=True,
cbar=True)
tlevel, bflag = flaremeter(data)
else:
print 'Time', dt.sec, 'is > 1200 s after last file of last NormalObserving scan. No plot created.'
scan = None
else:
print 'No files found for this scan ID', scan
scan = None
return scan, tlevel, bflag, times
def allday_udb(t=None,
doplot=True,
goes_plot=True,
savfig=False,
gain_corr=True):
# Plots (and returns) UDB data for an entire day
from flare_monitor import flare_monitor
if t is None:
t = Time.now()
# Cannot get a GOES plot unless doplot is True
if goes_plot: doplot = True
date = t.iso[:10].replace('-', '')
# Look also at the following day, up to 9 UT
date2 = Time(t.mjd + 1, format='mjd').iso[:10].replace('-', '')
year = date[:4]
files = glob.glob('/data1/eovsa/fits/UDB/' + year + '/UDB' + date + '*')
files.sort()
files2 = glob.glob('/data1/eovsa/fits/UDB/' + year + '/UDB' + date2 + '0*')
files2.sort()
files = np.concatenate((np.array(files), np.array(files2)))
# Eliminate files starting before 10 UT on date (but not on date2)
for i, file in enumerate(files):
if file[-6] != '0':
break
try:
files = files[i:]
except:
print 'No files found in /data1/eovsa/fits/UDB/ for', date
return {}
out = ri.read_idb(files, src='Sun')
if gain_corr:
import gaincal2 as gc
out = gc.apply_gain_corr(out)
trange = Time(out['time'][[0, -1]], format='jd')
fghz = out['fghz']
if doplot:
import matplotlib.pylab as plt
from matplotlib.dates import DateFormatter
f, ax = plt.subplots(1, 1, figsize=(14, 5))
pdata = np.sum(np.sum(np.abs(out['x'][0:11, :]), 1),
0) # Spectrogram to plot
X = np.sort(pdata.flatten()) # Sorted, flattened array
# Set any time gaps to nan
tdif = out['time'][1:] - out['time'][:-1]
bad, = np.where(tdif > 120. / 86400) # Time gaps > 2 minutes
pdata[:, bad] = 0
vmax = X[int(len(X) * 0.95)] # Clip at 5% of points
im = ax.pcolormesh(Time(out['time'], format='jd').plot_date,
out['fghz'],
pdata,
vmax=vmax)
plt.colorbar(im, ax=ax, label='Amplitude [arb. units]')
ax.xaxis_date()
ax.xaxis.set_major_formatter(DateFormatter("%H:%M"))
ax.set_ylim(fghz[0], fghz[-1])
ax.set_xlabel('Time [UT]')
ax.set_ylabel('Frequency [GHz]')
ax.set_title('EOVSA 1-min Data for ' + t.iso[:10])
f.autofmt_xdate(bottom=0.15)
if goes_plot:
#from sunpy import lightcurve
#from sunpy.time import TimeRange
# Initially assign GOES times as None
goes_t = None
goes_t2 = None
# Get GOES data for overplotting
#goes_tr = TimeRange(trange.iso)
goes_label = [' A', ' B', ' C', ' M', ' X']
# The GOES label is placed to start 20 min into the day
goes_label_time = Time(out['time'][[0]],
format='jd').plot_date + 0.014
rightaxis_label_time = trange[1].plot_date
# Retrieve GOES data for the day, but this only goes to end of UT day
goes_t, goes_data = get_goes_data(trange[0])
if int(trange[1].mjd) != int(trange[0].mjd):
goes_t2, goes_data2 = get_goes_data(trange[1])
if goes_t is None and goes_t2 is None:
ax.text(goes_label_time,
12,
'GOES soft x-ray data missing',
color='yellow')
else:
if not goes_t is None:
goes_data = 2 * (np.log10(goes_data + 1.e-9)) + 26
ax.plot_date(goes_t, goes_data, '-', color='yellow')
ytext = np.median(goes_data) - 1
if not goes_t2 is None:
goes_data2 = 2 * (np.log10(goes_data2 + 1.e-9)) + 26
ax.plot_date(goes_t2, goes_data2, '-', color='yellow')
ytext2 = np.median(goes_data2) - 1
if ytext:
ytext = (ytext + ytext2) / 2
else:
ytext = ytext2
ax.text(goes_label_time,
ytext,
'GOES soft x-ray data',
color='yellow')
# try:
# goes = lightcurve.GOESLightCurve.create(goes_tr)
# if len(np.where(goes.data['xrsb'] != 0.0)[0]) < 100:
# # Looks like the GOES data are all zero, so just skip it
# ax.text (goes_label_time, 12, 'GOES soft x-ray data missing', color = 'yellow')
# else:
# goes.data['xrsb'] = 2* (np.log10(goes.data['xrsb'] + 1.e-9)) + 26
# ytext = np.median(goes.data['xrsb']) - 1
# ax.text (goes_label_time, ytext, 'GOES soft x-ray data', color = 'yellow')
# goes.data['xrsb'].plot(color = 'yellow')
# except:
# # Looks like the GOES data do not exist, so just skip it
# ax.text (goes_label_time, 12, 'GOES soft x-ray data missing', color = 'yellow')
for k, i in enumerate([10, 12, 14, 16, 18]):
ax.text(rightaxis_label_time,
i - 0.4,
goes_label[k],
fontsize='12')
ax.plot_date(rightaxis_label_time + np.array([-0.005, 0.0]),
[i, i],
'-',
color='yellow')
# try:
# # If the day goes past 0 UT, get GOES data for the next UT day
# if int(trange[1].mjd) != int(trange[0].mjd):
# goes_tr2 = TimeRange([trange[1].iso[:10], trange[1].iso])
# goesday2 = lightcurve.GOESLightCurve.create(goes_tr2)
# if len(np.where(goesday2.data['xrsb'] != 0.0)[0]) < 100:
# pass
# else:
# goesday2.data['xrsb'] = 2* (np.log10(goesday2.data['xrsb'] + 1.e-9)) + 26
# goesday2.data['xrsb'].plot(color = 'yellow')
# except:
# # Looks like the GOES data do not exist, so just skip it
# pass
pstart = Time(t.iso[:10] + ' 13:30').plot_date
prange = [pstart, pstart + 13. / 24]
ax.set_xlim(prange)
ut, fl, projdict = flare_monitor(t)
if fl == []:
print 'Error retrieving data for', t.iso[:10], 'from SQL database.'
return
if projdict == {}:
print 'No annotation can be added to plot for', t.iso[:10]
else:
defcolor = '#ff7f0e'
nscans = len(projdict['Timestamp'])
SOS = Time(projdict['Timestamp'], format='lv').plot_date
EOS = Time(projdict['EOS'], format='lv').plot_date
yran = np.array(ax.get_ylim())
for i in range(nscans):
uti = SOS[i] * np.array([1., 1.])
#if uti[0] >= trange[0].plot_date:
ax.plot_date(uti, yran, 'g', lw=0.5)
if projdict['Project'][i] == 'NormalObserving' or projdict[
'Project'][i] == 'Normal Observing':
ax.text(uti[0],
yran[1] * 0.935,
'SUN',
fontsize=8,
color=defcolor,
clip_on=True)
elif projdict['Project'][i] == 'None':
ax.text(uti[0],
yran[1] * 0.975,
'IDLE',
fontsize=8,
color=defcolor,
clip_on=True)
elif projdict['Project'][i][:4] == 'GAIN':
ax.text(uti[0],
yran[1] * 0.955,
'GCAL',
fontsize=8,
color=defcolor,
clip_on=True)
elif projdict['Project'][i] == 'SOLPNTCAL':
ax.text(uti[0],
yran[1] * 0.955,
'TPCAL',
fontsize=8,
color=defcolor,
clip_on=True)
elif projdict['Project'][i] == 'PHASECAL':
ax.text(uti[0],
yran[1] * 0.955,
'PCAL',
fontsize=8,
color=defcolor,
clip_on=True)
else:
ax.text(uti[0],
yran[1] * 0.975,
projdict['Project'][i],
fontsize=8,
color=defcolor,
clip_on=True)
known = ['GAIN', 'PHAS',
'SOLP'] # known calibration types (first 4 letters)
for i in range(nscans):
uti = EOS[i] * np.array([1., 1.])
ax.plot_date(uti, yran, 'r--', lw=0.5)
uti = np.array([SOS[i], EOS[i]])
if projdict['Project'][i] == 'NormalObserving':
ax.plot_date(uti,
yran[1] * np.array([0.93, 0.93]),
ls='-',
marker='None',
color='#aaffaa',
lw=2,
solid_capstyle='butt')
elif projdict['Project'][i][:4] in known:
ax.plot_date(uti,
yran[1] * np.array([0.95, 0.95]),
ls='-',
marker='None',
color='#aaaaff',
lw=2,
solid_capstyle='butt')
else:
ax.plot_date(uti,
yran[1] * np.array([0.97, 0.97]),
ls='-',
marker='None',
color='#ffaaaa',
lw=2,
solid_capstyle='butt')
if savfig:
plt.savefig('/common/webplots/flaremon/daily/' + date[:4] +
'/XSP' + date + '.png',
bbox_inches='tight')
return out
def cal_qual(t=None, savfig=True):
''' Check the quality of the total power and gain calibrations for a given date
'''
import cal_header as ch
from stateframe import extract
import dump_tsys as dt
import pipeline_cal as pc
import matplotlib.pylab as plt
import rstn
from util import get_idbdir
import socket
if t is None:
t = Time.now()
mjd = t.mjd
# First check whether the total power calibration is current
caltype = 10
xml, buf = ch.read_cal(caltype, t=t)
tp_mjd = Time(extract(buf, xml['SQL_timestamp']), format='lv').mjd
if mjd - tp_mjd > 0.5:
print 'CAL_QUAL: Warning, TP Calibration not (yet) available for this date.'
# Find GCAL scan for this date
fdb = dt.rd_fdb(Time(mjd, format='mjd'))
gcidx, = np.where(fdb['PROJECTID'] == 'GAINCALTEST')
if len(gcidx) == 1:
datadir = get_idbdir(t) + fdb['FILE'][gcidx][0][3:11] + '/'
# List of GCAL files
gcalfile = [datadir + i for i in fdb['FILE'][gcidx]]
else:
print 'CAL_QUAL: Warning, no GAINCALTEST scan for this date. Will try using the GAINCALTEST from previous day.'
fdb = dt.rd_fdb(Time(mjd - 1, format='mjd'))
gcidx, = np.where(fdb['PROJECTID'] == 'GAINCALTEST')
if len(gcidx) == 1:
datadir = get_idbdir(t)
# Add date path if on pipeline
# if datadir.find('eovsa') != -1: datadir += fdb['FILE'][gcidx][0][3:11]+'/'
host = socket.gethostname()
if host == 'pipeline': datadir += fdb['FILE'][gcidx][0][3:11] + '/'
# List of GCAL files
gcalfile = [datadir + i for i in fdb['FILE'][gcidx]]
else:
print 'CAL_QUAL: Error, no GAINCALTEST scan for previous day.'
return
# Find SOLPNTCAL scan for this date
fdb = dt.rd_fdb(Time(mjd, format='mjd'))
gcidx, = np.where(fdb['PROJECTID'] == 'SOLPNTCAL')
if len(gcidx) > 0:
datadir = get_idbdir(t)
# Add date path if on pipeline
# if datadir.find('eovsa') != -1: datadir += fdb['FILE'][gcidx][0][3:11]+'/'
host = socket.gethostname()
if host == 'pipeline': datadir += fdb['FILE'][gcidx][0][3:11] + '/'
# List of SOLPNTCAL files
solpntfile = [datadir + i for i in fdb['FILE'][gcidx]]
else:
print 'CAL_QUAL: Error, no SOLPNTCAL scan(s) for this date.'
return
files = gcalfile + solpntfile
outnames = []
for file in files:
outnames.append(
pc.udb_corr(file, calibrate=True, attncal=True, desat=True))
out = ri.read_idb(outnames, srcchk=False)
nt = len(out['time'])
nf = len(out['fghz'])
tpfac = 500. / nf
frq, flux = rstn.rd_rstnflux(t)
s = rstn.rstn2ant(frq, flux, out['fghz'] * 1000., t)
fluximg = s.repeat(nt).reshape(nf, nt)
f, ax = plt.subplots(4, 7)
f.set_size_inches(16, 7, forward=True)
f.tight_layout(rect=[0.0, 0.0, 1, 0.95])
ax.shape = (2, 14)
for i in range(13):
for j in range(2):
ax[j, i].imshow(out['p'][i, j],
aspect='auto',
origin='lower',
vmax=np.max(s),
vmin=0)
ax[j, i].plot(np.clip(out['p'][i, j, int(nf / 3.)] / tpfac, 0, nf),
linewidth=1)
ax[j, i].plot(np.clip(out['p'][i, j, int(2 * nf / 3.)] / tpfac, 0,
nf),
linewidth=1)
ax[j, i].set_title('Ant ' + str(i + 1) + [' X Pol', ' Y Pol'][j],
fontsize=10)
for j in range(2):
ax[j, 13].imshow(fluximg,
aspect='auto',
origin='lower',
vmax=np.max(s),
vmin=0)
ax[j, 13].set_title('RSTN Flux', fontsize=10)
for i in range(13):
for j in range(2):
ax[j, i].plot(np.clip(fluximg[int(nf / 3.)] / tpfac, 0, nf),
'--',
linewidth=1,
color='C0')
ax[j, i].plot(np.clip(fluximg[int(2 * nf / 3.)] / tpfac, 0, nf),
'--',
linewidth=1,
color='C1')
f.suptitle('Total Power Calibration Quality for ' + t.iso[:10])
date = t.iso[:10].replace('-', '')
if savfig:
try:
plt.savefig('/common/webplots/flaremon/daily/' + date[:4] +
'/QUAL_' + date + 'TP.png')
except:
plt.savefig('/tmp/' + date[:4] + '/QUAL_' + date + 'TP.png')
print 'The .png file could not be created in the /common/webplots/flaremon/daily/ folder.'
print 'A copy was created in /tmp/.'
f, ax = plt.subplots(4, 7)
f.set_size_inches(16, 7, forward=True)
f.tight_layout(rect=[0.0, 0.0, 1, 0.95])
ax.shape = (2, 14)
for i in range(13):
for j in range(2):
ax[j, i].imshow(np.real(out['a'][i, j]),
aspect='auto',
origin='lower',
vmax=np.max(s),
vmin=0)
ax[j,
i].plot(np.clip(np.real(out['a'][i, j, int(nf / 3.)] / tpfac),
0, nf),
linewidth=1)
ax[j, i].plot(np.clip(
np.real(out['a'][i, j, int(2 * nf / 3.)] / tpfac), 0, nf),
linewidth=1)
ax[j, i].set_title('Ant ' + str(i + 1) + [' X Pol', ' Y Pol'][j],
fontsize=10)
for j in range(2):
ax[j, 13].imshow(fluximg,
aspect='auto',
origin='lower',
vmax=np.max(s),
vmin=0)
ax[j, 13].set_title('RSTN Flux', fontsize=10)
for i in range(13):
for j in range(2):
ax[j, i].plot(np.clip(fluximg[int(nf / 3.)] / tpfac, 0, nf),
'--',
linewidth=1,
color='C0')
ax[j, i].plot(np.clip(fluximg[int(2 * nf / 3.)] / tpfac, 0, nf),
'--',
linewidth=1,
color='C1')
f.suptitle('Cross-Power Calibration Quality for ' + t.iso[:10])
date = t.iso[:10].replace('-', '')
if savfig:
try:
plt.savefig('/common/webplots/flaremon/daily/' + date[:4] +
'/QUAL_' + date + 'XP.png')
except:
plt.savefig('/tmp/' + date[:4] + '/QUAL_' + date + 'XP.png')
print 'The .png file could not be created in the /common/webplots/flaremon/daily/ folder.'
print 'A copy was created in /tmp/.'
def fem_attn_anal(idb_calib='/dppdata1/IDB/IDB20160731231934/',doplot=False, wrt2sql=False):
import cal_header as ch
'''Calculate additional corrections to the FEM attenuators at each bit change (0, 1, 2, 4, 8, 16)dB;
Values are based on the measurement IDB20160731231934, the sequence is specified in
helios:Dropbox/PythonCode/Current/FEATTNTEST2.ctl:
1. FEMAUTO-OFF, 2. FEMATTN 15 (both 31 dB, to get the bkg),
3. Change the 1st H and V attn to 0, 1, 2, 4, 8, 16 dB every 30s, while keeping the 2nd to be 8 dB,
4. Change the 2nd H and V attn to 0, 1, 2, 4, 8, 16 dB every 30s, while keeping the 1st to be 8 dB
return value:
fem_attn_inc (nant, npol, # of FEM attn, 5):
additional corrections in dB w.r.t. the nominal values when bit changes'''
import matplotlib.pyplot as plt
out=ri.read_idb([idb_calib])
nant, npol, nf, nt=out['p'].shape
if doplot:
# show the auto-correlation, use it to find time indices for each attn state
f, ax = plt.subplots(5,3)
for i in range(15):
ax[i / 3, i % 3].imshow(out['p'][i,0])
# define time idx ranges
# each state lasted 30 s
tidxs=[25+i*30 for i in range(13)] #begin idx for avg
tidxe=[idx+20 for idx in tidxs] #end idx for avg
bkg=np.mean(out['p'][:,:,:,tidxs[0]:tidxe[0]],axis=3)
# measurements for 12 attn states
p_1=np.zeros([nant,npol,nf,6]) #power values for 6 states of 1st FEM attn change
p_2=np.zeros([nant,npol,nf,6]) #power values for 6 states of 2nd FEM attn change
rp_1=np.zeros([nant,npol,nf,6]) #ratio of the power regarding to the reference state
rp_2=np.zeros([nant,npol,nf,6])
rdb_1=np.zeros([nant,npol,nf,6]) #power ratio converted to dB
rdb_2=np.zeros([nant,npol,nf,6])
# nominal dB values
attns1=np.array([0.,1.,2.,4.,8.,16.])
attns2=np.array([0.,1.,2.,4.,8.,16.])
# reference state
attn_idx_ref=0 # 0 dB state
attns1-=attns1[attn_idx_ref]
attns2-=attns2[attn_idx_ref]
for i in range(12):
if i < 6:
p_1[:,:,:,i]=np.mean(out['p'][:,:,:,tidxs[i+1]:tidxe[i+1]],axis=3)-bkg
else:
p_2[:,:,:,i-6]=np.mean(out['p'][:,:,:,tidxs[i+1]:tidxe[i+1]],axis=3)-bkg
for i in range(6):
rp_1[:,:,:,i]=p_1[:,:,:,i]/p_1[:,:,:,attn_idx_ref]
rp_2[:,:,:,i]=p_2[:,:,:,i]/p_2[:,:,:,attn_idx_ref]
rdb_1=-10.*np.log10(rp_1)
rdb_2=-10.*np.log10(rp_2)
ddb_1=rdb_1-attns1 # additional dB correction wrt the nominal values, FEM attn 1
ddb_2=rdb_2-attns2 # additional dB correction wrt the nominal values, FEM attn 2
# plot the additional dB correction
if doplot:
f1, ax1 = plt.subplots(6,5)
for i in range(15):
ax1[i / 5, i % 5].imshow(ddb_1[i,0],vmin=-2,vmax=2)
ax1[i / 5+3, i % 5].imshow(ddb_1[i,1],vmin=-2,vmax=2)
f2, ax2 = plt.subplots(6,5)
for i in range(15):
ax2[i / 5, i % 5].imshow(ddb_2[i,0],vmin=-2,vmax=2)
ax2[i / 5+3, i % 5].imshow(ddb_2[i,1],vmin=-2,vmax=2)
# generate corrections for the nominal values
chran=[0,90] #range of frequency channels to average
fem_attn_bitv=np.zeros([16,npol,2,5],dtype=np.complex)
fem_attn_bitv[:,:,0,:]=np.nan_to_num(np.mean(rdb_1[:,:,:,1:],axis=2))+0j
fem_attn_bitv[:,:,1,:]=np.nan_to_num(np.mean(rdb_2[:,:,:,1:],axis=2))+0j
if wrt2sql:
ch.fem_attn_val2sql(fem_attn_bitv,t=Time(out['time'][0],format='jd'))
return fem_attn_bitv
def DCM_attn_anal(filename):
''' Analyze a DCMATTNTEST observation to determine the 2- and 4-bit
attenuation values. Input is a Miriad file. Returns two arrays,
at2 and at4 of size (nant,npol) = (13,2)
representing the attenuation, in dB, of the 2- and 4-bit, resp.
'''
import read_idb as ri
import dbutil as db
import cal_header as ch
import stateframe as stf
import copy
from util import Time
import matplotlib.pylab as plt
out = ri.read_idb([filename])
ts = int(Time(out['time'][0], format='jd').lv + 0.5)
te = int(Time(out['time'][-1], format='jd').lv + 0.5)
query = 'select Timestamp,DCM_Offset_Attn from fV65_vD15 where Timestamp between ' + str(
ts) + ' and ' + str(te) + ' order by Timestamp'
cursor = db.get_cursor()
data, msg = db.do_query(cursor, query)
cursor.close()
dcm_offset = data['DCM_Offset_Attn'].reshape(
len(data['DCM_Offset_Attn']) / 15, 15)
dcm_offset = dcm_offset[:, 0] # All antennas are the same
t = Time(out['time'][0], format='jd')
xml, buf = ch.read_cal(2, t)
table = stf.extract(buf, xml['Attenuation'])
bandlist = ((out['fghz'] - 0.5) * 2).astype(int)
tbl = table[bandlist - 1]
tbl.shape = (len(bandlist), 15, 2)
tbl = np.swapaxes(np.swapaxes(tbl, 0, -1), 0, 1)
tbl2 = np.broadcast_to(tbl, (out['time'].shape[0], 15, 2, 134))
tbl = copy.copy(np.rollaxis(tbl2, 0, 4)) # Shape (nant,npol,nf,nt)
pwr = out['p'][:15] # Shape (nant,npol,nf,nt)
# Add value of dcm_offset to table
for i, offset in enumerate(dcm_offset):
tbl[:, :, :, i] += offset
# Clip to valid attenuations
tbl = np.clip(tbl, 0, 30)
# Isolate good times in various attn states
goodm2, = np.where(dcm_offset == -2)
goodm2 = goodm2[2:-3]
good2, = np.where(dcm_offset == 2)
good2 = good2[2:-3]
good0, = np.where(dcm_offset[goodm2[-1]:good2[0]] == 0)
good0 += goodm2[-1]
good0 = good0[2:-3]
good4, = np.where(dcm_offset == 4)
good4 = good4[2:-3]
good6, = np.where(dcm_offset == 6)
good6 = good6[2:-3]
goodbg = good6 + 30 # Assumes FEMATTN 15 follows good6 30 s later
# Perform median over good times and create pwrmed with medians
# The 5 indexes correspond to dcm_offsets -2, 0, 2, 4 and 6
nant, npol, nf, nt = pwr.shape
pwrmed = np.zeros((nant, npol, nf, 5))
# Do not forget to subtract the background
bg = np.median(pwr[:, :, :, goodbg], 3)
pwrmed[:, :, :, 0] = np.median(pwr[:, :, :, goodm2], 3) - bg
pwrmed[:, :, :, 1] = np.median(pwr[:, :, :, good0], 3) - bg
pwrmed[:, :, :, 2] = np.median(pwr[:, :, :, good2], 3) - bg
pwrmed[:, :, :, 3] = np.median(pwr[:, :, :, good4], 3) - bg
pwrmed[:, :, :, 4] = np.median(pwr[:, :, :, good6], 3) - bg
good = np.array([goodm2[0], good0[0], good2[0], good4[0], good6[0]])
tbl = tbl[:, :, :, good]
at2 = np.zeros((13, 2), float)
at4 = np.zeros((13, 2), float)
at8 = np.zeros((13, 2), float)
f1, ax1 = plt.subplots(2, 13)
f2, ax2 = plt.subplots(2, 13)
f3, ax3 = plt.subplots(2, 13)
for ant in range(13):
for pol in range(2):
pts = []
for i in range(4):
for v in [0, 4, 8, 12, 16, 20, 24, 28]:
idx, = np.where(tbl[ant, pol, :, i] == v)
if len(idx) != 0:
good, = np.where((tbl[ant, pol, idx, i] +
2) == tbl[ant, pol, idx, i + 1])
if len(good) != 0:
pts.append(pwrmed[ant, pol, idx[good], i] /
pwrmed[ant, pol, idx[good], i + 1])
pts = np.concatenate(pts)
ax1[pol, ant].plot(pts, '.')
ax1[pol, ant].set_ylim(0, 2)
at2[ant, pol] = np.log10(np.median(pts)) * 10.
pts = []
for i in range(3):
for v in [0, 2, 8, 10, 16, 18, 24, 26]:
idx, = np.where(tbl[ant, pol, :, i] == v)
if len(idx) != 0:
good, = np.where((tbl[ant, pol, idx, i] +
4) == tbl[ant, pol, idx, i + 2])
if len(good) != 0:
pts.append(pwrmed[ant, pol, idx[good], i] /
pwrmed[ant, pol, idx[good], i + 2])
pts = np.concatenate(pts)
ax2[pol, ant].plot(pts, '.')
ax2[pol, ant].set_ylim(0, 3)
at4[ant, pol] = np.log10(np.median(pts)) * 10.
pts = []
i = 0
for v in [0, 2, 4, 6, 16, 18, 20, 22]:
idx, = np.where(tbl[ant, pol, :, i] == v)
if len(idx) != 0:
good, = np.where((tbl[ant, pol, idx, i] + 8) == tbl[ant,
pol,
idx,
i + 4])
if len(good) != 0:
pts.append(pwrmed[ant, pol, idx[good], i] /
pwrmed[ant, pol, idx[good], i + 4])
try:
pts = np.concatenate(pts)
except:
# Looks like there were no points for this antenna/polarization, so set to nominal attn
pts = [6.30957, 6.30957, 6.30957]
ax3[pol, ant].plot(pts, '.')
ax3[pol, ant].set_ylim(5, 8)
at8[ant, pol] = np.log10(np.median(pts)) * 10.
plt.show()
# Generate output table, a complex array of size (nant,npol,nbits)
attn = np.zeros((16, 2, 4), np.complex)
# Set to nominal values, then overwrite with measured ones
for i in range(16):
for j in range(2):
attn[i, j] = [2.0 + 0j, 4.0 + 0j, 8.0 + 0j, 16.0 + 0j]
attn[:13, :, 0] = at2 + 0j
attn[:13, :, 1] = at4 + 0j
attn[:13, :, 2] = at8 + 0j
return attn
def rd_refcal(trange, projid='PHASECAL', srcid=None, quackint=180., navg=3):
'''take a time range from the Time object, e.g., trange=Time(['2017-04-08T05:00','2017-04-08T15:30']),
a projectid, and source id, return visibility data for all baselines correlated with ant 14.
***Optional keywords***
projid: string -- predefined PROJECTID when setting up the observations. Default is 'PHASECAL'
srcid: string -- if provided, then only use the specified source. E.g., '1229+020' is often used for
reference calibration. Default is to use all scans.
quackint: interval in seconds to skip at the beginning of each scan
navg: number of data points to average
***Output dictionary***
'''
import struct, time, glob, sys, socket
import dbutil as db
sclist = findfiles(trange, projid, srcid)
scanlist = sclist['scanlist']
srclist = sclist['srclist']
# read scans one by one
nscan = len(scanlist)
vis = []
bandnames = []
times = []
fghzs = []
has = []
decs = []
good = []
for n, scan in enumerate(scanlist):
print 'Reading scan: ' + scan
out = ri.read_idb([scan], navg=navg, quackint=quackint)
nt = len(out['time'])
if nt == 0:
# If there are no times in the scan, skip this scan entirely
continue
good.append(n)
times.append(out['time'])
has.append(out['ha'])
decs.append(out['dec'])
bds, sidx = np.unique(out['band'], return_index=True)
nbd = len(bds)
eidx = np.append(sidx[1:], len(out['band']))
vs = np.zeros((15, 4, 34, nt), dtype=complex)
fghz = np.zeros(34)
# average over channels within each band
for b, bd in enumerate(bds):
fghz[bd - 1] = np.nanmean(out['fghz'][sidx[b]:eidx[b]])
for a in range(13):
for pl in range(4):
vs[a, pl, bd - 1] = np.mean(out['x'][bl2ord[a, 13], pl,
sidx[b]:eidx[b]],
axis=0)
vis.append(vs)
fghzs.append(fghz)
bandnames.append(bds)
# Keep only good scans
gscanlist = []
gsrclist = []
gtstlist = []
gtedlist = []
for i in good:
gscanlist.append(sclist['scanlist'][i])
gsrclist.append(sclist['srclist'][i])
gtstlist.append(sclist['tstlist'][i])
gtedlist.append(sclist['tedlist'][i])
return {
'scanlist': gscanlist,
'srclist': gsrclist,
'tstlist': gtstlist,
'tedlist': gtedlist,
'vis': vis,
'bandnames': bandnames,
'fghzs': fghzs,
'times': times,
'has': has,
'decs': decs
}
def graph(f, navg=None, path=None):
import matplotlib.pyplot as plt
from matplotlib.ticker import FormatStrFormatter
import struct, time, glob, sys, socket
import read_idb as ri
import dbutil as db
if navg is None:
navg = 60
if path is None:
path = ''
out = ri.read_idb(f, navg=navg)
if len(out['fghz']) == 0:
# This file is no good, so skip it
return
fig, ax = plt.subplots(4, 13, sharex=True, sharey=True)
trange = Time([ri.fname2mjd(f[0]),
ri.fname2mjd(f[-1]) + ten_minutes],
format='mjd')
times, wscram, avgwind = db.a14_wscram(trange)
nwind = len(wscram)
nbad = np.sum(wscram)
if nbad != 0:
warn = ' --> Windscram! (' + str(nbad) + ' of ' + str(nwind) + ')'
color = '#d62728' # Plot points with "warning" Red color
else:
warn = ''
color = '#1f77b4' # Plot points with "normal" Blue color
fig.set_size_inches(18, 6)
nf = len(out['fghz'])
fstr = str(out['fghz'][nf / 2] * 1000)[:5] + ' MHz '
for k in range(13):
for j in range(4):
ax[j, k].cla()
ax[j, k].plot(out['ha'],
np.angle(out['x'][ri.bl2ord[k, 13], j, nf / 2]),
'.',
color=color)
ax[j, k].set_ylim(-4, 4)
ax[j, k].xaxis.set_major_formatter(FormatStrFormatter('%.2f'))
if k in range(1, 13): ax[j, k].yaxis.set_visible(False)
if j in range(3): ax[j, k].xaxis.set_visible(False)
if j == 0: ax[0, k].title.set_text('antenna %d' % (k + 1))
fig.suptitle(out['source'] + ' ' +
Time(out['time'][0], format='jd').iso[:19] + ' UT ' + fstr +
warn)
ax[0, 0].set_ylabel('XX Phase')
ax[1, 0].set_ylabel('YY Phase')
ax[2, 0].set_ylabel('XY Phase')
ax[3, 0].set_ylabel('YX Phase')
fig.text(0.5, 0.04, 'Hour Angle', ha='center')
t = Time(out['time'][0],
format='jd').iso[:19].replace('-',
'').replace(':',
'').replace(' ', '')
s = out['source']
ofile = path + t[:14] + '_' + s + '.npz'
np.savez(open(ofile, 'wb'), out=out)
plt.savefig(path + 'pcT' + t + '_' + s + '.png', bbox_inches='tight')
plt.close(fig)
ph = np.angle(np.sum(out['x'], 3))
fig, ax = plt.subplots(4, 13)
fig.set_size_inches(18, 6)
for k in range(13):
for j in range(4):
ax[j, k].cla()
ax[j, k].plot(out['fghz'],
ph[ri.bl2ord[k, 13], j],
'.',
color=color)
ax[j, k].set_ylim(-4, 4)
ax[j, k].xaxis.set_major_formatter(FormatStrFormatter('%.2f'))
if k in range(1, 13): ax[j, k].yaxis.set_visible(False)
if j in range(3): ax[j, k].xaxis.set_visible(False)
if j == 0: ax[0, k].title.set_text('antenna %d' % (k + 1))
fig.suptitle(out['source'] + ' ' +
Time(out['time'][0], format='jd').iso[:19] + ' UT' + warn)
ax[0, 0].set_ylabel('XX Phase')
ax[1, 0].set_ylabel('YY Phase')
ax[2, 0].set_ylabel('XY Phase')
ax[3, 0].set_ylabel('YX Phase')
fig.text(0.5, 0.04, 'Frequency[GHz]', ha='center')
t = Time(out['time'][0],
format='jd').iso[:19].replace('-',
'').replace(':',
'').replace(' ', '')
plt.savefig(path + 'pcF' + t + '_' + s + '.png', bbox_inches='tight')
plt.close(fig)
def calpntanal(t,ant_str='ant1-13',do_plot=True,ax=None):
''' Does a complete analysis of CALPNTCAL, reading information from the SQL
database, finding the corresponding Miriad IDB data, and doing the
gaussian fit to the beam, to return the beam and offset parameters.
t Time object with a time near the start of the desired scan (finds the scan
that starts closest time to the given time)
ax If specified as a two-element array of axes, the plots will be placed
in an already existing window, allowing reuse of the same window.
Otherwise, a new figure is created (if do_plot is True).
Returns a dictionary containing
Source name, Time, HA, Dec, RA offset and Dec offset
'''
import matplotlib.pyplot as plt
from matplotlib.transforms import Bbox
import read_idb
import dbutil as db
bl2ord = read_idb.bl2ord
tdate = t.iso.replace('-','')[:8]
fdir = '/data1/eovsa/fits/IDB/'+tdate+'/'
fdb = dump_tsys.rd_fdb(t)
scanidx, = np.where(fdb['PROJECTID'] == 'CALPNTCAL')
# Set offset coordinates appropriate to the type of PROJECTID found
if scanidx.size == 0:
# No CALPNTCAL scans, so search for CALPNT2M
scanidx, = np.where(fdb['PROJECTID'] == 'CALPNT2M')
if scanidx.size == 0:
print 'No CALPNTCAL or CALPNT2M project IDs found for date'+t.iso[:10]
return {}
else:
# Found CALPNT2M, so set offset coordinates to match calpnt2m.trj
rao = np.array([-5.00, -2.0, -1.0, -0.5, 0.00, 0.5, 1.0, 2.0])
deco = np.array([-2.0, -1.0, -0.5, 0.00, 0.5, 1.0, 2.0, 5.00])
pltfac = 10.
else:
# Found CALPNTCAL, so set offset coordinates to match calpnt.trj
rao = np.array([-1.00, -0.20, -0.10, -0.05, 0.00, 0.05, 0.10, 0.20])
deco = np.array([-0.20, -0.10, -0.05, 0.00, 0.05, 0.10, 0.20, 1.00])
pltfac = 1.
scans,sidx = np.unique(fdb['SCANID'][scanidx],return_index=True)
tlist = Time(fdb['ST_TS'][scanidx[sidx]].astype(float).astype(int),format='lv')
idx, = nearest_val_idx([t.jd],tlist.jd)
filelist = [fdir+f for f in fdb['FILE'][np.where(fdb['SCANID'] == scans[idx])]]
# Read pointing data (timerange t must be accurate)
out = read_idb.read_idb(filelist, navg=30)
# Determine wanted baselines with ant 14 from ant_str
idx = read_idb.p.ant_str2list(ant_str)
idx1 = idx[idx>7] # Ants > 8
idx2 = idx[idx<8] # Ants <= 8
# Determine parallactic angle for azel antennas (they are all the same, so find median).
# If 0 < abs(chi) < 30, use channel XX
# If 30 < abs(chi) < 60, use sum of channel XX and XY
# If 60 < abs(chi) < 90, use channel XY
midtime = Time((out['time'][0] + out['time'][-1])/2.,format='jd')
times, chi = db.get_chi(Time([midtime.lv+1,midtime.lv + 10],format='lv'))
abschi = abs(lobe(np.median(chi[0,0:8])))
if pltfac == 1.:
# Case of 27m antenna pointing
# Do appropriate sums over frequency, polarization and baseline
if abschi >= 0 and abschi < np.pi/6:
pntdata = np.sum(np.abs(np.sum(out['x'][bl2ord[idx,13],0,:,:48],1)),0) # Use only XX
elif abschi >= np.pi/6 and abschi < np.pi/3:
pntdata1 = np.sum(np.abs(np.sum(out['x'][bl2ord[idx1,13],0,:,:48],1)),0) # Use XX only for ants > 8
pntdata2 = np.sum(np.abs(np.sum(out['x'][bl2ord[idx2,13],:,:,:48],2)),0) # Use sum of XX and XY for ants <= 8
pntdata = pntdata1 + np.sum(pntdata2[np.array([0,2])],0)
else:
pntdata1 = np.sum(np.abs(np.sum(out['x'][bl2ord[idx1,13],0,:,:48],1)),0) # Use XX only for ants > 8
pntdata2 = np.sum(np.abs(np.sum(out['x'][bl2ord[idx2,13],2,:,:48],1)),0) # Use sum of XY for ants <= 8
pntdata = pntdata1 + pntdata2
# Measurements are 90 s long, hence 3 consecutive 30 s points, so do final
# sum over these
pntdata.shape = (16,3)
stdev = np.std(pntdata,1)
pntdata = np.sum(pntdata,1)
radat = pntdata[:8]
decdat = pntdata[8:]
plsqr, xr, yr = solpnt.gausfit(rao, radat)
plsqd, xd, yd = solpnt.gausfit(deco, decdat)
midtime = Time((out['time'][0] + out['time'][-1])/2.,format='jd')
if (do_plot):
if ax is None:
f, ax = plt.subplots(1,2)
f.set_size_inches(2.5,1.5,forward=True)
ax[0].errorbar(rao,radat,yerr=stdev[:8],fmt='.')
ax[0].plot(xr,yr)
ax[0].axvline(x=0,color='k')
ax[0].axvline(x=plsqr[1],linestyle='--')
ax[1].errorbar(deco,decdat,yerr=stdev[8:],fmt='.')
ax[1].plot(xd,yd)
ax[1].axvline(x=0,color='k')
ax[1].axvline(x=plsqd[1],linestyle='--')
for j in range(2):
ax[j].set_xlim(-0.3, 0.3)
ax[j].grid()
ax[0].text(0.05,0.9,'RAO :'+str(plsqr[1])[:5],transform=ax[0].transAxes)
ax[0].text(0.55,0.9,'FWHM:'+str(plsqr[2])[:5],transform=ax[0].transAxes)
ax[0].set_xlabel('RA Offset [deg]')
ax[1].text(0.05,0.9,'DECO:'+str(plsqd[1])[:5],transform=ax[1].transAxes)
ax[1].text(0.55,0.9,'FWHM:'+str(plsqd[2])[:5],transform=ax[1].transAxes)
ax[1].set_xlabel('Dec Offset [deg]')
if ax is None:
f.suptitle('Pointing on '+out['source']+' at '+midtime.iso)
else:
ax[0].set_title(out['source']+' at')
ax[1].set_title(midtime.iso[:16])
plt.pause(0.5)
return {'source':out['source'],'ha':out['ha'][24]*180./np.pi,'dec':out['dec']*180/np.pi,
'rao':plsqr[1],'deco':plsqd[1],'time':midtime,'antidx':13}
else:
# Case of 2m antenna pointing
# Do appropriate sum over frequency and polarization but not baseline
if abschi >= 0 and abschi < np.pi/6:
pntdata = np.abs(np.sum(out['x'][bl2ord[idx,13],0,:,:48],1)) # Use only XX
elif abschi >= np.pi/6 and abschi < np.pi/3:
pntdata1 = np.abs(np.sum(out['x'][bl2ord[idx1,13],0,:,:48],1)) # Use XX only for ants > 8
pntdata2 = np.abs(np.sum(out['x'][bl2ord[idx2,13],:,:,:48],2)) # Use sum of XX and XY for ants <= 8
pntdata = np.concatenate((pntdata1,np.sum(pntdata2[:,np.array([0,2])],1)),0)
else:
pntdata1 = np.abs(np.sum(out['x'][bl2ord[idx1,13],0,:,:48],1)) # Use XX only for ants > 8
pntdata2 = np.abs(np.sum(out['x'][bl2ord[idx2,13],2,:,:48],1)) # Use sum of XY for ants <= 8
pntdata = np.concatenate((pntdata1,pntdata2),0)
# Measurements are 90 s long, hence 3 consecutive 30 s points, so do final
# sum over these
pntdata.shape = (idx.size,16,3)
stdev = np.std(pntdata,2)
pntdata = np.sum(pntdata,2)
rao_fit = np.zeros(idx.size,float)
deco_fit = np.zeros(idx.size,float)
if (do_plot):
if ax is None:
f, ax = plt.subplots(1,2*idx.size)
f.set_size_inches(2.5*idx.size,1.5,forward=True)
plt.subplots_adjust(left=0.03, right=0.97, top=0.89, bottom=0.3, wspace=0.1, hspace=0.1)
for k in range(idx.size):
radat = pntdata[k,:8]
decdat = pntdata[k,8:]
plsqr, xr, yr = solpnt.gausfit(rao, radat)
plsqd, xd, yd = solpnt.gausfit(deco, decdat)
midtime = Time((out['time'][0] + out['time'][-1])/2.,format='jd')
if (do_plot):
ax[k*2+0].errorbar(rao,radat,yerr=stdev[k,:8],fmt='.')
ax[k*2+0].plot(xr,yr)
ax[k*2+0].axvline(x=0,color='k')
ax[k*2+0].axvline(x=plsqr[1],linestyle='--')
ax[k*2+1].errorbar(deco,decdat,yerr=stdev[k,8:],fmt='.')
ax[k*2+1].plot(xd,yd)
ax[k*2+1].axvline(x=0,color='k')
ax[k*2+1].axvline(x=plsqd[1],linestyle='--')
for j in range(2):
ax[k*2+j].set_xlim(-3.0, 3.0)
ax[k*2+j].grid()
ax[k*2+0].text(0.05,0.7,'RAO :'+str(plsqr[1])[:5],transform=ax[k*2+0].transAxes,fontsize=9)
ax[k*2+0].text(0.05,0.5,'FWHM:'+str(plsqr[2])[:5],transform=ax[k*2+0].transAxes,fontsize=9)
ax[k*2+0].set_xlabel('RAO [deg]',fontsize=9)
ax[k*2+1].text(-0.2,0.85,'Ant :'+str(idx[k]+1),transform=ax[k*2+1].transAxes,fontsize=9)
ax[k*2+1].text(0.05,0.7,'DECO:'+str(plsqd[1])[:5],transform=ax[k*2+1].transAxes,fontsize=9)
ax[k*2+1].text(0.05,0.5,'FWHM:'+str(plsqd[2])[:5],transform=ax[k*2+1].transAxes,fontsize=9)
ax[k*2+1].set_yticklabels([])
ax[k*2+1].set_xlabel('DECO [deg]',fontsize=9)
if k == idx.size-1:
ax[k+0].set_title(out['source']+' at',fontsize=9)
ax[k+1].set_title(midtime.iso[:16],fontsize=9)
# Adjust the pairs of subplots to group RA/Dec for each antenna together
ax[k*2+0].set_position(Bbox(ax[k*2+0].get_position().get_points() + np.array([[0.0025,0],[0.0025,0]])))
ax[k*2+1].set_position(Bbox(ax[k*2+1].get_position().get_points() + np.array([[-0.0025,0],[-0.0025,0]])))
# Set to the same amplitude scale
ymin = min(ax[k*2+0].get_ylim()[0],ax[k*2+1].get_ylim()[0])
ymax = max(ax[k*2+0].get_ylim()[1],ax[k*2+1].get_ylim()[1])
ax[k*2+0].set_ylim(ymin,ymax)
ax[k*2+1].set_ylim(ymin,ymax)
plt.pause(0.5)
rao_fit[k] = plsqr[1]
deco_fit[k] = plsqd[1]
return {'source':out['source'],'ha':out['ha'][24]*180./np.pi,'dec':out['dec']*180/np.pi,
'rao':rao_fit,'deco':deco_fit,'time':midtime,'antidx':idx}
def rd_refcal(file, quackint=120., navg=3):
''' Reads a single UDB file representing a calibrator scan, and averages over the
bands in the file
'''
from read_idb import read_idb, bl2ord
from copy import deepcopy
import chan_util_bc as cu
import dbutil as db
out = read_idb([file], navg=navg, quackint=quackint)
bds = np.unique(out['band'])
nt = len(out['time'])
nbd = len(bds)
vis = np.zeros((15, 4, 34, nt), dtype=complex)
fghz = np.zeros(34)
# average over channels within each band
o = out['x'][bl2ord[13,:13]]
for bd in bds:
idx = np.where(out['band'] == bd)[0]
fghz[bd-1] = np.nanmean(out['fghz'][idx])
vis[:13,:,bd-1] = np.mean(o[:, :, idx], axis=2)
# Need to apply unrot to correct for feed rotation, before returning
xml, buf = ch.read_cal(11, Time(out['time'][0],format='jd'))
dph = extract(buf,xml['XYphase'])
xi_rot = extract(buf,xml['Xi_Rot'])
freq = extract(buf,xml['FGHz'])
freq = freq[np.where(freq != 0)]
band = []
for f in freq:
band.append(cu.freq2bdname(f))
bds, sidx = np.unique(band, return_index=True)
nbd = len(bds)
eidx = np.append(sidx[1:], len(band))
dxy = np.zeros((14, 34), dtype=np.float)
xi = np.zeros(34, dtype=np.float)
fghz = np.zeros(34)
# average dph and xi_rot frequencies within each band, to convert to 34-band representation
for b, bd in enumerate(bds):
fghz[bd - 1] = np.nanmean(freq[sidx[b]:eidx[b]])
xi[bd - 1] = np.nanmean(xi_rot[sidx[b]:eidx[b]])
for a in range(14):
dxy[a, bd - 1] = np.angle(np.sum(np.exp(1j * dph[a, sidx[b]:eidx[b]])))
# Read parallactic angles for this scan
trange = Time(out['time'][[0,-1]],format='jd')
times, chi = db.get_chi(trange)
tchi = times.jd
t = out['time']
if len(t) > 0:
vis2 = deepcopy(vis)
idx = nearest_val_idx(t, tchi)
pa = chi[idx] # Parallactic angle for the times of this refcal.
pa[:, [8, 9, 10, 12]] = 0.0
nt = len(idx) # Number of times in this refcal
# Apply X-Y delay phase correction
for a in range(13):
a1 = lobe(dxy[a] - dxy[13])
a2 = -dxy[13] - xi
a3 = dxy[a] - xi + np.pi
for j in range(nt):
vis2[a, 1, :, j] *= np.exp(1j * a1)
vis2[a, 2, :, j] *= np.exp(1j * a2)
vis2[a, 3, :, j] *= np.exp(1j * a3)
for j in range(nt):
for a in range(13):
vis[a, 0, :, j] = vis2[a, 0, :, j] * np.cos(pa[j, a]) + vis2[a, 3, :, j] * np.sin(pa[j, a])
vis[a, 2, :, j] = vis2[a, 2, :, j] * np.cos(pa[j, a]) + vis2[a, 1, :, j] * np.sin(pa[j, a])
vis[a, 3, :, j] = vis2[a, 3, :, j] * np.cos(pa[j, a]) - vis2[a, 0, :, j] * np.sin(pa[j, a])
vis[a, 1, :, j] = vis2[a, 1, :, j] * np.cos(pa[j, a]) - vis2[a, 2, :, j] * np.sin(pa[j, a])
# *******
return {'file': file, 'source': out['source'], 'vis': vis, 'bands': bds, 'fghz': fghz, 'times': out['time'], 'ha': out['ha'], 'dec': out['dec'], 'flag': np.zeros_like(vis, dtype=np.int)}
def xdata_display(t, ax=None):
""" Given the time as a Time object, search the FDB file for files
associated with the scan for that time and create a dynamic spectrogram
on the axis specified by ax, or on a new plot if no ax. If the requested
time is more than 10 minutes after the last file of that scan, returns
None to indicate no plot.
"""
import time
import dump_tsys
# import get_X_data2 as gd
import read_idb as ri
import spectrogram_fit as sp
fdb = dump_tsys.rd_fdb(t)
# Get files from next day, in case scan extends past current day
t1 = Time(t.mjd + 1, format="mjd")
fdb1 = dump_tsys.rd_fdb(t1)
# Concatenate the two days (if the second day exists)
if fdb1 != {}:
for key in fdb.keys():
fdb[key] = np.concatenate((fdb[key], fdb1[key]))
# Find unique scan IDs
scans, idx = np.unique(fdb["SCANID"], return_index=True)
# Limit to scans in 'NormalObserving' mode
good, = np.where(fdb["PROJECTID"][idx] == "NormalObserving")
if len(good) > 0:
scans = scans[good]
else:
print "No NormalObserving scans found."
return None, None, None
# Find scanID that starts earlier than, but closest to, the current time
for i, scan in enumerate(scans):
dt = t - Time(time.strftime("%Y-%m-%d %H:%M:%S", time.strptime(scan, "%y%m%d%H%M%S")))
if dt.sec > 0.0:
iout = i
scan = scans[iout]
# Find files for this scan
fidx, = np.where(fdb["SCANID"] == scan)
tlevel = None
bflag = None
if len(fidx) > 0:
files = fdb["FILE"][fidx]
# Find out how old last file of this scan is, and proceed only if less than 20 minutes
# earlier than the time given in t.
try:
dt = t - Time(time.strftime("%Y-%m-%d %H:%M:%S", time.strptime(files[-1], "IDB%Y%m%d%H%M%S")))
except:
dt = 10000.0 # Forces skip of plot creation
print "Unexpected FDB file format."
scan = None
if dt.sec < 1200.0:
# This is a currently active scan, so create the figure
for i in range(len(files)):
files[i] = "/data1/IDB/" + files[i]
# data, uvw, fghz, times = gd.get_X_data(files)
out = ri.read_idb(files)
out = ri.flag_sk(out)
fghz = out["fghz"]
times = Time(out["time"], format="jd")
data = out["x"]
if ax is not None:
datstr = times[0].iso[:10]
ax.set_xlabel("Time [UT on " + datstr + "]")
ax.set_ylabel("Frequency [GHz]")
ax.set_title("EOVSA Summed Cross-Correlation Amplitude for " + datstr)
sp.plot_spectrogram(
fghz, times, sum(sum(abs(data[0:11, :]), 1), 0), ax=ax, logsample=None, xdata=True, cbar=True
)
tlevel, bflag = flaremeter(data)
else:
print "Time", dt.sec, "is > 1200 s after last file of last NormalObserving scan. No plot created."
scan = None
else:
print "No files found for this scan ID", scan
scan = None
return scan, tlevel, bflag, times
def xdata_display(t,ax=None):
''' Given the time as a Time object, search the FDB file for files
associated with the scan for that time and create a dynamic spectrogram
on the axis specified by ax, or on a new plot if no ax. If the requested
time is more than 10 minutes after the last file of that scan, returns
None to indicate no plot.
Skip SK flagging [2017-Mar-20 DG]
'''
import time, os
import dump_tsys
#import get_X_data2 as gd
import read_idb as ri
import spectrogram_fit as sp
fdb = dump_tsys.rd_fdb(t)
# Get files from next day, in case scan extends past current day
t1 = Time(t.mjd + 1,format='mjd')
fdb1 = dump_tsys.rd_fdb(t1)
# Concatenate the two days (if the second day exists)
if fdb1 != {}:
for key in fdb.keys():
fdb[key] = np.concatenate((fdb[key],fdb1[key]))
# Find unique scan IDs
scans, idx = np.unique(fdb['SCANID'],return_index=True)
# Limit to scans in 'NormalObserving' mode
good, = np.where(fdb['PROJECTID'][idx] == 'NormalObserving')
if len(good) > 0:
scans = scans[good]
else:
print 'No NormalObserving scans found.'
return None, None, None
# Find scanID that starts earlier than, but closest to, the current time
for i,scan in enumerate(scans):
dt = t - Time(time.strftime('%Y-%m-%d %H:%M:%S',time.strptime(scan,'%y%m%d%H%M%S')))
if dt.sec > 0.:
iout = i
scan = scans[iout]
# Find files for this scan
fidx, = np.where(fdb['SCANID'] == scan)
tlevel = None
bflag = None
if len(fidx) > 0:
files = fdb['FILE'][fidx]
# Find out how old last file of this scan is, and proceed only if less than 20 minutes
# earlier than the time given in t.
try:
dt = t - Time(time.strftime('%Y-%m-%d %H:%M:%S',time.strptime(files[-1],'IDB%Y%m%d%H%M%S')))
except:
dt = 10000. # Forces skip of plot creation
print 'Unexpected FDB file format.'
scan = None
if dt.sec < 1200.:
# This is a currently active scan, so create the figure
path = '/data1/IDB/'
if not os.path.isdir(path+files[0]):
# Look in /dppdata1
datstr = t.iso[:10].replace('-','')
path = '/data1/eovsa/fits/IDB/'+datstr+'/'
if not os.path.isdir(path+files[0]):
print 'No files found for this scan ID',scan
scan = None
return scan, tlevel, bflag, times
filelist = files
files = []
for i,file in enumerate(filelist):
files.append(path+file)
# data, uvw, fghz, times = gd.get_X_data(files)
out = ri.read_idb(files)
#out = ri.flag_sk(out) # Skip flagging for sk
fghz = out['fghz']
times = Time(out['time'],format='jd')
data = out['x']
if ax is not None:
datstr = times[0].iso[:10]
ax.set_xlabel('Time [UT on '+datstr+']')
ax.set_ylabel('Frequency [GHz]')
ax.set_title('EOVSA Summed Cross-Correlation Amplitude for '+datstr)
pdata = np.sum(np.sum(np.abs(data[0:11,:]),1),0) # Spectrogram to plot
X = np.sort(pdata.flatten()) # Sorted, flattened array
dmax = X[int(len(X)*0.95)] # Clip at 5% of points
sp.plot_spectrogram(fghz, times, pdata,
ax=ax, logsample=None, xdata=True, cbar=True, dmax=dmax)
#tlevel, bflag = flaremeter(data)
else:
print 'Time',dt.sec,'is > 1200 s after last file of last NormalObserving scan. No plot created.'
scan = None
else:
print 'No files found for this scan ID',scan
scan = None
return scan, tlevel, bflag, times
def get_attncal(trange, do_plot=False, dataonly=False):
''' Finds GAINCALTEST scans from FDB files corresponding to the days
present in trange Time() object (can be multiple days), calculates
the attenuation differences for the various FEMATTN states 1-8
relative to FEMATTN state 0, and optionally plots the results for
states 1 and 2 (the most commonly used). To analyze only a single
day, trange Time() object can have the same time repeated, or can
be a single time.
Returns a list of dictionaries, each pertaining to one of the days
in trange, with keys defined as follows:
'time': The start time of the GAINCALTEST scan, as a Time() object
'fghz': The list of frequencies [GHz] at which attenuations are measured
'attn': The array of attenuations [dB] of size (nattn, nant, npol, nf),
where nattn = 8, nant = 13, npol = 2, and nf is variable
'rcvr': The array of receiver noise level (raw units) of size
(nant, npol, nf), where nant = 13, npol = 2, and nf is variable
'rcvr_auto': Same as rcvr, but for auto-correlation (hence it is complex)
N.B.: Ignores days with other than one GAINCALTEST measurement, e.g. 0 or 2,
the first is obvious, while the second is because there is no way to
tell which of the 2 are good.
The dataonly parameter tells the routine to skip calculating the attenuation
and only return the IDB data from the (first) gaincal.
'''
from util import get_idbdir, fname2mjd, nearest_val_idx
import socket
import dbutil
if type(trange.mjd) == np.float:
# Interpret single time as both start and end time
mjd1 = int(trange.mjd)
mjd2 = mjd1
else:
mjd1, mjd2 = trange.mjd.astype(int)
if do_plot:
import matplotlib.pylab as plt
f, ax = plt.subplots(4, 13)
f.set_size_inches((14, 5))
ax[0, 0].set_ylabel('Atn1X [dB]')
ax[1, 0].set_ylabel('Atn1Y [dB]')
ax[2, 0].set_ylabel('Atn2X [dB]')
ax[3, 0].set_ylabel('Atn2Y [dB]')
for i in range(13):
ax[0, i].set_title('Ant ' + str(i + 1))
ax[3, i].set_xlabel('Freq [GHz]')
for j in range(2):
ax[j, i].set_ylim(1, 3)
ax[j + 2, i].set_ylim(3, 5)
outdict = []
for mjd in range(mjd1, mjd2 + 1):
fdb = dt.rd_fdb(Time(mjd, format='mjd'))
gcidx, = np.where(fdb['PROJECTID'] == 'GAINCALTEST')
if len(gcidx) == 1:
print fdb['FILE'][gcidx]
gcidx = gcidx[0]
else:
for i, fname in enumerate(fdb['FILE'][gcidx]):
print str(i) + ': GAINCALTEST File', fname
idex = input('There is more than one GAINCALTEST. Select: ' +
str(np.arange(len(gcidx))) + ':')
gcidx = gcidx[idex]
datadir = get_idbdir(Time(mjd, format='mjd'))
# Add date path if on pipeline
# if datadir.find('eovsa') != -1: datadir += fdb['FILE'][gcidx][3:11]+'/'
host = socket.gethostname()
if host == 'pipeline': datadir += fdb['FILE'][gcidx][3:11] + '/'
file = datadir + fdb['FILE'][gcidx]
out = ri.read_idb([file])
if dataonly:
return out
# Get time from filename and read 120 records of attn state from SQL database
filemjd = fname2mjd(fdb['FILE'][gcidx])
cursor = dbutil.get_cursor()
d15 = dbutil.get_dbrecs(cursor,
dimension=15,
timestamp=Time(filemjd, format='mjd'),
nrecs=120)
cursor.close()
# Find time indexes of the 62 dB attn state
# Uses only ant 1 assuming all are the same
dtot = (d15['Ante_Fron_FEM_HPol_Atte_Second'] +
d15['Ante_Fron_FEM_HPol_Atte_First'])[:, 0]
# Use system clock day number to identify bad SQL entries and eliminate them
good, = np.where(d15['Ante_Cont_SystemClockMJDay'][:, 0] != 0)
#import pdb; pdb.set_trace()
# Indexes into SQL records where a transition occurred.
transitions, = np.where(dtot[good] - np.roll(dtot[good], 1) != 0)
# Eliminate any zero-index transition (if it exists)
if transitions[0] == 0:
transitions = transitions[1:]
# These now have to be translated into indexes into the data, using the times
idx = nearest_val_idx(d15['Timestamp'][good, 0][transitions],
Time(out['time'], format='jd').lv)
#import pdb; pdb.set_trace()
vx = np.nanmedian(
out['p'][:13, :, :, np.arange(idx[0] + 1, idx[1] - 1)], 3)
va = np.mean(out['a'][:13, :2, :,
np.arange(idx[0] + 1, idx[1] - 1)], 3)
vals = []
attn = []
for i in range(1, 10):
vals.append(
np.nanmedian(
out['p'][:13, :, :,
np.arange(idx[i] + 1, idx[i + 1] - 1)], 3) - vx)
attn.append(np.log10(vals[0] / vals[-1]) * 10.)
#vals = []
#attna = []
#for i in range(1,10):
# vals.append(np.median(out['a'][:13,:2,:,np.arange(idx[i],idx[i+1])],3) - va)
# attna.append(np.log10(vals[0]/vals[-1])*10.)
if do_plot:
for i in range(13):
for j in range(2):
ax[j, i].plot(out['fghz'],
attn[1][i, j],
'.',
markersize=3)
#ax[j,i].plot(out['fghz'],attna[1][i,j],'.',markersize=1)
ax[j + 2, i].plot(out['fghz'],
attn[2][i, j],
'.',
markersize=3)
#ax[j+2,i].plot(out['fghz'],attna[2][i,j],'.',markersize=1)
outdict.append({
'time': Time(out['time'][0], format='jd'),
'fghz': out['fghz'],
'rcvr_auto': va, # 'attna': np.array(attna[1:]),
'rcvr': vx,
'attn': np.array(attn[1:])
})
return outdict
def graph(f,navg=None,path=None):
import matplotlib.pyplot as plt
from matplotlib.ticker import FormatStrFormatter
import struct, time, glob, sys, socket
import read_idb as ri
import dbutil as db
if navg is None:
navg = 60
if path is None:
path = ''
out = ri.read_idb(f,navg=navg)
fig, ax = plt.subplots(4,13,sharex=True, sharey=True)
trange = Time([ri.fname2mjd(f[0]),ri.fname2mjd(f[-1]) + ten_minutes],format='mjd')
times, wscram, avgwind = db.a14_wscram(trange)
nwind = len(wscram)
nbad = np.sum(wscram)
if nbad != 0:
warn = ' --> Windscram! ('+str(nbad)+' of '+str(nwind)+')'
color = '#d62728' # Plot points with "warning" Red color
else:
warn = ''
color = '#1f77b4' # Plot points with "normal" Blue color
fig.set_size_inches(18,6)
nf = len(out['fghz'])
fstr = str(out['fghz'][nf/2]*1000)[:5]+' MHz '
for k in range(13):
for j in range(4):
ax[j,k].cla()
ax[j,k].plot(out['ha'],np.angle(out['x'][ri.bl2ord[k,13],j,nf/2]),'.',color=color)
ax[j,k].set_ylim(-4, 4)
ax[j,k].xaxis.set_major_formatter(FormatStrFormatter('%.2f'))
if k in range(1,13): ax[j,k].yaxis.set_visible(False)
if j in range(3): ax[j,k].xaxis.set_visible(False)
if j == 0: ax[0,k].title.set_text('antenna %d' %(k+1))
fig.suptitle(out['source']+' '+Time(out['time'][0],format='jd').iso[:19]+' UT '+fstr+warn)
ax[0,0].set_ylabel('XX Phase')
ax[1,0].set_ylabel('YY Phase')
ax[2,0].set_ylabel('XY Phase')
ax[3,0].set_ylabel('YX Phase')
fig.text(0.5, 0.04, 'Hour Angle', ha = 'center')
t = Time(out['time'][0],format='jd').iso[:19].replace('-','').replace(':','').replace(' ','')
s = out['source']
ofile = path + t[:14] +'_'+ s +'.npz'
np.savez(open(ofile,'wb'),out = out)
plt.savefig(path + 'pcT'+t+'_'+ s +'.png',bbox_inches='tight')
plt.close(fig)
ph = np.angle(np.sum(out['x'],3))
fig, ax = plt.subplots(4,13)
fig.set_size_inches(18,6)
for k in range(13):
for j in range(4):
ax[j,k].cla()
ax[j,k].plot(out['fghz'],ph[ri.bl2ord[k,13],j],'.',color=color)
ax[j,k].set_ylim(-4, 4)
ax[j,k].xaxis.set_major_formatter(FormatStrFormatter('%.2f'))
if k in range(1,13): ax[j,k].yaxis.set_visible(False)
if j in range(3): ax[j,k].xaxis.set_visible(False)
if j == 0: ax[0,k].title.set_text('antenna %d' %(k+1))
fig.suptitle(out['source']+' '+Time(out['time'][0],format='jd').iso[:19]+' UT'+warn)
ax[0,0].set_ylabel('XX Phase')
ax[1,0].set_ylabel('YY Phase')
ax[2,0].set_ylabel('XY Phase')
ax[3,0].set_ylabel('YX Phase')
fig.text(0.5, 0.04, 'Frequency[GHz]', ha = 'center')
t = Time(out['time'][0],format='jd').iso[:19].replace('-','').replace(':','').replace(' ','')
plt.savefig(path + 'pcF'+t+'_'+ s +'.png',bbox_inches='tight')
plt.close(fig)
def get_attncal(trange, do_plot=False):
''' Finds GAINCALTEST scans from FDB files corresponding to the days
present in trange Time() object (can be multiple days), calculates
the attenuation differences for the various FEMATTN states 1-8
relative to FEMATTN state 0, and optionally plots the results for
states 1 and 2 (the most commonly used). To analyze only a single
day, trange Time() object can have the same time repeated, or can
be a single time.
Returns a list of dictionaries, each pertaining to one of the days
in trange, with keys defined as follows:
'time': The start time of the GAINCALTEST scan, as a Time() object
'fghz': The list of frequencies [GHz] at which attenuations are measured
'attn': The array of attenuations [dB] of size (nattn, nant, npol, nf),
where nattn = 8, nant = 13, npol = 2, and nf is variable
'rcvr': The array of receiver noise level (raw units) of size
(nant, npol, nf), where nant = 13, npol = 2, and nf is variable
N.B.: Ignores days with other than one GAINCALTEST measurement, e.g. 0 or 2,
the first is obvious, while the second is because there is no way to
tell which of the 2 are good.
'''
if type(trange.mjd) == np.float:
# Interpret single time as both start and end time
mjd1 = int(trange.mjd)
mjd2 = mjd1
else:
mjd1, mjd2 = trange.mjd.astype(int)
if do_plot:
import matplotlib.pylab as plt
f, ax = plt.subplots(4, 13)
f.set_size_inches((14, 5))
ax[0, 0].set_ylabel('Atn1X [dB]')
ax[1, 0].set_ylabel('Atn1Y [dB]')
ax[2, 0].set_ylabel('Atn2X [dB]')
ax[3, 0].set_ylabel('Atn2Y [dB]')
for i in range(13):
ax[0, i].set_title('Ant ' + str(i + 1))
ax[3, i].set_xlabel('Freq [GHz]')
for j in range(2):
ax[j, i].set_ylim(1, 3)
ax[j + 2, i].set_ylim(3, 5)
outdict = []
for mjd in range(mjd1, mjd2 + 1):
fdb = dt.rd_fdb(Time(mjd, format='mjd'))
gcidx, = np.where(fdb['PROJECTID'] == 'GAINCALTEST')
if len(gcidx) == 1:
print fdb['FILE'][gcidx]
file = '/data1/eovsa/fits/IDB/' + fdb['FILE'][gcidx][0][
3:11] + '/' + fdb['FILE'][gcidx][0]
out = ri.read_idb([file])
vx = np.mean(out['p'][:13, :, :, 6:12], 3)
val0 = np.median(out['p'][:13, :, :, 16:22], 3) - vx
val1 = np.median(out['p'][:13, :, :, 26:32], 3) - vx
val2 = np.median(out['p'][:13, :, :, 36:42], 3) - vx
val3 = np.median(out['p'][:13, :, :, 46:52], 3) - vx
val4 = np.median(out['p'][:13, :, :, 56:62], 3) - vx
val5 = np.median(out['p'][:13, :, :, 66:72], 3) - vx
val6 = np.median(out['p'][:13, :, :, 76:82], 3) - vx
val7 = np.median(out['p'][:13, :, :, 86:92], 3) - vx
val8 = np.median(out['p'][:13, :, :, 96:102], 3) - vx
attn1 = np.log10(val0 / val1) * 10.
attn2 = np.log10(val0 / val2) * 10.
attn3 = np.log10(val0 / val3) * 10.
attn4 = np.log10(val0 / val4) * 10.
attn5 = np.log10(val0 / val5) * 10.
attn6 = np.log10(val0 / val6) * 10.
attn7 = np.log10(val0 / val7) * 10.
attn8 = np.log10(val0 / val8) * 10.
if do_plot:
for i in range(13):
for j in range(2):
ax[j, i].plot(out['fghz'], attn1[i, j], '.')
ax[j + 2, i].plot(out['fghz'], attn2[i, j], '.')
outdict.append({
'time':
Time(out['time'][0], format='jd'),
'fghz':
out['fghz'],
'rcvr':
vx,
'attn':
np.array(
[attn1, attn2, attn3, attn4, attn5, attn6, attn7, attn8])
})
return outdict
from matplotlib import pylab as plt
import glob
import read_idb as ri
from util import lobe
files = glob.glob('/data1/eovsa/fits/UDB/2018/UDB20180826*')
files.sort()
files = files[:20]
out1o = ri.read_idb(files, navg=20)
out2o = ri.read_idb(files[1:], navg=20)
ph1 = np.angle(out1o['x'][ri.bl2ord[13, :13]])
ph2 = np.angle(out2o['x'][ri.bl2ord[13, :13]])
ph_1 = np.zeros((13, 2, 12, 13), np.float)
ph_2 = np.zeros((13, 2, 12, 10), np.float)
for i in range(13):
for j in range(2):
for k in range(12):
if i == 0:
ph_1[i, 0, k] = lobe(ph1[i, j, k] - ph1[i, j, k, 0]
) * out1o['fghz'][0] / out1o['fghz'][k]
ph_2[i, 1, k] = lobe(ph2[i, j, k] - ph2[i, j, k, 0]
) * out2o['fghz'][0] / out2o['fghz'][k]
else:
ph_1[i, 0,
k] = -lobe(ph1[i, j, k] - ph1[0, j, k] - ph1[i, j, k, 0] +
ph1[0, j, k,
0]) * out1o['fghz'][0] / out1o['fghz'][k]
ph_2[i, 1,
k] = -lobe(ph2[i, j, k] - ph2[0, j, k] - ph2[i, j, k, 0] +
ph2[0, j, k,
0]) * out2o['fghz'][0] / out2o['fghz'][k]
# Mean over frequencies and polarizations, scaled by cos(dec)
