def main(parmdbfile, targetms, phaseonly = True):
if not os.path.exists(parmdbfile):
print "Parmdb file %s doesn't exist!" % parmdbfile
return(1)
if not os.path.exists(targetms):
print "Target ms %s doesn't exist!" % targetms
return(1)
msinfo = ReadMs(targetms)
# Open up the parmdb, get an example value
parmdb = pdb.parmdb(parmdbfile)
parnames = parmdb.getNames()
examplevalue = None
for name in parnames:
if "CS" in name:
examplevalue = parmdb.getValuesGrid(name)[name]
break
#print examplevalue.keys()
# Zero the phases of the example entry
if examplevalue == None:
print "Couldn't find an example entry"
return(1)
examplevalue['values'] = np.zeros(examplevalue['values'].shape)
if not(phaseonly):
examplevalue_ones = copy.deepcopy(examplevalue)
examplevalue_ones['values'] = np.ones(examplevalue_ones['values'].shape)
# Add the necessary stations
for antenna_id, antenna in enumerate(msinfo.stations):
if not "CS" in antenna and not "RS" in antenna:
ValueHolder = parmdb.makeValue(values=examplevalue['values'],
sfreq=examplevalue['freqs'],
efreq=examplevalue['freqwidths'],
stime=examplevalue['times'],
etime=examplevalue['timewidths'],
asStartEnd=False)
if phaseonly:
parmdb.addValues("Gain:0:0:Phase:" + antenna,ValueHolder)
parmdb.addValues("Gain:1:1:Phase:" + antenna,ValueHolder)
else:
ValueHolder_ones = parmdb.makeValue(values=examplevalue_ones['values'],
sfreq=examplevalue_ones['freqs'],
efreq=examplevalue_ones['freqwidths'],
stime=examplevalue_ones['times'],
etime=examplevalue_ones['timewidths'],
asStartEnd=False)
parmdb.addValues("Gain:0:0:Real:" + antenna,ValueHolder_ones)
parmdb.addValues("Gain:0:0:Imag:" + antenna,ValueHolder)
parmdb.addValues("Gain:1:1:Real:" + antenna,ValueHolder_ones)
parmdb.addValues("Gain:1:1:Imag:" + antenna,ValueHolder)
parmdb.flush()
parmdb = 0
return(0)
def createRMParmdb(MS,
parmdbname,
create=True,
patchname='',
server="ftp://cddis.gsfc.nasa.gov/gnss/products/ionex/",
prefix='codg',
ionexPath="IONEXdata/",
earth_rot=0,
timestep=900.,
stat_names=None):
myParmdb = parmdb.parmdb(parmdbname, create=create)
if create:
myParmdb.addDefValues("Gain:0:0:Ampl", 1.e-4)
myParmdb.addDefValues("DirectionalGain:0:0:Ampl", 1.)
myParmdb.addDefValues("DirectionalGain:1:1:Ampl", 1.)
myParmdb.addDefValues("Gain:0:0:Real", 1.e-4)
myParmdb.addDefValues("Gain:1:1:Real", 1.e-4)
myParmdb.addDefValues("DirectionalGain:0:0:Real", 1.)
myParmdb.addDefValues("DirectionalGain:1:1:Real", 1.)
myMS = tab.table(MS)
stations = tab.table(myMS.getkeyword('ANTENNA')).getcol('NAME')
stat_pos = tab.table(myMS.getkeyword('ANTENNA')).getcol('POSITION')
if not stat_names == None:
if not (stat_names == 'all'):
stat_pos = [stat_pos[stations.index(name)] for name in stat_names]
else:
stat_names = stations[:]
else:
stat_names = stations[2:3]
stat_pos = stat_pos[2:3]
result = gt.getRM(MS=MS,
server=server,
ionexPath=ionexPath,
prefix=prefix,
earth_rot=earth_rot,
timestep=timestep,
stat_names=stat_names,
stat_positions=stat_pos)
RM = result['RM']
for st in stations:
#print "storing station",st,(st in RM.keys())
if not (st in RM.keys()):
stname = RM.keys()[0]
else:
stname = st
RM[stname] = RM[stname].reshape(RM[stname].shape[:1] + (1, ))
myValue = myParmdb.makeValue(
values=RM[stname],
sfreq=1e10,
efreq=2e10,
stime=result['times'],
etime=np.ones(result['times'].shape, dtype=float) *
result['timestep'],
asStartEnd=False)
if patchname:
valuename = "RotationMeasure:%s:%s" % (st, patchname)
else:
valuename = "RotationMeasure:%s" % (st)
myParmdb.deleteValues(valuename)
myParmdb.addValues(valuename, myValue)
def solplot_phaseonly(parmdb,
imageroot,
refstationi,
plot_international=False):
parmdbmtable = lp.parmdb(parmdb)
soldict = parmdbmtable.getValuesGrid('*')
names = parmdbmtable.getNames()
'Gain:1:1:Phase:RS508HBA'
stationsnames = np.array([name.split(':')[-1] for name in names])
stationsnames = np.unique(stationsnames)
if not plot_international:
stationsnames = np.array(
[name for name in stationsnames if name[0] in ['C', 'R']])
Nstat = len(stationsnames)
refstation = stationsnames[refstationi]
phase11_ref = soldict['Gain:1:1:Phase:{s}'.format(s=refstation)]['values']
phase00_ref = soldict['Gain:0:0:Phase:{s}'.format(s=refstation)]['values']
times = soldict['Gain:1:1:Phase:{s}'.format(s=refstation)]['times']
#Nr = int(np.ceil(np.sqrt(Nstat)))
#Nc = int(np.ceil(np.float(Nstat)/Nr))
Nr = int(Nstat)
Nc = 1
f, ax = plt.subplots(Nr, Nc, sharex=True, sharey=True, figsize=(12, 72))
axs = ax.reshape((Nr * Nc, 1))
for istat, station in enumerate(stationsnames):
phase11 = soldict['Gain:1:1:Phase:{s}'.format(s=station)]['values']
phase00 = soldict['Gain:0:0:Phase:{s}'.format(s=station)]['values']
# don't plot flagged phases
phase00 = np.ma.masked_where(phase00 == 0, phase00)
phase11 = np.ma.masked_where(phase11 == 0, phase11)
#try:
#if len(phase11) > 1000:
#fmt = ','
#else:
#fmt = '.'
#except TypeError:
#print "no phases"
fmt = ','
axs[istat][0].plot(times,
normalize(phase00 - phase00_ref),
color='b',
marker=fmt,
label='Gain:0:0:Phase')
axs[istat][0].plot(times,
normalize(phase11 - phase11_ref),
color='g',
marker=fmt,
label='Gain:1:1:Phase')
axs[istat][0].set_ylim(-3.2, 3.2)
axs[istat][0].set_xlim(times.min(), times.max())
axs[istat][0].set_title(station)
f.savefig(imageroot + "_phase.png", dpi=100)
return
def solplot_tec(parmdb, imageroot, refstationi, plot_international=False, freq=None):
parmdbmtable = lp.parmdb(parmdb)
soldict = parmdbmtable.getValuesGrid('*')
if freq is None:
tfrange = parmdbmtable.getRange()
print 'freqrange', tfrange[0:2]
freq = np.average(tfrange[0:2])
print 'freq', freq/1e6, 'MHz'
names = parmdbmtable.getNames()
'Gain:1:1:Phase:RS508HBA'
stationsnames = np.array([name.split(':')[-1] for name in names])
stationsnames = np.unique(stationsnames)
if not plot_international:
stationsnames = np.array([name for name in stationsnames if name[0] in ['C','R'] ])
Nstat = len(stationsnames)
refstation = stationsnames[refstationi]
times = soldict['CommonScalarPhase:{s}'.format(s=refstation)]['times']
times = scaletimes(times)
phase_ref = soldict['CommonScalarPhase:{s}'.format(s=refstation)]['values']
tec_ref = soldict['TEC:{s}'.format(s=refstation)]['values']
num_channels = phase_ref.shape[1]
Nr = int(Nstat)
Nc = 1
for chan_indx in range(num_channels):
f, ax = plt.subplots(Nr, Nc, sharex=True, sharey=True, figsize=(12,72))
axs = ax.reshape((Nr*Nc,1))
ymin = 2
ymax = 0
for istat, station in enumerate(stationsnames):
phase = soldict['CommonScalarPhase:{s}'.format(s=station)]['values'][:, chan_indx]
tec = soldict['TEC:{s}'.format(s=station)]['values'][:, chan_indx]
phase_ref_chan = phase_ref[:, chan_indx]
tec_ref_chan = tec_ref[:, chan_indx]
phase = np.ma.masked_where(phase==0, phase)
if len(times) > 1000:
fmt = ','
else:
fmt = '.'
ls='none'
phasep = phase - phase_ref_chan
tecp = -8.44797245e9*(tec - tec_ref_chan)/freq
axs[istat][0].plot(times, np.mod(phasep+tecp +np.pi, 2*np.pi) - np.pi, color='b', marker=fmt, ls=ls, label='Phase+TEC', mec='b')
axs[istat][0].set_ylim(-np.pi, np.pi)
axs[istat][0].set_xlim(times.min(), times.max())
axs[istat][0].set_title(station)
f.savefig(imageroot+"_tec_channel{}.png".format(chan_indx),dpi=100)
plt.close(f)
parmdbmtable = False
del(soldict)
def addSourceTable (image, sourcedbName, minTime, maxTime):
# Create the table using TaQL.
tab = pt.taql ("create table '" + image.name() + "/LOFAR_SOURCE' " +
"SOURCE_ID int, \TIME double, INTERVAL double, " +
"NUM_LINES int, NAME string, " +
"DIRECTION double shape=[2], " +
"PROPER_MOTION double shape=[2], " +
"FLUX double shape=[4], " +
"SPINX double, REF_FREQUENCY double, " +
"SHAPE double shape=[3]")
tab.putcolkeyword ("TIME", "QuantumUnits", ["s"])
tab.putcolkeyword ("INTERVAL", "QuantumUnits", ["s"])
tab.putcolkeyword ("DIRECTION", "QuantumUnits", ["rad"])
tab.putcolkeyword ("PROPER_MOTION", "QuantumUnits", ["rad/s"])
tab.putcolkeyword ("FLUX", "QuantumUnits", ["Jy"])
tab.putcolkeyword ("REF_FREQUENCY", "QuantumUnits", ["MHz"])
tab.putcolkeyword ("SHAPE", "QuantumUnits", ["rad", "rad", "rad"])
tab.putcolkeyword ("TIME", "MEASINFO", {"Ref":"UTC", "type":"epoch"})
tab.putcolkeyword ("DIRECTION", "MEASINFO", {"Ref":"J2000", "type":"direction"})
tab.flush()
image.putkeyword ("ATTRGROUPS." + "LOFAR_SOURCE", tab)
# Get all parameters from the source parmdb.
midtime = (minTime + maxTime) / 2
inttime = maxTime - minTime
sourcedb = pdb.parmdb(sourcedbName)
# Get all source names by getting the Ra parms from DEFAULTVALUES
names = [name[3:] for name in sourcedb.getDefNames ("Ra:*")]
values = sourcedb.getDefValues()
sourcedb = 0 # close
row = 0
tab.addrows (len(names))
# Add the info of all sources.
# The field names below are as used in SourceDB.
fldnames = ["Ra", "Dec", "I", "Q", "U", "V", "SpectralIndex:0",
"ReferenceFrequency", "Orientation", "MajorAxis", "MinorAxis"]
vals = [0. for fld in fldnames]
for name in names:
for i in range(len(fldnames)):
key = fldnames[i] + ":" + name
if values.has_key (key):
vals[i] = values[key][0][0]
else:
vals[i] = 0.
tab.putcell ("SOURCE_ID", row, row)
tab.putcell ("TIME", row, midtime);
tab.putcell ("INTERVAL", row, inttime);
tab.putcell ("NUM_LINES", row, 0);
tab.putcell ("NAME", row, name);
tab.putcell ("DIRECTION", row, vals[:2]);
tab.putcell ("PROPER_MOTION", row, (0.,0.));
tab.putcell ("FLUX", row, vals[2:6]);
tab.putcell ("SPINX", row, vals[6]);
tab.putcell ("REF_FREQUENCY", row, vals[7]);
tab.putcell ("SHAPE", row, vals[8:11]);
row += 1
# Ready.
tab.close()
print "Added subtable LOFAR_SOURCE containing", row, "rows"
def solplot_clock(parmdb, imageroot, refstationi, plot_international=False):
parmdbmtable = lp.parmdb(parmdb)
soldict = parmdbmtable.getValuesGrid('*')
names = parmdbmtable.getNames()
'Gain:1:1:Phase:RS508HBA'
stationsnames = np.array([name.split(':')[-1] for name in names])
stationsnames = np.unique(stationsnames)
if not plot_international:
stationsnames = np.array(
[name for name in stationsnames if name[0] in ['C', 'R']])
Nstat = len(stationsnames)
refstation = stationsnames[refstationi]
#phase_ref = soldict['Clock:{s}'.format(s=refstation)]['values']
times = soldict['Clock:1:{s}'.format(s=refstation)]['times']
Nr = int(np.ceil(np.sqrt(Nstat)))
Nc = int(np.ceil(np.float(Nstat) / Nr))
f, ax = plt.subplots(Nr, Nc, sharex=True, sharey=True, figsize=(16, 12))
axs = ax.reshape((Nr * Nc, 1))
ymin = 2
ymax = 0
for istat, station in enumerate(stationsnames):
clock00 = soldict['Clock:0:{s}'.format(s=station)]['values']
clock11 = soldict['Clock:1:{s}'.format(s=station)]['values']
if len(clock00) > 0:
ymax = max(np.max(clock00), ymax)
if len(clock11) > 0:
ymax = max(np.max(clock11), ymax)
if len(clock00) > 0:
ymin = min(np.min(clock00), ymin)
if len(clock11) > 0:
ymin = min(np.min(clock11), ymin)
fmt = '.'
ls = 'none'
axs[istat][0].plot(times,
clock00,
color='b',
marker=fmt,
ls=ls,
label='Clock 0:0',
mec='b')
axs[istat][0].plot(times,
clock11,
color='g',
marker=fmt,
ls=ls,
label='Clock 1:1',
mec='g')
axs[istat][0].set_ylim(ymin, ymax)
axs[istat][0].set_xlim(times.min(), times.max())
axs[istat][0].set_title(station)
f.savefig(imageroot + "_clock.png", dpi=100)
return
def main(parmdb_in, parmdb_out ,debug=False):
"""
Merges facet selfcal parmdbs into a parmdb for a single band
Parameters
----------
parmdb_in : str
Name of (path to) the parmdb with Gain in polar coordinates.
parmdb_out : str
Name of the new parmdb that will be written
"""
pdb_in = pdb.parmdb(parmdb_in,create=False)
pdb_out = make_empty_parmdb(parmdb_out)
inparms = pdb_in.getValuesGrid('*')
names_in = inparms.keys()
for parmname in names_in:
nameparts = parmname.split(':')
if (len(nameparts) == 5
and (nameparts[0] == 'Gain' or nameparts[0] == 'DirectionalGain')):
if nameparts[3] == 'Phase' :
amplname = nameparts[0]+':'+nameparts[1]+':'+nameparts[2]+':Ampl:'+nameparts[4]
if amplname in names_in:
amps = inparms[amplname]['values']
else:
amps = np.ones_like(inparms[parmname]['values'])
realname = nameparts[0]+':'+nameparts[1]+':'+nameparts[2]+':Real:'+nameparts[4]
imagname = nameparts[0]+':'+nameparts[1]+':'+nameparts[2]+':Imag:'+nameparts[4]
realvals = amps * np.cos(inparms[parmname]['values'])
imagvals = amps * np.sin(inparms[parmname]['values'])
if debug: print 'Adding parameters:',realname,imagname
ValueHolder = pdb_out.makeValue(values=realvals,
sfreq=inparms[parmname]['freqs'],
efreq=inparms[parmname]['freqwidths'],
stime=inparms[parmname]['times'],
etime=inparms[parmname]['timewidths'],
asStartEnd=False)
pdb_out.addValues(realname,ValueHolder)
ValueHolder = pdb_out.makeValue(values=imagvals,
sfreq=inparms[parmname]['freqs'],
efreq=inparms[parmname]['freqwidths'],
stime=inparms[parmname]['times'],
etime=inparms[parmname]['timewidths'],
asStartEnd=False)
pdb_out.addValues(imagname,ValueHolder)
elif nameparts[3] == 'Real' or nameparts[3] == 'Imag' :
print 'Gain value:',parmname,'not in polar coordinates!'
else:
ValueHolder = pdb_out.makeValue(values=inparms[parmname]['values'],
sfreq=inparms[parmname]['freqs'],
efreq=inparms[parmname]['freqwidths'],
stime=inparms[parmname]['times'],
etime=inparms[parmname]['timewidths'],
asStartEnd=False)
pdb_out.addValues(parmname,ValueHolder)
pdb_out.flush()
def main(parmdb_p, parmdb_a, parmdb_out, clobber=True):
"""
Merges facet selfcal parmdbs into a single parmdb
Parameters
----------
parmdb_p : str
Filename of CommonScalarPhase and TEC parmdb
parmdb_a : str
Filename of Gain parmdb. The nearset match in frequency to that of the
input band will be used
parmdb_out : str
Filename of output file
clobber : bool, optional
If True, overwrite existing output file
"""
if type(clobber) is str:
if clobber.lower() == 'true':
clobber = True
else:
clobber = False
if os.path.exists(parmdb_out):
if clobber:
shutil.rmtree(parmdb_out)
else:
return
os.system('cp -r {0} {1}'.format(parmdb_p,parmdb_out))
## Copy over the Gains
pdb_out = pdb.parmdb(parmdb_out)
pdb_a = pdb.parmdb(parmdb_a)
for parmname in pdb_a.getNames():
parms = pdb_a.getValuesGrid(parmname)
ValueHolder = pdb_out.makeValue(values=parms[parmname]['values'],
sfreq=parms[parmname]['freqs'],
efreq=parms[parmname]['freqwidths'],
stime=parms[parmname]['times'],
etime=parms[parmname]['timewidths'],
asStartEnd=False)
pdb_out.addValues(parmname,ValueHolder)
pdb_out.flush()
def merge_parmdb(parmdb_p, parmdb_a, parmdb_out, clobber=False):
"""
Merges facet selfcal parmdbs into a parmdb for a single band
Parameters
----------
parmdb_p : str
Filename of CommonScalarPhase and TEC parmdb
parmdb_a : str
Filename of Gain parmdb. The nearset match in frequency to that of the
input band will be used
parmdb_out : str
Filename of output file
clobber : bool, optional
If True, overwrite existing output file
"""
if type(clobber) is str:
if clobber.lower() == 'true':
clobber = True
else:
clobber = False
if os.path.exists(parmdb_out):
if clobber:
shutil.rmtree(parmdb_out)
else:
return
pdb_out = parmdb.parmdb(parmdb_out, create=True)
# Copy over the CommonScalar phases and TEC
pdb_p = parmdb.parmdb(parmdb_p)
for parmname in pdb_p.getNames():
parms = pdb_p.getValuesGrid(parmname)
pdb_out.addValues(parms)
# Copy over the Gains
pdb_a = parmdb.parmdb(parmdb_a)
for parmname in pdb_a.getNames():
parms = pdb_a.getValuesGrid(parmname)
pdb_out.addValues(parms)
# Write values
pdb_out.flush()
def main(parmdb_p, parmdb_a, parmdb_out, clobber=True):
"""
Merges facet selfcal parmdbs into a single parmdb
Parameters
----------
parmdb_p : str
Filename of CommonScalarPhase and TEC parmdb
parmdb_a : str
Filename of Gain parmdb. The nearset match in frequency to that of the
input band will be used
parmdb_out : str
Filename of output file
clobber : bool, optional
If True, overwrite existing output file
"""
if type(clobber) is str:
if clobber.lower() == 'true':
clobber = True
else:
clobber = False
if os.path.exists(parmdb_out):
if clobber:
shutil.rmtree(parmdb_out)
else:
return
os.system('cp -r {0} {1}'.format(parmdb_p, parmdb_out))
## Copy over the Gains
pdb_out = pdb.parmdb(parmdb_out)
pdb_a = pdb.parmdb(parmdb_a)
for parmname in pdb_a.getNames():
parms = pdb_a.getValuesGrid(parmname)
ValueHolder = pdb_out.makeValue(values=parms[parmname]['values'],
sfreq=parms[parmname]['freqs'],
efreq=parms[parmname]['freqwidths'],
stime=parms[parmname]['times'],
etime=parms[parmname]['timewidths'],
asStartEnd=False)
pdb_out.addValues(parmname, ValueHolder)
pdb_out.flush()
def solplot_clock(parmdb, imageroot, refstationi, plot_international=False):
parmdbmtable = lp.parmdb(parmdb)
soldict = parmdbmtable.getValuesGrid('*')
names = parmdbmtable.getNames()
'Gain:1:1:Phase:RS508HBA'
stationsnames = np.array([name.split(':')[-1] for name in names])
stationsnames = np.unique(stationsnames)
if not plot_international:
stationsnames = np.array([name for name in stationsnames if name[0] in ['C','R'] ])
Nstat = len(stationsnames)
refstation = stationsnames[refstationi]
#phase_ref = soldict['Clock:{s}'.format(s=refstation)]['values']
times= soldict['Clock:1:{s}'.format(s=refstation)]['times']
Nr = int(np.ceil(np.sqrt(Nstat)))
Nc = int(np.ceil(np.float(Nstat)/Nr))
f, ax = plt.subplots(Nr, Nc, sharex=True, sharey=True, figsize=(16,12))
axs = ax.reshape((Nr*Nc,1))
ymin = 2
ymax = 0
for istat, station in enumerate(stationsnames):
clock00 = soldict['Clock:0:{s}'.format(s=station)]['values']
clock11 = soldict['Clock:1:{s}'.format(s=station)]['values']
if len(clock00) > 0:
ymax = max(np.max(clock00),ymax)
if len(clock11) > 0:
ymax = max(np.max(clock11),ymax)
if len(clock00) > 0:
ymin = min(np.min(clock00),ymin)
if len(clock11) > 0:
ymin = min(np.min(clock11),ymin)
# don't plot flagged phases
#phase = np.ma.masked_where(phase==0, phase)
#try:
#if len(phase11) > 1000:
#fmt = ','
#else:
#fmt = '.'
#except TypeError:
#print "no phases"
fmt = '.'
ls='none'
axs[istat][0].plot(times, clock00, color='b', marker=fmt, ls=ls, label='Clock 0:0', mec='b')
axs[istat][0].plot(times, clock11, color='g', marker=fmt, ls=ls, label='Clock 1:1', mec='g')
axs[istat][0].set_ylim(ymin, ymax)
axs[istat][0].set_xlim(times.min(), times.max())
axs[istat][0].set_title(station)
f.savefig(imageroot+"_clock.png",dpi=100)
return
def solplot_phase_phasors(parmdb, imageroot, refstationi, plot_international=False):
parmdbmtable = lp.parmdb(parmdb)
soldict = parmdbmtable.getValuesGrid('*')
names = parmdbmtable.getNames()
'Gain:1:1:Phase:RS508HBA'
stationsnames = np.array([name.split(':')[-1] for name in names])
stationsnames = np.unique(stationsnames)
if not plot_international:
stationsnames = np.array([name for name in stationsnames if name[0] in ['C','R'] ])
Nstat = len(stationsnames)
refstation = stationsnames[refstationi]
phase11_ref = soldict['Gain:1:1:Phase:{s}'.format(s=refstation)]['values']
phase00_ref = soldict['Gain:0:0:Phase:{s}'.format(s=refstation)]['values']
times= soldict['Gain:1:1:Phase:{s}'.format(s=refstation)]['times']
Nr = int(np.ceil(np.sqrt(Nstat)))
Nc = int(np.ceil(np.float(Nstat)/Nr))
f, ax = plt.subplots(Nr, Nc, sharex=True, sharey=True, figsize=(16,12))
axs = ax.reshape((Nr*Nc,1))
for istat, station in enumerate(stationsnames):
phase11 = soldict['Gain:1:1:Phase:{s}'.format(s=station)]['values']
phase00 = soldict['Gain:0:0:Phase:{s}'.format(s=station)]['values']
# don't plot flagged phases
phase00 = np.ma.masked_where(phase00==0, phase00)
phase11 = np.ma.masked_where(phase11==0, phase11)
#try:
#if len(phase11) > 1000:
#fmt = ','
#else:
#fmt = '.'
#except TypeError:
#print "no phases"
if len(times) > 1000:
fmt = ','
else:
fmt = '.'
ls='none'
axs[istat][0].plot(times, normalize(phase00-phase00_ref), color='b', marker=fmt, ls=ls, label='Gain:0:0:Phase',mec='b')
axs[istat][0].plot(times, normalize(phase11-phase11_ref), color='g', marker=fmt, ls=ls, label='Gain:1:1:Phase',mec='g')
axs[istat][0].set_ylim(-3.2, 3.2)
axs[istat][0].set_xlim(times.min(), times.max())
axs[istat][0].set_title(station)
f.savefig(imageroot+"_phase.png",dpi=100)
return
def make_empty_parmdb(outname):
myParmdb=pdb.parmdb(outname,create=True)
myParmdb.addDefValues("Gain:0:0:Ampl",1.)
myParmdb.addDefValues("Gain:1:1:Ampl",1.)
myParmdb.addDefValues("DirectionalGain:0:0:Ampl",1.)
myParmdb.addDefValues("DirectionalGain:1:1:Ampl",1.)
myParmdb.addDefValues("Gain:0:0:Real",1.)
myParmdb.addDefValues("Gain:1:1:Real",1.)
myParmdb.addDefValues("DirectionalGain:0:0:Real",1.)
myParmdb.addDefValues("DirectionalGain:1:1:Real",1.)
myParmdb.addDefValues("AntennaOrientation",5.497787144)
myParmdb.addDefValues("RotationMeasure",1e-6)
return myParmdb
def make_empty_parmdb(outname):
myParmdb = pdb.parmdb(outname, create=True)
myParmdb.addDefValues("Gain:0:0:Ampl", 1.)
myParmdb.addDefValues("Gain:1:1:Ampl", 1.)
myParmdb.addDefValues("DirectionalGain:0:0:Ampl", 1.)
myParmdb.addDefValues("DirectionalGain:1:1:Ampl", 1.)
myParmdb.addDefValues("Gain:0:0:Real", 1.)
myParmdb.addDefValues("Gain:1:1:Real", 1.)
myParmdb.addDefValues("DirectionalGain:0:0:Real", 1.)
myParmdb.addDefValues("DirectionalGain:1:1:Real", 1.)
myParmdb.addDefValues("AntennaOrientation", 5.497787144)
myParmdb.addDefValues("RotationMeasure", 1e-6)
return myParmdb
def solplot_phase_phasors(parmdb, imageroot, refstationi, plot_international=False):
parmdbmtable = lp.parmdb(parmdb)
soldict = parmdbmtable.getValuesGrid('*')
names = parmdbmtable.getNames()
'Gain:1:1:Phase:RS508HBA'
stationsnames = np.array([name.split(':')[-1] for name in names])
stationsnames = np.unique(stationsnames)
if not plot_international:
stationsnames = np.array([name for name in stationsnames if name[0] in ['C','R'] ])
Nstat = len(stationsnames)
refstation = stationsnames[refstationi]
phase11_ref = soldict['Gain:1:1:Phase:{s}'.format(s=refstation)]['values']
phase00_ref = soldict['Gain:0:0:Phase:{s}'.format(s=refstation)]['values']
times= soldict['Gain:1:1:Phase:{s}'.format(s=refstation)]['times']
num_channels = phase11_ref.shape[1]
Nr = int(np.ceil(np.sqrt(Nstat)))
Nc = int(np.ceil(np.float(Nstat)/Nr))
for chan_indx in range(num_channels):
f, ax = plt.subplots(Nr, Nc, sharex=True, sharey=True, figsize=(16,12))
axs = ax.reshape((Nr*Nc,1))
for istat, station in enumerate(stationsnames):
phase11 = soldict['Gain:1:1:Phase:{s}'.format(s=station)]['values'][:, chan_indx]
phase00 = soldict['Gain:0:0:Phase:{s}'.format(s=station)]['values'][:, chan_indx]
phase00_ref_chan = phase00_ref[:, chan_indx]
phase11_ref_chan = phase11_ref[:, chan_indx]
# don't plot flagged phases
phase00 = np.ma.masked_where(phase00==0, phase00)
phase11 = np.ma.masked_where(phase11==0, phase11)
if len(times) > 1000:
fmt = ','
else:
fmt = '.'
ls='none'
axs[istat][0].plot(times, normalize(phase00-phase00_ref_chan), color='b', marker=fmt, ls=ls, label='Gain:0:0:Phase',mec='b')
axs[istat][0].plot(times, normalize(phase11-phase11_ref_chan), color='g', marker=fmt, ls=ls, label='Gain:1:1:Phase',mec='g')
axs[istat][0].set_ylim(-3.2, 3.2)
axs[istat][0].set_xlim(times.min(), times.max())
axs[istat][0].set_title(station)
f.savefig(imageroot+"_phase_channel{}.png".format(chan_indx),dpi=100)
plt.close(f)
parmdbmtable = False
del(soldict)
def getbadstations(parmdbname, cutoff, antnames):
''' Flag bad stations based on amplitudes in a parmdb.
Returns a semicolon separated list, to be used in msselect'''
sl = set([])
p = lp.parmdb(parmdbname)
dv = p.getDefValues('*Ampl*')
for val in dv:
if dv[val][0][0] < cutoff:
sl.add(val.split(':')[-1])
outstring = ''
for s in sl:
if s in antnames:
outstring = outstring + '!' + s + ';'
return outstring[:-1]
def getbadstations(parmdbname,cutoff,antnames):
''' Flag bad stations based on amplitudes in a parmdb.
Returns a semicolon separated list, to be used in msselect'''
sl = set([])
p = lp.parmdb(parmdbname)
dv = p.getDefValues('*Ampl*')
for val in dv:
if dv[val][0][0] < cutoff:
sl.add(val.split(':')[-1])
outstring = ''
for s in sl:
if s in antnames:
outstring = outstring + '!' + s + ';'
return outstring[:-1]
def makePhasePlots(self,instTable, outDir):
"""
For a given instrument table, this function plots the phase solutions for each station with respect to the first station
To do: Internally the code now assumes that the phase solutions were derived using BBS with phasors=True in the SOLVE step. In later versions, check the form in which the gain solutions are stored and plot accordingly.
Written by Sarrvesh S. Sridhar
corrected by Nicolas Vilchez
v1.0 created 11 Feb 2014
"""
# Read the input instrument table
if instTable == '':
print 'An instrument table must be specified.'; return
try:
thisTable = lp.parmdb(instTable)
except:
print 'Error: Unable to read the instrument table. Will skip plotting the solutions.'
return
# Create the output directory
if not exists(outDir):
makedirs(outDir)
# Get the list of gain terms in this instrument table
gTabList = thisTable.getNames(parmnamepattern='Gain*Phase*')
nGainTerms = len(gTabList)
# Get the name of the first station
refStationName = gTabList[0].split(':')[-1]
refValues = thisTable.getValuesGrid(parmnamepattern=gTabList[0])[gTabList[0]]['values']
# Get the common time axis for all stations
timeAxis = thisTable.getValuesGrid(parmnamepattern=gTabList[0])[gTabList[0]]['times']
timeSinceStart = timeAxis - timeAxis[0]
del timeAxis
# Plot the phase for each entry in the gain table
for i in range(1,nGainTerms):
thisStationName = gTabList[i].split(':')[-1]
elementName = 'G'+str(gTabList[i].split(':')[1])+str(gTabList[i].split(':')[2])
print 'Plotting the phase solutions in', elementName, 'for station', thisStationName
values = thisTable.getValuesGrid(parmnamepattern=gTabList[i])[gTabList[i]]['values']
plt.figure(1)
plt.plot(timeSinceStart, self.normalize(values-refValues), 'ro')
plt.xlabel('Time (seconds)')
plt.ylabel('Phase (radians)')
fileName = outDir+'/'+thisStationName+'_'+elementName+'.png'
plt.title('Station: '+thisStationName+' '+elementName)
plt.grid(True)
plt.savefig(fileName, bbox_inches='tight')
plt.close()
def solplot_scalarphase(parmdb, imageroot, refstationi, plot_international=False):
parmdbmtable = lp.parmdb(parmdb)
soldict = parmdbmtable.getValuesGrid('*')
names = parmdbmtable.getNames()
'Gain:1:1:Phase:RS508HBA'
stationsnames = np.array([name.split(':')[-1] for name in names])
stationsnames = np.unique(stationsnames)
if not plot_international:
stationsnames = np.array([name for name in stationsnames if name[0] in ['C','R'] ])
Nstat = len(stationsnames)
refstation = stationsnames[refstationi]
phase_ref = soldict['CommonScalarPhase:{s}'.format(s=refstation)]['values']
times= soldict['CommonScalarPhase:{s}'.format(s=refstation)]['times']
num_channels = phase_ref.shape[1]
Nr = int(Nstat)
Nc = 1
for chan_indx in range(num_channels):
f, ax = plt.subplots(Nr, Nc, sharex=True, sharey=True, figsize=(12,72))
axs = ax.reshape((Nr*Nc,1))
for istat, station in enumerate(stationsnames):
phase = soldict['CommonScalarPhase:{s}'.format(s=station)]['values'][:, chan_indx]
phase_ref_chan = phase_ref[:, chan_indx]
# don't plot flagged phases
phase = np.ma.masked_where(phase==0, phase)
#try:
if len(phase) > 1000:
fmt = ','
else:
fmt = '.'
ls= 'none'
axs[istat][0].plot(times, normalize(phase-phase_ref_chan), color='b', marker=fmt, ls=ls, label='CommonScalarPhase',mec='b')
axs[istat][0].set_ylim(-3.2, 3.2)
axs[istat][0].set_xlim(times.min(), times.max())
axs[istat][0].set_title(station)
f.savefig(imageroot+"_scalarphase_channel{}.png".format(chan_indx),dpi=100)
plt.close(f)
parmdbmtable = False
del(soldict)
def __init__(self, name):
self._reader = None
self._table = None
self._table_name = None
idx = name.find(':')
assert(idx > 0)
package_name = name[:idx]
self._table_name = name[idx+1:]
if package_name == "bbs":
self._table = parmdb.parmdb(self._table_name)
self._reader = read_bbs_gain
elif package_name == "casa":
self._table = tb.table(self._table_name)
self._reader = read_casa_gain
else:
assert(false)
def createRMParmdb(MS,parmdbname,create=True,patchname='',
server="ftp://ftp.unibe.ch/aiub/CODE/",
prefix='CODG',
ionexPath="IONEXdata/",
earth_rot=0,
timestep=900.,
stat_names=None):
myParmdb=parmdb.parmdb(parmdbname,create=create)
if create:
myParmdb.addDefValues("Gain:0:0:Ampl",1.e-4)
myParmdb.addDefValues("DirectionalGain:0:0:Ampl",1.)
myParmdb.addDefValues("DirectionalGain:1:1:Ampl",1.)
myParmdb.addDefValues("Gain:0:0:Real",1.e-4)
myParmdb.addDefValues("Gain:1:1:Real",1.e-4)
myParmdb.addDefValues("DirectionalGain:0:0:Real",1.)
myParmdb.addDefValues("DirectionalGain:1:1:Real",1.)
myMS=tab.table(MS)
stations=tab.table(myMS.getkeyword('ANTENNA')).getcol('NAME')
stat_pos=tab.table(myMS.getkeyword('ANTENNA')).getcol('POSITION')
if not stat_names==None:
if not(stat_names=='all'):
stat_pos=[stat_pos[stations.index(name)] for name in stat_names]
else:
stat_names=stations[:]
else:
stat_names=stations[2:3]
stat_pos=stat_pos[2:3]
result=gt.getRM(MS=MS,server=server,ionexPath=ionexPath,prefix=prefix,earth_rot=earth_rot,timestep=timestep,stat_names=stat_names,stat_positions=stat_pos)
RM=result['RM']
for st in stations:
#print "storing station",st,(st in RM.keys())
if not (st in RM.keys()):
stname=RM.keys()[0]
else:
stname=st
RM[stname]=RM[stname].reshape(RM[stname].shape[:1]+(1,))
myValue=myParmdb.makeValue(values=RM[stname], sfreq=1e10, efreq=2e10, stime=result['times'], etime=np.ones(result['times'].shape,dtype=float)*result['timestep'], asStartEnd=False)
if patchname:
valuename = "RotationMeasure:%s:%s"%(st,patchname)
else:
valuename = "RotationMeasure:%s"%(st)
myParmdb.deleteValues(valuename)
myParmdb.addValues(valuename,myValue)
def __init__(self, name):
self._reader = None
self._table = None
self._table_name = None
idx = name.find(":")
assert idx > 0
package_name = name[:idx]
self._table_name = name[idx + 1 :]
if package_name == "bbs":
self._table = parmdb.parmdb(self._table_name)
self._reader = read_bbs_gain
elif package_name == "casa":
self._table = tb.table(self._table_name)
self._reader = read_casa_gain
else:
assert false
def main(parmdbfile, targetms):
if not os.path.exists(parmdbfile):
print "Parmdb file %s doesn't exist!" % parmdbfile
return(1)
if not os.path.exists(targetms):
print "Target ms %s doesn't exist!" % targetms
return(1)
msinfo = ReadMs(targetms)
# Open up the parmdb, get an example value
parmdb = pdb.parmdb(parmdbfile)
parnames = parmdb.getNames()
STvalue = None
for name in parnames:
if "ST" in name:
STvalue = parmdb.getValuesGrid(name)[name]
break
if STvalue == None:
print "Couldn't find station ST001 entry"
return(1)
# Add the necessary stations
for antenna_id, antenna in enumerate(msinfo.stations):
if "CS" in antenna:
ValueHolder = parmdb.makeValue(values=STvalue['values'],
sfreq=STvalue['freqs'],
efreq=STvalue['freqwidths'],
stime=STvalue['times'],
etime=STvalue['timewidths'],
asStartEnd=False)
csparname = ':'.join(name.split(':')[0:-1]) + ':' + antenna
## in case the values already exist (and may be NaN) they need to be deleted first
parmdb.deleteValues(csparname)
parmdb.addValuesGrid(csparname,ValueHolder)
parmdb.flush()
parmdb = 0
return(0)
def parmDBs2h5parm(h5parmName,
parmDBs,
antennaFile,
fieldFile,
skydbFile=None,
compression=5,
solsetName=None):
"""
Write the contents of a list of parmDBs into a losoto-style hdf5-file
h5parmName - name (path) of the hdf5 file to generate
parmDBs - list with the names of the parmDBs
antennaFile - name (path) of an ANTENNA table of the observation
fieldFile - name (path) of a FIELD table of the observation
skydbFile - name (path) of a skydb table of the calibration run (Needed for direction dependent parameters)
compresion - compression level for the hdf5-file (0 - none ; 9 - highest)
solsetName - name of the solset to generate (default: "sol000")
"""
# open/create the h5parm file and the solution-set
h5parmDB = h5parm(h5parmName, readonly=False, complevel=compression)
solset = h5parmDB.makeSolset(solsetName)
#open the first instrument table, so that we know where to look for names and stuff
firstInst = pdb.parmdb(parmDBs[0])
# get unique list of solution types
solTypes = list(set(x1.split(":")[0] for x1 in firstInst.getNames()))
# rewrite solTypes in order to put together
# Gain <-> DirectionalGain
# CommonRotationAngle <-> RotationAngle
# CommonScalarPhase <-> ScalarPhase
# it also separate Real/Imag/Ampl/Phase into different solTypes
if "Gain" in solTypes:
solTypes.remove('Gain')
solTypes.append('*Gain:*:Real')
solTypes.append('*Gain:*:Imag')
solTypes.append('*Gain:*:Ampl')
solTypes.append('*Gain:*:Phase')
if "DirectionalGain" in solTypes:
solTypes.remove('DirectionalGain')
solTypes.append('*Gain:*:Real')
solTypes.append('*Gain:*:Imag')
solTypes.append('*Gain:*:Ampl')
solTypes.append('*Gain:*:Phase')
if "RotationAngle" in solTypes:
solTypes.remove('RotationAngle')
solTypes.append('*RotationAngle')
if "CommonRotationAngle" in solTypes:
solTypes.remove('CommonRotationAngle')
solTypes.append('*RotationAngle')
if "RotationMeasure" in solTypes:
solTypes.remove('RotationMeasure')
solTypes.append('*RotationMeasure')
if "ScalarPhase" in solTypes:
solTypes.remove('ScalarPhase')
solTypes.append('*ScalarPhase')
if "CommonScalarPhase" in solTypes:
solTypes.remove('CommonScalarPhase')
solTypes.append('*ScalarPhase')
if "CommonScalarAmplitude" in solTypes:
solTypes.remove('CommonScalarAmplitude')
solTypes.append('*ScalarAmplitude')
# and remove duplicate entries
solTypes = list(set(solTypes))
for solType in solTypes:
if len(firstInst.getNames(solType + ':*')) == 0: continue
pols = set()
dirs = set()
ants = set()
freqs = set()
times = set()
ptype = set()
for pDBname in parmDBs:
instrumentdb = pdb.parmdb(pDBname)
# create the axes grid, necessary if not all entries have the same axes lenght
data = instrumentdb.getValuesGrid(solType + ':*')
for solEntry in data:
pol, dir, ant, parm = parmdbToAxes(solEntry)
if pol != None: pols |= set([pol])
if dir != None: dirs |= set([dir])
if ant != None: ants |= set([ant])
freqs |= set(data[solEntry]['freqs'])
times |= set(data[solEntry]['times'])
#close the parmDB
instrumentdb = 0
pols = np.sort(list(pols))
dirs = np.sort(list(dirs))
ants = np.sort(list(ants))
freqs = np.sort(list(freqs))
times = np.sort(list(times))
shape = [
i
for i in (len(pols), len(dirs), len(ants), len(freqs), len(times))
if i != 0
]
vals = np.empty(shape)
vals[:] = np.nan
weights = np.zeros(shape)
for pDBname in parmDBs:
instrumentdb = pdb.parmdb(pDBname)
# fill the values
data = instrumentdb.getValuesGrid(solType + ':*')
if 'Real' in solType:
dataIm = instrumentdb.getValuesGrid(
solType.replace('Real', 'Imag') + ':*')
if 'Imag' in solType:
dataRe = instrumentdb.getValuesGrid(
solType.replace('Imag', 'Real') + ':*')
for solEntry in data:
pol, dir, ant, parm = parmdbToAxes(solEntry)
ptype |= set([solEntry.split(':')[0]
]) # original parmdb solution type
freq = data[solEntry]['freqs']
time = data[solEntry]['times']
val = data[solEntry]['values']
# convert Real and Imag in Amp and Phase respectively
if parm == 'Real':
solEntryIm = solEntry.replace('Real', 'Imag')
valI = dataIm[solEntryIm]['values']
val = np.sqrt((val**2) + (valI**2))
if parm == 'Imag':
solEntryRe = solEntry.replace('Imag', 'Real')
valR = dataRe[solEntryRe]['values']
val = np.arctan2(val, valR)
coords = []
if pol != None:
polCoord = np.searchsorted(pols, pol)
coords.append(polCoord)
if dir != None:
dirCoord = np.searchsorted(dirs, dir)
coords.append(dirCoord)
if ant != None:
antCoord = np.searchsorted(ants, ant)
coords.append(antCoord)
freqCoord = np.searchsorted(freqs, freq)
timeCoord = np.searchsorted(times, time)
vals[tuple(coords)][np.ix_(freqCoord, timeCoord)] = val.T
weights[tuple(coords)][np.ix_(freqCoord, timeCoord)] = 1
#close the parmDB
instrumentdb = 0
vals = np.nan_to_num(vals) # replace nans with 0 (flagged later)
if solType == '*RotationAngle':
np.putmask(weights, vals == 0., 0) # flag where val=0
h5parmDB.makeSoltab(solset, 'rotation', axesNames=['dir','ant','freq','time'], \
axesVals=[dirs,ants,freqs,times], vals=vals, weights=weights, parmdbType=', '.join(list(ptype)))
elif solType == '*RotationMeasure':
np.putmask(weights, vals == 0., 0) # flag where val=0
h5parm.makeSoltab(solset, 'rotationmeasure', axesNames=['dir','ant','freq','time'], \
axesVals=[dirs,ants,freqs,times], vals=vals, weights=weights, parmdbType=', '.join(list(ptype)))
elif solType == '*ScalarPhase':
np.putmask(weights, vals == 0., 0)
h5parmDB.makeSoltab(solset, 'scalarphase', axesNames=['dir','ant','freq','time'], \
axesVals=[dirs,ants,freqs,times], vals=vals, weights=weights, parmdbType=', '.join(list(ptype)))
elif solType == '*ScalarAmplitude':
np.putmask(weights, vals == 0., 0)
h5parm.makeSoltab(solset, 'scalaramplitude', axesNames=['dir','ant','freq','time'], \
axesVals=[dirs,ants,freqs,times], vals=vals, weights=weights, parmdbType=', '.join(list(ptype)))
elif solType == 'Clock':
np.putmask(weights, vals == 0., 0)
# clock may be diag or scalar
if len(pols) == 0:
h5parm.makeSoltab(solset, 'clock', axesNames=['ant','freq','time'], \
axesVals=[ants,freqs,times], vals=vals, weights=weights, parmdbType=', '.join(list(ptype)))
else:
h5parm.makeSoltab(solset, 'clock', axesNames=['pol','ant','freq','time'], \
axesVals=[pol,ants,freqs,times], vals=vals, weights=weights, parmdbType=', '.join(list(ptype)))
elif solType == 'TEC':
np.putmask(weights, vals == 0., 0)
# tec may be diag or scalar
if len(pols) == 0:
h5parm.makeSoltab(solset, 'tec', axesNames=['dir','ant','freq','time'], \
axesVals=[dirs,ants,freqs,times], vals=vals, weights=weights, parmdbType=', '.join(list(ptype)))
else:
h5parm.makeSoltab(solset, 'tec', axesNames=['pol','dir','ant','freq','time'], \
axesVals=[pols,dirs,ants,freqs,times], vals=vals, weights=weights, parmdbType=', '.join(list(ptype)))
elif solType == '*Gain:*:Real' or solType == '*Gain:*:Ampl':
np.putmask(
vals, vals == 0.,
1) # nans end up into 1s (as BBS output, flagged next line)
np.putmask(weights, vals == 1., 0) # flag where val=1
h5parmDB.makeSoltab(solset, 'amplitude', axesNames=['pol','dir','ant','freq','time'], \
axesVals=[pols,dirs,ants,freqs,times], vals=vals, weights=weights, parmdbType=', '.join(list(ptype)))
elif solType == '*Gain:*:Imag' or solType == '*Gain:*:Phase':
np.putmask(weights, vals == 0., 0) # falg where val=0
h5parmDB.makeSoltab(solset, 'phase', axesNames=['pol','dir','ant','freq','time'], \
axesVals=[pols,dirs,ants,freqs,times], vals=vals, weights=weights, parmdbType=', '.join(list(ptype)))
antennaTable = pt.table(antennaFile, ack=False)
antennaNames = antennaTable.getcol('NAME')
antennaPositions = antennaTable.getcol('POSITION')
antennaTable.close()
antennaTable = solset._f_get_child('antenna')
antennaTable.append(zip(*(antennaNames, antennaPositions)))
fieldTable = pt.table(fieldFile, ack=False)
phaseDir = fieldTable.getcol('PHASE_DIR')
pointing = phaseDir[0, 0, :]
fieldTable.close()
sourceTable = solset._f_get_child('source')
# add the field centre, that is also the direction for Gain and CommonRotationAngle
sourceTable.append([('pointing', pointing)])
dirs = []
for tab in solset._v_children:
c = solset._f_get_child(tab)
if c._v_name != 'antenna' and c._v_name != 'source':
dirs.extend(list(set(c.dir)))
# remove duplicates
dirs = list(set(dirs))
# remove any pointing (already in the table)
if 'pointing' in dirs:
dirs.remove('pointing')
if dirs != []:
if skydbFile == None:
logging.critical(
'No sky table given, but direction dependent parameters in parmDB. Exiting!'
)
sys.exit(1)
sourceFile = skydbFile + '/SOURCES'
src_table = pt.table(sourceFile, ack=False)
sub_tables = src_table.getsubtables()
vals = []
ra = dec = np.nan
has_patches_subtable = False
for sub_table in sub_tables:
if 'PATCHES' in sub_table:
has_patches_subtable = True
if has_patches_subtable:
# Read values from PATCHES subtable
src_table.close()
sourceFile = skydbFile + '/SOURCES/PATCHES'
src_table = pt.table(sourceFile, ack=False)
patch_names = src_table.getcol('PATCHNAME')
patch_ras = src_table.getcol('RA')
patch_decs = src_table.getcol('DEC')
for source in dirs:
try:
patch_indx = patch_names.index(source)
ra = patch_ras[patch_indx]
dec = patch_decs[patch_indx]
except ValueError:
ra = np.nan
dec = np.nan
vals.append([ra, dec])
src_table.close()
else:
# Try to read default values from parmdb instead
skydb = pdb.parmdb(skydbFile)
vals = []
ra = dec = np.nan
for source in dirs:
try:
ra = skydb.getDefValues('Ra:' + source)['Ra:' +
source][0][0]
dec = skydb.getDefValues('Dec:' + source)['Dec:' +
source][0][0]
except KeyError:
# Source not found in skymodel parmdb, try to find components
ra = np.array(
skydb.getDefValues('Ra:*' + source + '*').values())
dec = np.array(
skydb.getDefValues('Dec:*' + source + '*').values())
if len(ra) == 0 or len(dec) == 0:
ra = np.nan
dec = np.nan
else:
ra = ra.mean()
dec = dec.mean()
vals.append([ra, dec])
sourceTable.append(zip(*(dirs, vals)))
solsetname = solset._v_name
# close the hdf5-file
h5parmDB.close()
return solsetname
snapshot1_copy_list = glob.glob(str(input) + '/1stsnap/*.MS.copy')
snapshot2_copy_list = glob.glob(str(input) + '/2ndsnap/*.MS.copy')
#run RMextract on dataset, once for each snapshot
print 'Running RM extract'
createRMparm(snapshot1_copy_list, 1)
os.system('mv RMPARMDB.1SNAPSHOT ' + str(input))
createRMparm(snapshot2_copy_list, 2)
os.system('mv RMPARMDB.2SNAPSHOT ' + str(input))
#output RM extract info
inputparmdb1 = pdb.parmdb(str(input) + '/RMPARMDB.1SNAPSHOT')
inputparmdb2 = pdb.parmdb(str(input) + '/RMPARMDB.2SNAPSHOT')
length1 = len(inputparmdb1.getNames())
length2 = len(inputparmdb2.getNames())
#some code here to find the average RM from the parmdb files and output it onto a text file.
fullavgval1 = np.array([])
for i in range(length1):
value = np.mean(
inputparmdb1.getValuesGrid(
inputparmdb1.getNames()[i])[inputparmdb1.getNames()[i]]['values']
) #find the mean RM correction of a single station for the snapshot
fullavgval1 = np.append(fullavgval1, value)
finalavg1 = np.mean(fullavgval1)
def main(input_mslist, parmdb_name, outparmdb, clobber=True, scratch_dir=None):
"""
Merges parmdbs in time into a single parmdb
The parmdbs are assumed to be located in the input MS with name
parmdb_name
Parameters
----------
input_mslist : list
List of input MS file names
parmdb_name : str
Name of parmdb (relative to MS files)
outparmdb : str
Name of output merged parmdb
clobber : bool, optional
If True, overwrite existing output file
scratch_dir : str, optional
Scratch directory for temp storage
"""
if type(input_mslist) is str:
input_mslist = input_mslist.strip('[]').split(',')
input_mslist = [f.strip() for f in input_mslist]
inparmdbs = [os.path.join(ms, parmdb_name) for ms in input_mslist]
if type(clobber) is str:
if clobber.lower() == 'true':
clobber = True
else:
clobber = False
if os.path.exists(outparmdb):
if clobber:
os.system('rm -rf {0}'.format(outparmdb))
else:
return
# Copy to scratch directory if specified
if scratch_dir is not None:
inparmdbs_orig = inparmdbs
inparmdbs = [os.path.join(scratch_dir, os.path.basename(inp)+'_{}'.format(i))
for i, inp in enumerate(inparmdbs_orig)]
outparmdb_orig = outparmdb
outparmdb = os.path.join(scratch_dir, os.path.basename(outparmdb_orig))
for inp_orig, inp in zip(inparmdbs_orig, inparmdbs):
shutil.copytree(inp_orig, inp)
pdb_concat = pdb.parmdb(outparmdb, create=True)
for i, inparmdb in enumerate(inparmdbs):
pdb_add = pdb.parmdb(inparmdb)
parms = pdb_add.getValuesGrid('*')
for parmname in pdb_add.getNames():
flagged = np.where(np.logical_or(parms[parmname]['values'] == 0.0,
np.isnan(parms[parmname]['values'])))
parms[parmname]['values'][flagged] = np.nan
for t, (t1, t2, tw1, tw2) in enumerate(zip(parms[parmname]['times'][:-1], parms[parmname]['times'][1:],
parms[parmname]['timewidths'][:-1], parms[parmname]['timewidths'][1:])):
if (t2 - t1) < (tw1 + tw2) / 2.0:
# check for overlaps and adjust widths if needed (but don't adjust last
# solution's width, as it may be truncated)
parms[parmname]['timewidths'][t] = t2 - t1
if t < len(parms[parmname]['times'])-2:
parms[parmname]['timewidths'][t+1] = t2 - t1
pdb_concat.addValues(parmname, parms[parmname]['values'],
sfreq=parms[parmname]['freqs'],
efreq=parms[parmname]['freqwidths'],
stime=parms[parmname]['times'],
etime=parms[parmname]['timewidths'], asStartEnd=False)
pdb_concat.flush()
pdb_add = False
pdb_concat = False
# Copy output to original path and delete copies if scratch directory is specified
if scratch_dir is not None:
shutil.copytree(outparmdb, outparmdb_orig)
shutil.rmtree(outparmdb)
for inp in inparmdbs:
shutil.rmtree(inp)
def solplot_phase(parmdb, imageroot, refstationi, norm_amp_lim=False, median_amp=False, plot_international=False):
parmdbmtable = lp.parmdb(parmdb)
soldict = parmdbmtable.getValuesGrid('*')
names = parmdbmtable.getNames()
'Gain:1:1:Phase:RS508HBA'
stationsnames = np.array([name.split(':')[-1] for name in names])
stationsnames = np.unique(stationsnames)
if not plot_international:
stationsnames = np.array([name for name in stationsnames if name[0] in ['C','R'] ])
Nstat = len(stationsnames)
refstation = stationsnames[refstationi]
times = soldict['Gain:1:1:Real:{s}'.format(s=refstation)]['times']
times = scaletimes(times)
real11_ref = soldict['Gain:1:1:Real:{s}'.format(s=refstation)]['values']
real00_ref = soldict['Gain:0:0:Real:{s}'.format(s=refstation)]['values']
imag11_ref = soldict['Gain:1:1:Imag:{s}'.format(s=refstation)]['values']
imag00_ref = soldict['Gain:0:0:Imag:{s}'.format(s=refstation)]['values']
num_channels = real11_ref.shape[1]
valscorr00 = real00_ref +1.j*imag00_ref
valscorr11 = real11_ref +1.j*imag11_ref
phase00_ref = np.angle(valscorr00)
phase11_ref = np.angle(valscorr11)
Nr = int(np.ceil(np.sqrt(Nstat)))
Nc = int(np.ceil(np.float(Nstat)/Nr))
for chan_indx in range(num_channels):
fp, axp = plt.subplots(Nr, Nc, sharex=True, sharey=True, figsize=(16,12))
axsp = axp.reshape((Nr*Nc,1))
for istat, station in enumerate(stationsnames):
real11 = soldict['Gain:1:1:Real:{s}'.format(s=station)]['values'][:, chan_indx]
real00 = soldict['Gain:0:0:Real:{s}'.format(s=station)]['values'][:, chan_indx]
imag11 = soldict['Gain:1:1:Imag:{s}'.format(s=station)]['values'][:, chan_indx]
imag00 = soldict['Gain:0:0:Imag:{s}'.format(s=station)]['values'][:, chan_indx]
valscorr00 = real00 +1.j*imag00
valscorr11 = real11 +1.j*imag11
phase00_ref_chan = phase00_ref[:, chan_indx]
phase11_ref_chan = phase11_ref[:, chan_indx]
if len(np.unique(real11)) > 500:
fmt = ','
else:
fmt = '.'
ls='none'
phase00 = np.angle(valscorr00)
phase11 = np.angle(valscorr11)
# don't plot flagged phases
phase00 = np.ma.masked_where(phase00==0, phase00)
phase11 = np.ma.masked_where(phase11==0, phase11)
axsp[istat][0].plot(times, normalize(phase00-phase00_ref_chan), color='b', marker=fmt, ls=ls, label='Gain:0:0:Phase',mec='b')
axsp[istat][0].plot(times, normalize(phase11-phase11_ref_chan), color='g', marker=fmt, ls=ls, label='Gain:1:1:Phase',mec='g')
axsp[istat][0].set_ylim(-3.2, 3.2)
axsp[istat][0].set_xlim(times.min(), times.max())
axsp[istat][0].set_title(station)
fp.savefig(imageroot+"_phase_channel{}.png".format(chan_indx),dpi=100)
plt.close(fp)
parmdbmtable = False
del(soldict)
def getStructure(pdname,antennas,nr_grid=1,doplot=True,outbasename='ionosphere',dofilter=True):
outfile = open(outbasename +'_structure.txt','w')
myt=pd.parmdb(pdname)
val=myt.getValuesGrid("*")
stat=pt.table(antennas)
names=stat.getcol('NAME')
pos=stat.getcol('POSITION')
stations={}
for i,j in zip(names,pos):
stations[i]=j
phx=[]
phy=[]
allposx=[]
allposy=[]
nrtimes=val[val.keys()[0]]['times'].shape[0]
timestep=int(nrtimes/nr_grid)
for i in sorted(val.keys()):
if not "CS" in i:
continue
#if 'CS201' in i:
# continue
#if 'CS030' in i:
# continue
if "0:0" in i:
phx.append(val[i]['values'].flatten())
allposx.append(stations[i.split(":")[-1]])
if "1:1" in i:
phy.append(val[i]['values'].flatten())
allposy.append(stations[i.split(":")[-1]])
phx=np.array(phx)
phy=np.array(phy)
allposx=np.array(allposx)
D=allposx[:,np.newaxis,:]-allposx[np.newaxis,:]
D2=np.sqrt(np.sum(D**2,axis=-1))
allposy=np.array(allposy)
Dy=allposy[:,np.newaxis,:]-allposy[np.newaxis,:]
D2y=np.sqrt(np.sum(Dy**2,axis=-1))
S0s=[]
betas=[]
for itime in xrange(nr_grid+1):
tm=[0,1e9]
if itime<nr_grid:
tm[0]=itime*timestep
tm[1]=tm[0]+timestep
dphx=phx[:,np.newaxis,tm[0]:tm[1]]-phx[np.newaxis,:,tm[0]:tm[1]]
dphy=phy[:,np.newaxis,tm[0]:tm[1]]-phy[np.newaxis,:,tm[0]:tm[1]]
dphx=np.unwrap(np.remainder(dphx-dphx[:,:,0][:,:,np.newaxis]+np.pi,2*np.pi))
dvarx=np.var(dphx,axis=-1)
dphy=np.unwrap(np.remainder(dphy-dphy[:,:,0][:,:,np.newaxis]+np.pi,2*np.pi))
dvary=np.var(dphy,axis=-1)
myselect=np.logical_and(D2>0,np.logical_and(np.any(np.logical_and(dvarx>1e-7,dvarx<.1),axis=0)[np.newaxis],np.any(np.logical_and(dvarx>1e-7,dvarx<.1),axis=1)[:,np.newaxis]))
if dofilter:
x=D2[myselect]
y=dvarx[myselect]
# Seems like real values dont occur above 1.0
flagselect = np.where(y > 1.0)
xplot = np.delete(x,flagselect)
yplot = np.delete(y,flagselect)
bins = np.logspace(np.log10(np.min(xplot)),np.log10(np.max(xplot)),10)
binys = []
binxs = []
for i in range(0,len(bins)-1):
binmin = bins[i]
binmax = bins[i+1]
inbin = np.intersect1d(np.where(xplot > binmin),np.where(xplot < binmax))
binxs.append((binmin+binmax)/2.0)
#binys.append(np.percentile(y[inbin],90))
binys.append(np.median(yplot[inbin])+10*mad(yplot[inbin]))
x0 = [0.1,1.0,1.0]
xfit, flag = scipy.optimize.leastsq(residuals, x0, args=(binys,binxs))
flagselect = np.where(y > model(x,xfit))
print xfit,'fitting'
if doplot:
x=D2[myselect]
y=dvarx[myselect]
subplot(2,1,1)
scatter(x,y,color='b')
if dofilter:
y2 = y[flagselect]
x2 = x[flagselect]
scatter(x2,y2,color='r')
scatter(x,model(x,xfit),color='gray',linestyle='-')#,'gray',linestyle='-',linewidth=3)
x=D2[np.logical_and(D2>1e3,myselect)]
y=dvarx[np.logical_and(D2>1e3,myselect)]
if dofilter:
flagselect = np.where(y > model(x,xfit))
x = np.delete(x,flagselect)
y = np.delete(y,flagselect)
A=np.ones((2,x.shape[0]),dtype=float)
A[1,:]=np.log10(x)
par=np.dot(np.linalg.inv(np.dot(A,A.T)),np.dot(A,np.log10(y)))
S0=10**(-1*np.array(par)[0]/np.array(par)[1])
if doplot:
plot(x,10**(par[0]+par[1]*np.log10(x)),'r-')
xscale("log")
yscale("log")
#ylim(1e-3,2)
xlim(30,4000)
myselect=np.logical_and(D2y>0,np.logical_and(np.any(np.logical_and(dvary>1e-7,dvary<.1),axis=0)[np.newaxis],np.any(np.logical_and(dvary>1e-7,dvary<.1),axis=1)[:,np.newaxis]))
if dofilter:
x=D2y[myselect]
y=dvary[myselect]
# seems like real values dont occur above 1.0
flagselect = np.where(y > 1.0)
xplot = np.delete(x,flagselect)
yplot = np.delete(y,flagselect)
bins = np.logspace(np.log10(np.min(xplot)),np.log10(np.max(xplot)),20)
binys = []
binxs = []
for i in range(0,len(bins)-1):
binmin = bins[i]
binmax = bins[i+1]
inbin = np.intersect1d(np.where(xplot > binmin),np.where(xplot < binmax))
binxs.append((binmin+binmax)/2.0)
#binys.append(np.percentile(y[inbin],90))
binys.append(np.median(yplot[inbin])+10*mad(yplot[inbin]))
x0 = [0.1,1.0,1.0]
xfit, flag = scipy.optimize.leastsq(residuals, x0, args=(binys,binxs))
flagselect = np.where(y > model(x,xfit))
print xfit,'fitting'
if doplot:
x=D2y[myselect]
y=dvary[myselect]
subplot(2,1,2)
scatter(x,y,color='b')
if dofilter:
y2 = y[flagselect]
x2 = x[flagselect]
scatter(x2,y2,color='r')
scatter(x,model(x,xfit),color='gray',linestyle='-')#,'gray',linestyle='-',linewidth=3)
x=D2y[np.logical_and(D2y>1e3,myselect)]
y=dvary[np.logical_and(D2y>1e3,myselect)]
if dofilter:
flagselect = np.where(y > model(x,xfit))
x = np.delete(x,flagselect)
y = np.delete(y,flagselect)
A=np.ones((2,x.shape[0]),dtype=float)
A[1,:]=np.log10(x)
pary=np.dot(np.linalg.inv(np.dot(A,A.T)),np.dot(A,np.log10(y)))
S0y=10**(-1*np.array(pary)[0]/np.array(pary)[1])
if doplot:
plot(x,10**(pary[0]+pary[1]*np.log10(x)),'r-')
xscale("log")
yscale("log")
#ylim(1e-3,2)
xlim(30,4000)
savefig('%s_structure.png'%outbasename)
close()
cla()
S0s.append([S0,S0y])
betas.append([par[1],pary[1]])
outfile.write('S0s ****%s**** %s beta %s %s\n'%(S0,S0y,par[1],pary[1]))
return S0s,betas
def solplot_tec(parmdb, imageroot, refstationi, plot_international=False, freq=None):
parmdbmtable = lp.parmdb(parmdb)
soldict = parmdbmtable.getValuesGrid('*')
if freq is None:
tfrange = parmdbmtable.getRange()
print 'freqrange', tfrange[0:2]
freq = np.average(tfrange[0:2])
print 'freq', freq/1e6, 'MHz'
names = parmdbmtable.getNames()
'Gain:1:1:Phase:RS508HBA'
stationsnames = np.array([name.split(':')[-1] for name in names])
stationsnames = np.unique(stationsnames)
if not plot_international:
stationsnames = np.array([name for name in stationsnames if name[0] in ['C','R'] ])
Nstat = len(stationsnames)
refstation = stationsnames[refstationi]
times = soldict['CommonScalarPhase:{s}'.format(s=refstation)]['times']
times = scaletimes(times)
phase_ref = soldict['CommonScalarPhase:{s}'.format(s=refstation)]['values']
tec_ref = soldict['TEC:{s}'.format(s=refstation)]['values']
#Nr = int(np.ceil(np.sqrt(Nstat)))
#Nc = int(np.ceil(np.float(Nstat)/Nr))
Nr = int(Nstat)
Nc = 1
f, ax = plt.subplots(Nr, Nc, sharex=True, sharey=True, figsize=(12,72))
axs = ax.reshape((Nr*Nc,1))
ymin = 2
ymax = 0
for istat, station in enumerate(stationsnames):
phase = soldict['CommonScalarPhase:{s}'.format(s=station)]['values']
tec = soldict['TEC:{s}'.format(s=station)]['values']
phase = np.ma.masked_where(phase==0, phase)
if len(times) > 1000:
fmt = ','
else:
fmt = '.'
ls='none'
phasep = phase - phase_ref
tecp = -8.44797245e9*(tec - tec_ref)/freq
axs[istat][0].plot(times, np.mod(phasep+tecp +np.pi, 2*np.pi) - np.pi, color='b', marker=fmt, ls=ls, label='Phase+TEC', mec='b')
axs[istat][0].set_ylim(-np.pi, np.pi)
axs[istat][0].set_xlim(times.min(), times.max())
axs[istat][0].set_title(station)
#phase_ref = parms['CommonScalarPhase:'+ ref_antenna]['values'][::]
#phase = parms['CommonScalarPhase:'+ antenna ]['values'][::]
##print phase-phase_ref
#clock = parms['Clock:0:'+ antenna ]['values'][::]
#clock_ref = parms['Clock:0:'+ ref_antenna ]['values'][::]
#TEC = parms['TEC:'+ antenna ]['values'][::]
#TEC_ref = parms['TEC:'+ ref_antenna ]['values'][::]
#phasep = phase-phase_ref
#clockp = 2.*pi*(clock-clock_ref)*freq
#tecp = -8.44797245e9*(TEC-TEC_ref)/freq
##plot(numpy.mod(phasep+tecp + clockp +pi, 2*pi) - pi, '.')
f.savefig(imageroot+"_tec.png",dpi=100)
return
def main(parmdb_p, parmdb_a, parmdb_out, clobber=True, scratch_dir=None):
"""
Merges facet selfcal parmdbs into a single parmdb
Parameters
----------
parmdb_p : str
Filename of CommonScalarPhase and TEC parmdb
parmdb_a : str
Filename of Gain parmdb. The nearset match in frequency to that of the
input band will be used
parmdb_out : str
Filename of output file
clobber : bool, optional
If True, overwrite existing output file
scratch_dir : str, optional
Scratch directory for temp storage
"""
if type(clobber) is str:
if clobber.lower() == 'true':
clobber = True
else:
clobber = False
if os.path.exists(parmdb_out):
if clobber:
shutil.rmtree(parmdb_out)
else:
return
# Copy to scratch directory if specified
if scratch_dir is not None:
parmdb_p_orig = parmdb_p
parmdb_p = os.path.join(scratch_dir, os.path.basename(parmdb_p_orig))
parmdb_a_orig = parmdb_a
parmdb_a = os.path.join(scratch_dir, os.path.basename(parmdb_a_orig))
parmdb_out_orig = parmdb_out
parmdb_out = os.path.join(scratch_dir,
os.path.basename(parmdb_out_orig))
shutil.copytree(parmdb_p_orig, parmdb_p)
shutil.copytree(parmdb_a_orig, parmdb_a)
shutil.copytree(parmdb_p, parmdb_out)
## Copy over the Gains
pdb_out = pdb.parmdb(parmdb_out)
pdb_a = pdb.parmdb(parmdb_a)
parms = pdb_a.getValuesGrid('*')
for parmname in pdb_a.getNames():
# Set flagged solutions to NaN
flagged = np.where(
np.logical_or(parms[parmname]['values'] == 0.0,
np.isnan(parms[parmname]['values'])))
parms[parmname]['values'][flagged] = np.nan
ValueHolder = pdb_out.makeValue(values=parms[parmname]['values'],
sfreq=parms[parmname]['freqs'],
efreq=parms[parmname]['freqwidths'],
stime=parms[parmname]['times'],
etime=parms[parmname]['timewidths'],
asStartEnd=False)
pdb_out.addValues(parmname, ValueHolder)
pdb_out.flush()
# Copy output to original path and delete copies if scratch directory is specified
if scratch_dir is not None:
shutil.copytree(parmdb_out, parmdb_out_orig)
shutil.rmtree(parmdb_out)
shutil.rmtree(parmdb_p)
shutil.rmtree(parmdb_a)
def solplot_tec(parmdb,
imageroot,
refstationi,
plot_international=False,
freq=None,
stations=[]):
parmdbmtable = lp.parmdb(parmdb)
soldict = parmdbmtable.getValuesGrid('*')
if freq is None:
tfrange = parmdbmtable.getRange()
print 'freqrange', tfrange[0:2]
freq = np.average(tfrange[0:2])
print 'freq', freq / 1e6, 'MHz'
names = parmdbmtable.getNames()
'Gain:1:1:Phase:RS508HBA'
stationsnames = np.array([name.split(':')[-1] for name in names])
stationsnames = np.unique(stationsnames)
if not plot_international:
stationsnames = np.array(
[name for name in stationsnames if name[0] in ['C', 'R']])
# specify list of stations (none for all)#
if len(stations) > 0:
selstationsnames = []
for s in stations:
if s in stationsnames:
selstationsnames.append(s)
else:
print "invalid station {s}".format(s=s)
stationsnames = selstationsnames
Nstat = len(stationsnames)
refstation = stationsnames[refstationi]
times = soldict['CommonScalarPhase:{s}'.format(s=refstation)]['times']
times = scaletimes(times)
phase_ref = soldict['CommonScalarPhase:{s}'.format(s=refstation)]['values']
tec_ref = soldict['TEC:{s}'.format(s=refstation)]['values']
Nr = int(np.ceil(np.sqrt(Nstat)))
Nc = int(np.ceil(np.float(Nstat) / Nr))
f, ax = plt.subplots(Nr, Nc, sharex=True, sharey=True, figsize=(16, 12))
plt.subplots_adjust(left=0.05,
bottom=0.05,
right=0.95,
top=0.95,
wspace=0,
hspace=0)
axs = ax.reshape((Nr * Nc, 1))
ymin = 2
ymax = 0
for istat, station in enumerate(stationsnames):
phase = soldict['CommonScalarPhase:{s}'.format(s=station)]['values']
tec = soldict['TEC:{s}'.format(s=station)]['values']
phase = np.ma.masked_where(phase == 0, phase)
if len(times) > 1000:
fmt = ','
else:
fmt = '.'
ls = 'none'
phasep = phase - phase_ref
tecp = -8.44797245e9 * (tec - tec_ref) / freq
axs[istat][0].plot(times,
np.mod(phasep + tecp + np.pi, 2 * np.pi) - np.pi,
color='b',
marker=fmt,
ls=ls,
label='Phase+TEC',
mec='b')
axs[istat][0].set_ylim(-np.pi, np.pi)
axs[istat][0].set_xlim(times.min(), times.max())
#axs[istat][0].set_title(station)
axs[istat][0].text(0.5,
0.9,
station,
horizontalalignment='center',
verticalalignment='center',
transform=axs[istat][0].transAxes,
backgroundcolor='white',
color='k')
#phase_ref = parms['CommonScalarPhase:'+ ref_antenna]['values'][::]
#phase = parms['CommonScalarPhase:'+ antenna ]['values'][::]
##print phase-phase_ref
#clock = parms['Clock:0:'+ antenna ]['values'][::]
#clock_ref = parms['Clock:0:'+ ref_antenna ]['values'][::]
#TEC = parms['TEC:'+ antenna ]['values'][::]
#TEC_ref = parms['TEC:'+ ref_antenna ]['values'][::]
#phasep = phase-phase_ref
#clockp = 2.*pi*(clock-clock_ref)*freq
#tecp = -8.44797245e9*(TEC-TEC_ref)/freq
##plot(numpy.mod(phasep+tecp + clockp +pi, 2*pi) - pi, '.')
f.savefig(imageroot + "_tec.png", dpi=100)
return
def solplot_phase(parmdb,
imageroot,
refstationi,
norm_amp_lim=False,
median_amp=False,
plot_international=False):
parmdbmtable = lp.parmdb(parmdb)
soldict = parmdbmtable.getValuesGrid('*')
names = parmdbmtable.getNames()
'Gain:1:1:Phase:RS508HBA'
stationsnames = np.array([name.split(':')[-1] for name in names])
stationsnames = np.unique(stationsnames)
if not plot_international:
stationsnames = np.array(
[name for name in stationsnames if name[0] in ['C', 'R']])
Nstat = len(stationsnames)
refstation = stationsnames[refstationi]
times = soldict['Gain:1:1:Real:{s}'.format(s=refstation)]['times']
times = scaletimes(times)
real11_ref = soldict['Gain:1:1:Real:{s}'.format(s=refstation)]['values']
real00_ref = soldict['Gain:0:0:Real:{s}'.format(s=refstation)]['values']
imag11_ref = soldict['Gain:1:1:Imag:{s}'.format(s=refstation)]['values']
imag00_ref = soldict['Gain:0:0:Imag:{s}'.format(s=refstation)]['values']
num_channels = real11_ref.shape[1]
valscorr00 = real00_ref + 1.j * imag00_ref
valscorr11 = real11_ref + 1.j * imag11_ref
phase00_ref = np.angle(valscorr00)
phase11_ref = np.angle(valscorr11)
Nr = int(np.ceil(np.sqrt(Nstat)))
Nc = int(np.ceil(np.float(Nstat) / Nr))
for chan_indx in range(num_channels):
fp, axp = plt.subplots(Nr,
Nc,
sharex=True,
sharey=True,
figsize=(16, 12))
axsp = axp.reshape((Nr * Nc, 1))
for istat, station in enumerate(stationsnames):
real11 = soldict['Gain:1:1:Real:{s}'.format(
s=station)]['values'][:, chan_indx]
real00 = soldict['Gain:0:0:Real:{s}'.format(
s=station)]['values'][:, chan_indx]
imag11 = soldict['Gain:1:1:Imag:{s}'.format(
s=station)]['values'][:, chan_indx]
imag00 = soldict['Gain:0:0:Imag:{s}'.format(
s=station)]['values'][:, chan_indx]
valscorr00 = real00 + 1.j * imag00
valscorr11 = real11 + 1.j * imag11
phase00_ref_chan = phase00_ref[:, chan_indx]
phase11_ref_chan = phase11_ref[:, chan_indx]
if len(np.unique(real11)) > 500:
fmt = ','
else:
fmt = '.'
ls = 'none'
phase00 = np.angle(valscorr00)
phase11 = np.angle(valscorr11)
# don't plot flagged phases
phase00 = np.ma.masked_where(phase00 == 0, phase00)
phase11 = np.ma.masked_where(phase11 == 0, phase11)
axsp[istat][0].plot(times,
normalize(phase00 - phase00_ref_chan),
color='b',
marker=fmt,
ls=ls,
label='Gain:0:0:Phase',
mec='b')
axsp[istat][0].plot(times,
normalize(phase11 - phase11_ref_chan),
color='g',
marker=fmt,
ls=ls,
label='Gain:1:1:Phase',
mec='g')
axsp[istat][0].set_ylim(-3.2, 3.2)
axsp[istat][0].set_xlim(times.min(), times.max())
axsp[istat][0].set_title(station)
fp.savefig(imageroot + "_phase_channel{}.png".format(chan_indx),
dpi=100)
return
def main(fast_parmdb, slow_parmdb, output_file, freqstep=1, preapply_parmdb=None,
scratch_dir=None, cache_values=False):
"""
Converts multiple selfcal tables to single gain table
Parameters
----------
fast_parmdb : str
File with fast phase (TEC and CommonScalarPhase) solutions
fast_parmdb : str
File with slow gain solutions
output_file : str
Output filename
freqstep : int
Frequency step to divide up frequency width of solutions
preapply_parmdb : str
File with combined fast phase (TEC and CommonScalarPhase) and slow phase
solutions for pre-application
scratch_dir : str, optional
Scratch directory for temp storage
cache_values : bool, optional
Cache parmdbs in memory. Caching speeds up the script but uses more memory
"""
freqstep = int(freqstep)
# Copy to scratch directory if specified
if scratch_dir is not None:
fast_parmdb_orig = fast_parmdb
fast_parmdb = os.path.join(scratch_dir, os.path.basename(fast_parmdb_orig))
slow_parmdb_orig = slow_parmdb
slow_parmdb = os.path.join(scratch_dir, os.path.basename(slow_parmdb_orig))
output_file_orig = output_file
output_file = os.path.join(scratch_dir, os.path.basename(output_file_orig))
shutil.copytree(fast_parmdb_orig, fast_parmdb)
shutil.copytree(slow_parmdb_orig, slow_parmdb)
# Get various quantities over which we must iterate
fast_pdb = lp.parmdb(fast_parmdb)
fast_soldict = fast_pdb.getValuesGrid('*')
station_names = list(set([s.split(':')[-1] for s in fast_pdb.getNames()]))
fast_times = fast_soldict['CommonScalarPhase:{s}'.format(s=station_names[0])]['times']
fast_timewidths = fast_soldict['CommonScalarPhase:{s}'.format(s=station_names[0])]['timewidths']
fast_timestep = np.mean(fast_timewidths)
key_names = fast_soldict.keys()
del(fast_soldict)
if preapply_parmdb is not None:
preapply_pdb = lp.parmdb(preapply_parmdb)
preapply_soldict = preapply_pdb.getValuesGrid('*')
fast_times_preapply = preapply_soldict['Gain:0:0:Phase:{s}'.format(s=station_names[0])]['times']
fast_timewidths_preapply = preapply_soldict['Gain:0:0:Phase:{s}'.format(s=station_names[0])]['timewidths']
fast_timestep_preapply = np.mean(fast_timewidths_preapply)
slow_freqs_preapply = preapply_soldict['Gain:0:0:Phase:{s}'.format(s=station_names[0])]['freqs']
slow_freqwidths_preapply = preapply_soldict['Gain:0:0:Phase:{s}'.format(s=station_names[0])]['freqwidths']
slow_freqstep_preapply = np.mean(slow_freqwidths_preapply)
del(preapply_soldict)
slow_pdb = lp.parmdb(slow_parmdb)
slow_soldict = slow_pdb.getValuesGrid('*')
output_pdb = lp.parmdb(output_file, create=True)
slow_freqs = slow_soldict['Gain:0:0:Real:{s}'.format(s=station_names[0])]['freqs']
slow_freqwidths = slow_soldict['Gain:0:0:Real:{s}'.format(s=station_names[0])]['freqwidths']
slow_freqstep = np.mean(slow_freqwidths)
del(slow_soldict)
if any('Gain:0:1:' in s for s in key_names):
pol_list = ['0:0', '1:1', '0:1', '1:0']
else:
pol_list = ['0:0', '1:1']
# Check preapply to make sure we use the finest grid
if preapply_parmdb is not None:
if slow_freqstep_preapply < slow_freqstep:
slow_freqs = slow_freqs_preapply
slow_freqwidths = slow_freqwidths_preapply
if fast_timestep_preapply < fast_timestep:
fast_times = fast_times_preapply
fast_timewidths = fast_timewidths_preapply
# Determine final time and frequency grid. This step is not needed if a
# preapply_parmdb is specified, as it will already be made on the final grids
if freqstep > 1 and preapply_parmdb is None:
final_freqs = []
final_freqwidths = []
for freq, freqwidth in zip(slow_freqs, slow_freqwidths):
final_freqwidth = freqwidth / freqstep
low_freq = freq - freqwidth / 2
for i in range(freqstep):
final_freqs.append(low_freq + final_freqwidth * (i + 0.5))
final_freqwidths.append(final_freqwidth)
final_freqs = np.array(final_freqs)
final_freqwidths = np.array(final_freqwidths)
else:
final_freqs = slow_freqs
final_freqwidths = slow_freqwidths
# Identify any gaps in time (frequency gaps are not allowed), as we need to handle
# each section separately if gaps are present
delta_times = fast_times[1:] - fast_times[:-1]
gaps = np.where( np.logical_or(delta_times > fast_timewidths[:-1]*2., np.array([d%99 for d in range(delta_times.shape[0])]) == 0) )
gaps_ind = gaps[0] + 1
gaps_ind = np.append(gaps_ind, np.array([len(fast_times)]))
# Get values on the final time and frequency grid
g_start = 0
for g in gaps_ind:
if cache_values:
if preapply_parmdb is not None:
preapply_soldict = preapply_pdb.getValues('*', final_freqs, final_freqwidths,
fast_times[g_start:g], fast_timewidths[g_start:g], asStartEnd=False)
fast_soldict = fast_pdb.getValues('*', final_freqs, final_freqwidths,
fast_times[g_start:g], fast_timewidths[g_start:g], asStartEnd=False)
slow_soldict = slow_pdb.getValues('*', final_freqs, final_freqwidths,
fast_times[g_start:g], fast_timewidths[g_start:g], asStartEnd=False)
# Add various phase and amp corrections together
for station in station_names:
if not cache_values:
if preapply_parmdb is not None:
preapply_soldict = preapply_pdb.getValues('*{}*'.format(station), final_freqs, final_freqwidths,
fast_times[g_start:g], fast_timewidths[g_start:g], asStartEnd=False)
fast_soldict = fast_pdb.getValues('*{}*'.format(station), final_freqs, final_freqwidths,
fast_times[g_start:g], fast_timewidths[g_start:g], asStartEnd=False)
slow_soldict = slow_pdb.getValues('*{}*'.format(station), final_freqs, final_freqwidths,
fast_times[g_start:g], fast_timewidths[g_start:g], asStartEnd=False)
try:
fast_phase = np.copy(fast_soldict['CommonScalarPhase:{s}'.format(s=station)]['values'])
tec = np.copy(fast_soldict['TEC:{s}'.format(s=station)]['values'])
tec_phase = -8.44797245e9 * tec / final_freqs
for pol in pol_list:
slow_real = np.copy(slow_soldict['Gain:'+pol+':Real:{s}'.format(s=station)]['values'])
slow_imag = np.copy(slow_soldict['Gain:'+pol+':Imag:{s}'.format(s=station)]['values'])
slow_amp = np.sqrt((slow_real**2) + (slow_imag**2))
slow_phase = np.arctan2(slow_imag, slow_real)
total_amp = slow_amp
if preapply_parmdb is not None:
fast_phase_preapply = np.copy(preapply_soldict['Gain:'+pol+':Phase:{s}'.format(s=station)]['values'])
total_phase = np.mod(fast_phase + tec_phase + slow_phase +
fast_phase_preapply + np.pi, 2*np.pi) - np.pi
# Identify zero phase solutions and set the corresponding entries in total_phase and total_amp to NaN
total_phase = np.where(fast_phase_preapply == 0.0, np.nan, total_phase)
total_amp = np.where(fast_phase_preapply == 0.0, np.nan, total_amp)
else:
total_phase = np.mod(fast_phase + tec_phase + slow_phase + np.pi, 2*np.pi) - np.pi
# Identify zero phase solutions and set the corresponding entries in total_phase and total_amp to NaN
total_phase = np.where(np.logical_or(fast_phase == 0.0, tec_phase == 0.0), np.nan, total_phase)
total_amp = np.where(np.logical_or(fast_phase == 0.0, tec_phase == 0.0), np.nan, total_amp)
output_pdb.addValues('Gain:'+pol+':Phase:{}'.format(station),
total_phase, final_freqs, final_freqwidths,
fast_times[g_start:g], fast_timewidths[g_start:g], asStartEnd=False)
output_pdb.addValues('Gain:'+pol+':Ampl:{}'.format(station),
total_amp, final_freqs, final_freqwidths,
fast_times[g_start:g], fast_timewidths[g_start:g], asStartEnd=False)
except KeyError:
# The station in question does not have solutions for this
# time period
continue
g_start = g
# Write values
output_pdb.flush()
# Copy output to original path and delete copies if scratch directory is specified
if scratch_dir is not None:
if os.path.exists(output_file_orig):
shutil.rmtree(output_file_orig)
shutil.copytree(output_file, output_file_orig)
shutil.rmtree(output_file)
shutil.rmtree(fast_parmdb)
shutil.rmtree(slow_parmdb)
rah = dirs[0, 0] * 12.0 / np.pi #getting right acesion
decr = dirs[0, 1] # getting declination
R = np.zeros((nspec, nfld))
Q = np.zeros((nspec, nfld))
U = np.zeros((nspec, nfld))
mask = np.ones((nspec, nfld), dtype=bool)
QU = {}
rah = dirs[0, 0] * 12.0 / np.pi
dech = dirs[0, 1]
table = pdb.parmdb(options.inputin)
times = table.getValuesGrid(table.getNames()[0])[table.getNames()[0]][
'times'] #only need to get one row of times from the gain table
#time0=86400.0*floor(times[0]/86400.0) **** Why were these lines not included? TC ****
#rtimes=times-time0
gainsXX = table.getValuesGrid(table.getNames()[0])[table.getNames()[0]][
'values'] # **** 00 is XX and 11 is YY TC ****
gainsYY = table.getValuesGrid(
table.getNames()[6])[table.getNames()[6]]['values']
freqs = table.getValuesGrid(table.getNames()[0])[table.getNames()[0]]['freqs']
# Getting the range of parangles
parang = np.zeros(len(times))
##sblist = [ int(d.split('/')[-2].replace('SB','')) for d in dirlist if (os.path.exists(d+'3C196_24_{sb}.ms/instrument'.format(sb=d.split('/')[-2]))) & (d.split('/')[-2] not in badsb)]
sblist = []
##instlist = [ '/data2/wwilliams/hba_bootes_lc015/BOOTES_24_V2/SB{sb:03d}/3C196_24_SB{sb:03d}.ms/instrument'.format(sb=sb) for sb in sblist]
##mslist = [ '/data2/wwilliams/hba_bootes_lc015/BOOTES_24_V2/SB{sb:03d}/3C196_24_SB{sb:03d}.ms'.format(sb=sb) for sb in sblist]
args = sys.argv
mslist = args[1][1:-1].split(',')
prefix = args[2]
instlist = []
for item in mslist:
instlist.append(item + '/instrument')
#sblist.append(os.path.basename(item)+'/instrument')
sblist.append(float(item.split('_SB')[-1].split('_')[0]))
#/data2/wwilliams/hba_bootes_lc015/BOOTES_24_V2/SB190/3C196_24_SB190.ms/instrument
print 'MSLIST: ',mslist
print 'INSTLIST: ',instlist
parmdbmtable = lp.parmdb(instlist[0])
names = parmdbmtable.getNames()
print names
stationsnames = numpy.array([name.split(':')[-1] for name in names])
stationsnames = numpy.unique(stationsnames)
refstation = stationsnames[1]
freq_per_sb = numpy.zeros(len(instlist))
for iinst,inst in enumerate(instlist):
ms = mslist[iinst]
print inst, ms
t = pt.table(ms, readonly=False)
freq_tab = pt.table(ms + '/SPECTRAL_WINDOW')
freq = freq_tab.getcol('REF_FREQUENCY')
def parmDBs2h5parm(h5parmName,parmDBs,antennaFile,fieldFile,skydbFile,compression=5,solsetName=None):
"""
Write the contents of a list of parmDBs into a losoto-style hdf5-file
h5parmName - name (path) of the hdf5 file to generate
parmDBs - list with the names of the parmDBs
antennaFile - name (path) of an ANTENNA table of the observation
fieldFile - name (path) of a FIELD table of the observation
skydbFile - name (path) of a skydb table of the calibration run
compresion - compression level for the hdf5-file (0 - none ; 9 - highest)
solsetName - name of the solset to generate (default: "sol000")
"""
# open/create the h5parm file and the solution-set
h5parmDB = h5parm(h5parmName, readonly = False, complevel = compression)
solset = h5parmDB.makeSolset(solsetName)
#open the first instrument table, so that we know where to look for names and stuff
firstInst = pdb.parmdb(parmDBs[0])
# get unique list of solution types
solTypes = list(set(x1.split(":")[0] for x1 in firstInst.getNames()))
# rewrite solTypes in order to put together
# Gain <-> DirectionalGain
# CommonRotationAngle <-> RotationAngle
# CommonScalarPhase <-> ScalarPhase
# it also separate Real/Imag/Ampl/Phase into different solTypes
if "Gain" in solTypes:
solTypes.remove('Gain')
solTypes.append('*Gain:*:Real')
solTypes.append('*Gain:*:Imag')
solTypes.append('*Gain:*:Ampl')
solTypes.append('*Gain:*:Phase')
if "DirectionalGain" in solTypes:
solTypes.remove('DirectionalGain')
solTypes.append('*Gain:*:Real')
solTypes.append('*Gain:*:Imag')
solTypes.append('*Gain:*:Ampl')
solTypes.append('*Gain:*:Phase')
if "RotationAngle" in solTypes:
solTypes.remove('RotationAngle')
solTypes.append('*RotationAngle')
if "CommonRotationAngle" in solTypes:
solTypes.remove('CommonRotationAngle')
solTypes.append('*RotationAngle')
if "ScalarPhase" in solTypes:
solTypes.remove('ScalarPhase')
solTypes.append('*ScalarPhase')
if "CommonScalarPhase" in solTypes:
solTypes.remove('CommonScalarPhase')
solTypes.append('*ScalarPhase')
# and remove duplicate entries
solTypes = list(set(solTypes))
for solType in solTypes:
if len(firstInst.getNames(solType+':*')) == 0: continue
pols = set(); dirs = set(); ants = set();
freqs = set(); times = set(); ptype = set()
for pDBname in parmDBs:
instrumentdb = pdb.parmdb(pDBname)
# create the axes grid, necessary if not all entries have the same axes lenght
data = instrumentdb.getValuesGrid(solType+':*')
for solEntry in data:
pol, dir, ant, parm = parmdbToAxes(solEntry)
if pol != None: pols |= set([pol])
if dir != None: dirs |= set([dir])
if ant != None: ants |= set([ant])
freqs |= set(data[solEntry]['freqs'])
times |= set(data[solEntry]['times'])
#close the parmDB
instrumentdb = 0
pols = np.sort(list(pols)); dirs = np.sort(list(dirs));
ants = np.sort(list(ants)); freqs = np.sort(list(freqs));
times = np.sort(list(times))
shape = [i for i in (len(pols), len(dirs), len(ants), len(freqs), len(times)) if i != 0]
vals = np.empty(shape)
vals[:] = np.nan
weights = np.zeros(shape)
for pDBname in parmDBs:
instrumentdb = pdb.parmdb(pDBname)
# fill the values
data = instrumentdb.getValuesGrid(solType+':*')
if 'Real' in solType: dataIm = instrumentdb.getValuesGrid(solType.replace('Real','Imag')+':*')
if 'Imag' in solType: dataRe = instrumentdb.getValuesGrid(solType.replace('Imag','Real')+':*')
for solEntry in data:
pol, dir, ant, parm = parmdbToAxes(solEntry)
ptype |= set([solEntry.split(':')[0]]) # original parmdb solution type
freq = data[solEntry]['freqs']
time = data[solEntry]['times']
val = data[solEntry]['values']
# convert Real and Imag in Amp and Phase respectively
if parm == 'Real':
solEntryIm = solEntry.replace('Real','Imag')
valI = dataIm[solEntryIm]['values']
val = np.sqrt((val**2)+(valI**2))
if parm == 'Imag':
solEntryRe = solEntry.replace('Imag','Real')
valR = dataRe[solEntryRe]['values']
val = np.arctan2(val, valR)
coords = []
if pol != None:
polCoord = np.searchsorted(pols, pol)
coords.append(polCoord)
if dir != None:
dirCoord = np.searchsorted(dirs, dir)
coords.append(dirCoord)
if ant != None:
antCoord = np.searchsorted(ants, ant)
coords.append(antCoord)
freqCoord = np.searchsorted(freqs, freq)
timeCoord = np.searchsorted(times, time)
vals[tuple(coords)][np.ix_(freqCoord,timeCoord)] = val.T
weights[tuple(coords)][np.ix_(freqCoord,timeCoord)] = 1
#close the parmDB
instrumentdb = 0
vals = np.nan_to_num(vals) # replace nans with 0 (flagged later)
if solType == '*RotationAngle':
np.putmask(weights, vals == 0., 0) # flag where val=0
h5parmDB.makeSoltab(solset, 'rotation', axesNames=['dir','ant','freq','time'], \
axesVals=[dirs,ants,freqs,times], vals=vals, weights=weights, parmdbType=', '.join(list(ptype)))
elif solType == '*ScalarPhase':
np.putmask(weights, vals == 0., 0)
h5parmDB.makeSoltab(solset, 'scalarphase', axesNames=['dir','ant','freq','time'], \
axesVals=[dirs,ants,freqs,times], vals=vals, weights=weights, parmdbType=', '.join(list(ptype)))
elif solType == 'Clock':
np.putmask(weights, vals == 0., 0)
h5parmDB.makeSoltab(solset, 'clock', axesNames=['ant','freq','time'], \
axesVals=[ants,freqs,times], vals=vals, weights=weights, parmdbType=', '.join(list(ptype)))
elif solType == 'TEC':
np.putmask(weights, vals == 0., 0)
h5parmDB.makeSoltab(solset, 'tec', axesNames=['dir','ant','freq','time'], \
axesVals=[dirs,ants,freqs,times], vals=vals, weights=weights, parmdbType=', '.join(list(ptype)))
elif solType == '*Gain:*:Real' or solType == '*Gain:*:Ampl':
np.putmask(vals, vals == 0., 1) # nans end up into 1s (as BBS output, flagged next line)
np.putmask(weights, vals == 1., 0) # flag where val=1
h5parmDB.makeSoltab(solset, 'amplitude', axesNames=['pol','dir','ant','freq','time'], \
axesVals=[pols,dirs,ants,freqs,times], vals=vals, weights=weights, parmdbType=', '.join(list(ptype)))
elif solType == '*Gain:*:Imag' or solType == '*Gain:*:Phase':
#np.putmask(weights, vals == 0., 0) # TODO: NDPPP bug which put at 0 the reference antenna phase
h5parmDB.makeSoltab(solset, 'phase', axesNames=['pol','dir','ant','freq','time'], \
axesVals=[pols,dirs,ants,freqs,times], vals=vals, weights=weights, parmdbType=', '.join(list(ptype)))
antennaTable = pt.table(antennaFile, ack=False)
antennaNames = antennaTable.getcol('NAME')
antennaPositions = antennaTable.getcol('POSITION')
antennaTable.close()
antennaTable = solset._f_get_child('antenna')
antennaTable.append(zip(*(antennaNames,antennaPositions)))
fieldTable = pt.table(fieldFile, ack=False)
phaseDir = fieldTable.getcol('PHASE_DIR')
pointing = phaseDir[0, 0, :]
fieldTable.close()
sourceTable = solset._f_get_child('source')
# add the field centre, that is also the direction for Gain and CommonRotationAngle
sourceTable.append([('pointing',pointing)])
dirs = []
for tab in solset._v_children:
c = solset._f_getChild(tab)
if c._v_name != 'antenna' and c._v_name != 'source':
dirs.extend(list(set(c.dir)))
# remove duplicates
dirs = list(set(dirs))
# remove any pointing (already in the table)
if 'pointing' in dirs:
dirs.remove('pointing')
if dirs != []:
sourceFile = skydbFile + '/SOURCES'
src_table = pt.table(sourceFile, ack=False)
sub_tables = src_table.getsubtables()
vals = []
ra = dec = np.nan
has_patches_subtable = False
for sub_table in sub_tables:
if 'PATCHES' in sub_table:
has_patches_subtable = True
if has_patches_subtable:
# Read values from PATCHES subtable
src_table.close()
sourceFile = skydbFile + '/SOURCES/PATCHES'
src_table = pt.table(sourceFile, ack=False)
patch_names = src_table.getcol('PATCHNAME')
patch_ras = src_table.getcol('RA')
patch_decs = src_table.getcol('DEC')
for source in dirs:
try:
patch_indx = patch_names.index(source)
ra = patch_ras[patch_indx]
dec = patch_decs[patch_indx]
except ValueError:
ra = np.nan
dec = np.nan
vals.append([ra, dec])
src_table.close()
else:
# Try to read default values from parmdb instead
skydb = pdb.parmdb(skydbFile)
vals = []
ra = dec = np.nan
for source in dirs:
try:
ra = skydb.getDefValues('Ra:' + source)['Ra:' + source][0][0]
dec = skydb.getDefValues('Dec:' + source)['Dec:' + source][0][0]
except KeyError:
# Source not found in skymodel parmdb, try to find components
ra = np.array(skydb.getDefValues('Ra:*' + source + '*').values())
dec = np.array(skydb.getDefValues('Dec:*' + source + '*').values())
if len(ra) == 0 or len(dec) == 0:
ra = np.nan
dec = np.nan
else:
ra = ra.mean()
dec = dec.mean()
vals.append([ra, dec])
sourceTable.append(zip(*(dirs,vals)))
solsetname = solset._v_name
# close the hdf5-file
h5parmDB.close()
return solsetname
def solplot_phase(parmdb,
imageroot,
refstationi,
norm_amp_lim=False,
median_amp=False,
plot_international=False,
stations=[]):
parmdbmtable = lp.parmdb(parmdb)
soldict = parmdbmtable.getValuesGrid('*')
names = parmdbmtable.getNames()
'Gain:1:1:Phase:RS508HBA'
stationsnames = np.array([name.split(':')[-1] for name in names])
stationsnames = np.unique(stationsnames)
if not plot_international:
stationsnames = np.array(
[name for name in stationsnames if name[0] in ['C', 'R']])
# specify list of stations (none for all)#
if len(stations) > 0:
selstationsnames = []
for s in stations:
if s in stationsnames:
selstationsnames.append(s)
else:
print "invalid station {s}".format(s=s)
stationsnames = selstationsnames
Nstat = len(stationsnames)
refstation = stationsnames[refstationi]
times = soldict['Gain:1:1:Real:{s}'.format(s=refstation)]['times']
times = scaletimes(times)
real11_ref = soldict['Gain:1:1:Real:{s}'.format(s=refstation)]['values']
real00_ref = soldict['Gain:0:0:Real:{s}'.format(s=refstation)]['values']
imag11_ref = soldict['Gain:1:1:Imag:{s}'.format(s=refstation)]['values']
imag00_ref = soldict['Gain:0:0:Imag:{s}'.format(s=refstation)]['values']
valscorr00 = real00_ref + 1.j * imag00_ref
valscorr11 = real11_ref + 1.j * imag11_ref
phase00_ref = np.angle(valscorr00)
phase11_ref = np.angle(valscorr11)
Nr = int(np.ceil(np.sqrt(Nstat)))
Nc = int(np.ceil(np.float(Nstat) / Nr))
fp, axp = plt.subplots(Nr, Nc, sharex=True, sharey=True, figsize=(16, 12))
plt.subplots_adjust(left=0.05,
bottom=0.05,
right=0.95,
top=0.95,
wspace=0,
hspace=0)
axsp = axp.reshape((Nr * Nc, 1))
for istat, station in enumerate(stationsnames):
real11 = soldict['Gain:1:1:Real:{s}'.format(s=station)]['values']
real00 = soldict['Gain:0:0:Real:{s}'.format(s=station)]['values']
imag11 = soldict['Gain:1:1:Imag:{s}'.format(s=station)]['values']
imag00 = soldict['Gain:0:0:Imag:{s}'.format(s=station)]['values']
valscorr00 = real00 + 1.j * imag00
valscorr11 = real11 + 1.j * imag11
if len(np.unique(real11)) > 500:
fmt = ','
else:
fmt = '.'
ls = 'none'
phase00 = np.angle(valscorr00)
phase11 = np.angle(valscorr11)
#phase11 = soldict['Gain:1:1:Phase:{s}'.format(s=station)]
#phase00 = soldict['Gain:0:0:Phase:{s}'.format(s=station)]
# don't plot flagged phases
phase00 = np.ma.masked_where(phase00 == 0, phase00)
phase11 = np.ma.masked_where(phase11 == 0, phase11)
axsp[istat][0].plot(times,
normalize(phase00 - phase00_ref),
color='b',
marker=fmt,
ls=ls,
label='Gain:0:0:Phase',
mec='b')
axsp[istat][0].plot(times,
normalize(phase11 - phase11_ref),
color='g',
marker=fmt,
ls=ls,
label='Gain:1:1:Phase',
mec='g')
axsp[istat][0].set_ylim(-3.2, 3.2)
axsp[istat][0].set_xlim(times.min(), times.max())
#axsp[istat][0].set_title(station)
axsp[istat][0].text(0.5,
0.9,
station,
horizontalalignment='center',
verticalalignment='center',
transform=axsp[istat][0].transAxes,
backgroundcolor='white',
color='k')
fp.savefig(imageroot + "_phase.png", dpi=100)
return
def main(parmdb_in, parmdb_out, debug=False):
"""
Merges facet selfcal parmdbs into a parmdb for a single band
Parameters
----------
parmdb_in : str
Name of (path to) the parmdb with Gain in polar coordinates.
parmdb_out : str
Name of the new parmdb that will be written
"""
pdb_in = pdb.parmdb(parmdb_in, create=False)
pdb_out = make_empty_parmdb(parmdb_out)
inparms = pdb_in.getValuesGrid('*')
names_in = inparms.keys()
for parmname in names_in:
nameparts = parmname.split(':')
if (len(nameparts) == 5 and
(nameparts[0] == 'Gain' or nameparts[0] == 'DirectionalGain')):
if nameparts[3] == 'Phase':
amplname = nameparts[0] + ':' + nameparts[1] + ':' + nameparts[
2] + ':Ampl:' + nameparts[4]
if amplname in names_in:
amps = inparms[amplname]['values']
else:
amps = np.ones_like(inparms[parmname]['values'])
realname = nameparts[0] + ':' + nameparts[1] + ':' + nameparts[
2] + ':Real:' + nameparts[4]
imagname = nameparts[0] + ':' + nameparts[1] + ':' + nameparts[
2] + ':Imag:' + nameparts[4]
realvals = amps * np.cos(inparms[parmname]['values'])
imagvals = amps * np.sin(inparms[parmname]['values'])
if debug: print 'Adding parameters:', realname, imagname
ValueHolder = pdb_out.makeValue(
values=realvals,
sfreq=inparms[parmname]['freqs'],
efreq=inparms[parmname]['freqwidths'],
stime=inparms[parmname]['times'],
etime=inparms[parmname]['timewidths'],
asStartEnd=False)
pdb_out.addValues(realname, ValueHolder)
ValueHolder = pdb_out.makeValue(
values=imagvals,
sfreq=inparms[parmname]['freqs'],
efreq=inparms[parmname]['freqwidths'],
stime=inparms[parmname]['times'],
etime=inparms[parmname]['timewidths'],
asStartEnd=False)
pdb_out.addValues(imagname, ValueHolder)
elif nameparts[3] == 'Real' or nameparts[3] == 'Imag':
print 'Gain value:', parmname, 'not in polar coordinates!'
else:
ValueHolder = pdb_out.makeValue(
values=inparms[parmname]['values'],
sfreq=inparms[parmname]['freqs'],
efreq=inparms[parmname]['freqwidths'],
stime=inparms[parmname]['times'],
etime=inparms[parmname]['timewidths'],
asStartEnd=False)
pdb_out.addValues(parmname, ValueHolder)
pdb_out.flush()
def solplot_scalarphase(parmdb,
imageroot,
refstationi,
plot_international=False,
stations=[]):
parmdbmtable = lp.parmdb(parmdb)
soldict = parmdbmtable.getValuesGrid('*')
names = parmdbmtable.getNames()
'Gain:1:1:Phase:RS508HBA'
stationsnames = np.array([name.split(':')[-1] for name in names])
stationsnames = np.unique(stationsnames)
if not plot_international:
stationsnames = np.array(
[name for name in stationsnames if name[0] in ['C', 'R']])
# specify list of stations (none for all)#
if len(stations) > 0:
selstationsnames = []
for s in stations:
if s in stationsnames:
selstationsnames.append(s)
else:
print "invalid station {s}".format(s=s)
stationsnames = selstationsnames
Nstat = len(stationsnames)
refstation = stationsnames[refstationi]
phase_ref = soldict['CommonScalarPhase:{s}'.format(s=refstation)]['values']
times = soldict['CommonScalarPhase:{s}'.format(s=refstation)]['times']
Nr = int(np.ceil(np.sqrt(Nstat)))
Nc = int(np.ceil(np.float(Nstat) / Nr))
f, ax = plt.subplots(Nr, Nc, sharex=True, sharey=True, figsize=(16, 12))
plt.subplots_adjust(left=0.05,
bottom=0.05,
right=0.95,
top=0.95,
wspace=0,
hspace=0)
axs = ax.reshape((Nr * Nc, 1))
for istat, station in enumerate(stationsnames):
phase = soldict['CommonScalarPhase:{s}'.format(s=station)]['values']
# don't plot flagged phases
phase = np.ma.masked_where(phase == 0, phase)
#try:
#if len(phase11) > 1000:
#fmt = ','
#else:
#fmt = '.'
#except TypeError:
#print "no phases"
fmt = '.'
ls = 'none'
axs[istat][0].plot(times,
normalize(phase - phase_ref),
color='b',
marker=fmt,
ls=ls,
label='CommonScalarPhase',
mec='b')
axs[istat][0].set_ylim(-3.2, 3.2)
axs[istat][0].set_xlim(times.min(), times.max())
#axs[istat][0].set_title(station)
axs[istat][0].text(0.5,
0.9,
station,
horizontalalignment='center',
verticalalignment='center',
transform=axs[istat][0].transAxes,
backgroundcolor='white',
color='k')
f.savefig(imageroot + "_scalarphase.png", dpi=100)
return
def main(input_mslist, parmdb_name, outparmdb, clobber=True):
"""
Merges parmdbs in time into a single parmdb
The parmdbs are assumed to be located in the input MS with name
parmdb_name
Parameters
----------
input_mslist : list
List of input MS file names
parmdb_name : str
Name of parmdb (relative to MS files)
outparmdb : str
Name of output merged parmdb
clobber : bool, optional
If True, overwrite existing output file
"""
if type(input_mslist) is str:
input_mslist = input_mslist.strip('[]').split(',')
input_mslist = [f.strip() for f in input_mslist]
inparmdbs = [os.path.join(ms, parmdb_name) for ms in input_mslist]
# Sort the parmdbs by start time (needed for the timewidth checks below)
if len(inparmdbs) > 1:
start_times = []
start_times_dicts = [{} for i in range(len(inparmdbs))]
for i, inparmdb in enumerate(inparmdbs):
pdb_in = pdb.parmdb(inparmdb)
for j, parmname in enumerate(pdb_in.getNames()):
parms = pdb_in.getValuesGrid(parmname)
if j == 0:
start_times.append(parms[parmname]['times'][0])
start_times_dicts[i].update({parmname: parms[parmname]['times'][0]})
pdb_in = False
inparmdbs = np.array(inparmdbs)[np.argsort(start_times)].tolist()
start_times_dicts = np.array(start_times_dicts)[np.argsort(start_times)].tolist()
start_times.sort()
if type(clobber) is str:
if clobber.lower() == 'true':
clobber = True
else:
clobber = False
if os.path.exists(outparmdb):
if clobber:
os.system('rm -rf {0}'.format(outparmdb))
else:
return
pdb_concat = pdb.parmdb(outparmdb, create=True)
for i, inparmdb in enumerate(inparmdbs):
pdb_add = pdb.parmdb(inparmdb)
parms = pdb_add.getValuesGrid('*')
for parmname in pdb_add.getNames():
# Adjust last timewidth if necessary, as DPPP GainCal (as of 2.16.4) does not
# truncate last solution timewidth to end of MS
if i < len(inparmdbs) - 1:
inter_chunk_timewidth = start_times_dicts[i+1][parmname] - parms[parmname]['times'][-1]
if inter_chunk_timewidth < parms[parmname]['timewidths'][-1]:
parms[parmname]['timewidths'][-1] = inter_chunk_timewidth
# Adjust following chunk as well
pdb_next = pdb.parmdb(inparmdbs[i+1])
parms_next = pdb_next.getValuesGrid(parmname)
parms_next[parmname]['timewidths'][0] = inter_chunk_timewidth
pdb_next.deleteValues(parmname)
pdb_next.addValues(parmname, parms_next[parmname])
pdb_next.flush()
pdb_next = False
ValueHolder = pdb_concat.makeValue(values=parms[parmname]['values'],
sfreq=parms[parmname]['freqs'],
efreq=parms[parmname]['freqwidths'],
stime=parms[parmname]['times'],
etime=parms[parmname]['timewidths'],
asStartEnd=False)
pdb_concat.addValues(parmname, ValueHolder)
pdb_concat.flush()
pdb_add = False
pdb_concat = False
def solplot_amp(parmdb,
imageroot,
refstationi,
norm_amp_lim=False,
median_amp=False,
plot_international=False,
stations=[]):
parmdbmtable = lp.parmdb(parmdb)
soldict = parmdbmtable.getValuesGrid('*')
names = parmdbmtable.getNames()
'Gain:1:1:Phase:RS508HBA'
stationsnames = np.array([name.split(':')[-1] for name in names])
stationsnames = np.unique(stationsnames)
if not plot_international:
stationsnames = np.array(
[name for name in stationsnames if name[0] in ['C', 'R']])
# specify list of stations (none for all)#
if len(stations) > 0:
selstationsnames = []
for s in stations:
if s in stationsnames:
selstationsnames.append(s)
else:
print "invalid station {s}".format(s=s)
stationsnames = selstationsnames
Nstat = len(stationsnames)
refstation = stationsnames[refstationi]
times = soldict['Gain:1:1:Real:{s}'.format(s=refstation)]['times']
times = scaletimes(times)
real11_ref = soldict['Gain:1:1:Real:{s}'.format(s=refstation)]['values']
real00_ref = soldict['Gain:0:0:Real:{s}'.format(s=refstation)]['values']
imag11_ref = soldict['Gain:1:1:Imag:{s}'.format(s=refstation)]['values']
imag00_ref = soldict['Gain:0:0:Imag:{s}'.format(s=refstation)]['values']
valscorr00 = real00_ref + 1.j * imag00_ref
valscorr11 = real11_ref + 1.j * imag11_ref
amp00_ref = np.abs(valscorr00)
amp11_ref = np.abs(valscorr11)
Nr = int(np.ceil(np.sqrt(Nstat)))
Nc = int(np.ceil(np.float(Nstat) / Nr))
fa, axa = plt.subplots(Nr, Nc, sharex=True, sharey=True, figsize=(16, 12))
plt.subplots_adjust(left=0.05,
bottom=0.05,
right=0.95,
top=0.95,
wspace=0,
hspace=0)
axsa = axa.reshape((Nr * Nc, 1))
ymin = 2
ymax = 0
for istat, station in enumerate(stationsnames):
real11 = soldict['Gain:1:1:Real:{s}'.format(s=station)]['values']
real00 = soldict['Gain:0:0:Real:{s}'.format(s=station)]['values']
imag11 = soldict['Gain:1:1:Imag:{s}'.format(s=station)]['values']
imag00 = soldict['Gain:0:0:Imag:{s}'.format(s=station)]['values']
valscorr00 = real00 + 1.j * imag00
valscorr11 = real11 + 1.j * imag11
#if len(valscorr11) > 1000:
#fmt = ','
#else:
#fmt = '.'
if len(np.unique(real11)) > 500:
fmt = ','
else:
fmt = '.'
ls = 'none'
amp00 = np.abs(valscorr00)
amp11 = np.abs(valscorr11)
#phase11 = soldict['Gain:1:1:Phase:{s}'.format(s=station)]
#phase00 = soldict['Gain:0:0:Phase:{s}'.format(s=station)]
## for y scale: check max and min values
amp00m = np.ma.masked_where(amp00 == 1, amp00).compressed()
amp11m = np.ma.masked_where(amp11 == 1, amp11).compressed()
if len(amp00m) > 0:
ymax = max(np.max(amp00m), ymax)
if len(amp11m) > 0:
ymax = max(np.max(amp11m), ymax)
if len(amp00m) > 0:
ymin = min(np.min(amp00m), ymin)
if len(amp11m) > 0:
ymin = min(np.min(amp11m), ymin)
# don't plot flagged amplitudes
amp00 = np.ma.masked_where(amp00 == 1, amp00)
amp11 = np.ma.masked_where(amp11 == 1, amp11)
axsa[istat][0].plot(times,
amp00,
color='b',
marker=fmt,
ls=ls,
label='Gain:0:0:Amp',
mec='b')
axsa[istat][0].plot(times,
amp11,
color='g',
marker=fmt,
ls=ls,
label='Gain:1:1:Amp',
mec='g')
if median_amp:
median_amp00 = np.median(amp00)
median_amp11 = np.median(amp11)
#if abs(median_amp00) < 1.0e-10:
#if np.sum(amp00==median_amp00) != len(amp00):
#amp00 = np.ma.masked_where(amp00==1, amp00).compressed()
#median_amp00 = np.median(amp00)
#if abs(median_amp11) < 1.0e-10:
#if np.sum(amp11==median_amp11) != len(amp11):
#amp11 = np.ma.masked_where(amp11==1, amp11).compressed()
#median_amp11 = np.median(amp11)
#print median_amp00,median_amp11
axsa[istat][0].plot([times[0], times[-1]],
[median_amp00, median_amp00],
color='b',
label='<Gain:0:0:Amp>')
axsa[istat][0].plot([times[0], times[-1]],
[median_amp11, median_amp11],
color='g',
label='<Gain:1:1:Amp>')
if norm_amp_lim:
axsa[istat][0].set_ylim(0, 2)
else:
axsa[istat][0].set_ylim(ymin, ymax)
#print istat, station,ymin, ymax
axsa[istat][0].set_xlim(times.min(), times.max())
#axsa[istat][0].set_title(station)
axsa[istat][0].text(0.5,
0.9,
station,
horizontalalignment='center',
verticalalignment='center',
transform=axsa[istat][0].transAxes,
backgroundcolor='white',
color='k')
fa.savefig(imageroot + "_amp.png", dpi=100)
return
def main(parmdb_p, parmdb_a, parmdb_out, clobber=True, scratch_dir=None):
"""
Merges facet selfcal parmdbs into a single parmdb
Parameters
----------
parmdb_p : str
Filename of CommonScalarPhase and TEC parmdb
parmdb_a : str
Filename of Gain parmdb. The nearset match in frequency to that of the
input band will be used
parmdb_out : str
Filename of output file
clobber : bool, optional
If True, overwrite existing output file
scratch_dir : str, optional
Scratch directory for temp storage
"""
if type(clobber) is str:
if clobber.lower() == 'true':
clobber = True
else:
clobber = False
if os.path.exists(parmdb_out):
if clobber:
shutil.rmtree(parmdb_out)
else:
return
# Copy to scratch directory if specified
if scratch_dir is not None:
parmdb_p_orig = parmdb_p
parmdb_p = os.path.join(scratch_dir, os.path.basename(parmdb_p_orig))
parmdb_a_orig = parmdb_a
parmdb_a = os.path.join(scratch_dir, os.path.basename(parmdb_a_orig))
parmdb_out_orig = parmdb_out
parmdb_out = os.path.join(scratch_dir, os.path.basename(parmdb_out_orig))
shutil.copytree(parmdb_p_orig, parmdb_p)
shutil.copytree(parmdb_a_orig, parmdb_a)
shutil.copytree(parmdb_p, parmdb_out)
## Copy over the Gains
pdb_out = pdb.parmdb(parmdb_out)
pdb_a = pdb.parmdb(parmdb_a)
parms = pdb_a.getValuesGrid('*')
for parmname in pdb_a.getNames():
# Set flagged solutions to NaN
flagged = np.where(np.logical_or(parms[parmname]['values'] == 0.0,
np.isnan(parms[parmname]['values'])))
parms[parmname]['values'][flagged] = np.nan
ValueHolder = pdb_out.makeValue(values=parms[parmname]['values'],
sfreq=parms[parmname]['freqs'],
efreq=parms[parmname]['freqwidths'],
stime=parms[parmname]['times'],
etime=parms[parmname]['timewidths'],
asStartEnd=False)
pdb_out.addValues(parmname,ValueHolder)
pdb_out.flush()
# Copy output to original path and delete copies if scratch directory is specified
if scratch_dir is not None:
shutil.copytree(parmdb_out, parmdb_out_orig)
shutil.rmtree(parmdb_out)
shutil.rmtree(parmdb_p)
shutil.rmtree(parmdb_a)
def solplot_phase_phasors(parmdb, imageroot, refstationi, plot_international=False, fourpol=False):
parmdbmtable = lp.parmdb(parmdb)
soldict = parmdbmtable.getValuesGrid('*')
names = parmdbmtable.getNames()
'Gain:1:1:Phase:RS508HBA'
stationsnames = np.array([name.split(':')[-1] for name in names])
stationsnames = np.unique(stationsnames)
if not plot_international:
stationsnames = np.array([name for name in stationsnames if name[0] in ['C','R'] ])
Nstat = len(stationsnames)
refstation = stationsnames[refstationi]
phase11_ref = soldict['Gain:1:1:Phase:{s}'.format(s=refstation)]['values']
phase00_ref = soldict['Gain:0:0:Phase:{s}'.format(s=refstation)]['values']
if fourpol:
phase10_ref = soldict['Gain:1:0:Phase:{s}'.format(s=refstation)]['values']
phase01_ref = soldict['Gain:0:1:Phase:{s}'.format(s=refstation)]['values']
times= soldict['Gain:1:1:Phase:{s}'.format(s=refstation)]['times']
num_channels = phase11_ref.shape[1]
Nr = int(np.ceil(np.sqrt(Nstat)))
Nc = int(np.ceil(np.float(Nstat)/Nr))
for chan_indx in range(num_channels):
f, ax = plt.subplots(Nr, Nc, sharex=True, sharey=True, figsize=(16,12))
axs = ax.reshape((Nr*Nc,1))
for istat, station in enumerate(stationsnames):
phase11 = soldict['Gain:1:1:Phase:{s}'.format(s=station)]['values'][:, chan_indx]
phase00 = soldict['Gain:0:0:Phase:{s}'.format(s=station)]['values'][:, chan_indx]
phase00_ref_chan = phase00_ref[:, chan_indx]
phase11_ref_chan = phase11_ref[:, chan_indx]
# don't plot flagged phases
phase00 = np.ma.masked_where(phase00==0, phase00)
phase11 = np.ma.masked_where(phase11==0, phase11)
if fourpol:
phase10 = soldict['Gain:1:0:Phase:{s}'.format(s=station)]['values'][:, chan_indx]
phase01 = soldict['Gain:0:1:Phase:{s}'.format(s=station)]['values'][:, chan_indx]
phase01_ref_chan = phase01_ref[:, chan_indx]
phase10_ref_chan = phase10_ref[:, chan_indx]
# don't plot flagged phases
phase01 = np.ma.masked_where(phase01==0, phase01)
phase10 = np.ma.masked_where(phase10==0, phase10)
if len(times) > 1000:
fmt = ','
else:
fmt = '.'
ls='none'
if fourpol:
axs[istat][0].plot(times, normalize(phase01-phase01_ref_chan), color='orange', marker=fmt, ls=ls, label='Gain:0:1:Phase',mec='orange')
axs[istat][0].plot(times, normalize(phase10-phase10_ref_chan), color='red', marker=fmt, ls=ls, label='Gain:1:0:Phase',mec='red')
axs[istat][0].plot(times, normalize(phase00-phase00_ref_chan), color='b', marker=fmt, ls=ls, label='Gain:0:0:Phase',mec='b')
axs[istat][0].plot(times, normalize(phase11-phase11_ref_chan), color='g', marker=fmt, ls=ls, label='Gain:1:1:Phase',mec='g')
axs[istat][0].set_ylim(-3.2, 3.2)
axs[istat][0].set_xlim(times.min(), times.max())
axs[istat][0].set_title(station)
f.savefig(imageroot+"_phase_channel{}.png".format(chan_indx),dpi=100)
plt.close(f)
parmdbmtable = False
del(soldict)
def main(opts,args):
# Get the min and max y axis values
l = opts.amplim.split(',')
if l[0] == '': ampmin = None
else: ampmin = float(l[0])
if l[1] == '': ampmax = None
else: ampmax = float(l[1])
directionmode=opts.directional
if directionmode:
if opts.directionsource=="":
print "Directional Mode chosen but no source selected! Use the -S option."
exit()
# Open the parmdb
p = pdb.parmdb(args[0])
if opts.title != '':
title = opts.title
else:
title = args[0]
ofile = None
if opts.stats != '':
if os.path.isfile(opts.stats):
print 'Error: ' + opts.stats + ' already exists'
exit()
ofile = open(opts.stats, 'w')
ofile.write(OFILE_PATTERN % ('#station', 'type:corr', 'num', 'mean', 'median', 'std'))
# Get names (of the params) that are related to the selectes stations
if directionmode:
names = p.getNames('DirectionalGain*:%s'%(opts.stations))
ptype= DDGAIN_TYPE
source=opts.directionsource
else:
names = p.getNames('*:%s'%(opts.stations))
ptype = GAIN_TYPE
if not len(names): print '\nNo stations matched the filter'
# ptype = GAIN_TYPE
corrSet = set([])
stationSet = set([])
cordSet = set([])
sourceSet = set([])
# We assume that at least
for name in names:
# print name
sname = name.split(':')
# Add the station and correlation to its sets
corrSet.add(sname[1] + ':' + sname[2])
stationSet.add(sname[4])
cordSet.add(sname[3])
# if directionmode:
# sourceSet.add(sname[5])
# Check that we are handling only Gain or TEC cases
if sname[0] == TEC_TYPE: ptype=TEC_TYPE
# if sname[0] == DDGAIN_TYPE: ptype=DDGAIN_TYPE
elif sname[0] != GAIN_TYPE and sname[0]!= DDGAIN_TYPE:
print 'Error: parmdb is not '+GAIN_TYPE+' or '+TEC_TYPE+' or '+DDGAIN_TYPE
exit()
# We get if data is in polar format (preferable)
if AMPL_COORD in cordSet or PHASE_COORD in cordSet:
polar = True
elif REAL_COORD in cordSet or IMAG_COORD in cordSet:
polar = False
else:
print 'Error: paramdb contains ' + ','.join(cordSet)
exit()
# Check the found correlations: autocorrelations (0:0 and 1:1) are plotted always,
# crosscorrelations are plotted only if the users asks for itand
for corr in AUTO_CORR:
if corr not in corrSet:
print 'Warning: ' + corr + ' is not found in the paramdb'
for corr in CROSS_CORR:
if corr in corrSet:
if not opts.allpol:
print 'Warning: ' + corr + ' is found in paramdb but will be ignored'
corrSet.remove(corr)
elif opts.allpol:
print 'Warning: ' + corr + ' was asked but is not found in paramdb'
if not len(corrSet):
print 'Error: no gain correlations to plot'
exit()
# Make lists of the set objects
stations = sorted(list(stationSet))
cords = list(cordSet)
corrs = sorted(list(corrSet))
# sources = sorted(list(sourceSet))
# Get the reference stations
if opts.reference == '':
refstation = stations[0]
else:
refstation = opts.reference
print 'Using %s for reference station'%(refstation)
# Get ampltude and phase data for the reference station
(refamp,refphase,reftimes,averaged) = getAmplPhase(p, refstation, ptype, corrs , polar, cords, opts.last, source)
if averaged:
print 'Warning: data is multi-channel. It is averaged to single channel for plotting purposes (this may also corrupt the statistics)'
# This color code will be used in titles
colorCode = ''
for corr in corrs:
colorCode += CORR_COLOR_NAME[corr] + '=' + ptype + corr + ','
colorCode = colorCode[:-1]
if opts.quicklook:
params = {'axes.labelsize': 6,
'text.fontsize': 6,
'legend.fontsize': 6,
'xtick.labelsize': 6,
'ytick.labelsize': 6,
'figure.figsize': (8.27,11.69)}
pylab.rcParams.update(params)
ny = int(float(len(stations))/2.+0.5)
pltamp = None
pltphs = None
deffigsize = (10.,6.)
# Create figures and set the titles of the plots (they depend on the plotted correlations)
if isValid(refamp):
pltamp = pylab.figure(figsize=deffigsize)
pylab.suptitle((title + ' Amplitude (' + colorCode + '), Ref = ' + refstation),fontsize=8)
if isValid(refphase):
pltphs = pylab.figure(figsize=deffigsize)
pylab.suptitle((title + ' Phase (' + colorCode + '), Ref = ' + refstation),fontsize=8)
c = 0 # subplot counter
for station in stations:
c += 1
# Get the data for this station
(valsamp,valsphase,times,averaged) = getAmplPhase(p, station, ptype, corrs , polar, cords, opts.last, source)
fig = None
if isValid(valsamp):
# Create the sub-plot for the amplitude
if opts.quicklook:
pylab.figure(num=pltamp.number)
pylab.subplot(ny,2,c)
else:
pylab.figure()
pylab.subplot(211)
pylab.suptitle('%s, Ref = %s'%(title,refstation))
# Set the arguments for the pylab plot
# And compute statistics if necessary
plotArgs = []
medians = {}
for corr in corrs:
valscorramp = valsamp[corr]
plotArgs.extend([times,valscorramp,CORR_COLOR_CODE[corr]+'.'])
if ofile != None:
numSamples = len(valscorramp)
mean = pylab.mean(valscorramp)
median = pylab.median(valscorramp)
medians[corr] = median
std = pylab.std(valscorramp)
ofile.write(OFILE_PATTERN % (station, (ptype+':'+corr), ('%d'%numSamples),('%.3f'% mean), ('%.3f'% median), ('%.3f'% std)))
#Flush the stats file just in case the script is finished unexpectacly
if ofile != None:
ofile.flush()
# Launch the pylab plotter por the amplitude for the selected correlations
pylab.plot(*tuple(plotArgs))
if opts.median:
# Plot the median
for corr in corrs:
if len(medians):
median = medians[corr]
else:
median = pylab.median(valsamp[corr])
pylab.plot([times[0],times[-1]],[median,median],CORR_COLOR_CODE[corr]+'-')
# Set labels
if opts.quicklook:
pylab.ylabel(station,rotation='horizontal')
if c<len(stations)-1: pylab.gca().set_xticklabels([])
else: pylab.xlabel('Time [sec]')
else:
pylab.ylabel('Amplitude')
pylab.gca().set_xticklabels([])
pylab.title(station + colorCode)
pylab.subplots_adjust(hspace=0.)
# Set min and maximum values of amplitude plot (if provided)
if ampmin is not None: pylab.ylim(ymin=ampmin)
if ampmax is not None: pylab.ylim(ymax=ampmax)
if isValid(valsphase):
# Create the sub-plot for the phase (not for TEC, there is no phase)
if opts.quicklook:
pylab.figure(num=pltphs.number)
pylab.subplot(ny,2,c)
else:
if fig==None:
# No figure was created in the AMPL
pylab.figure()
pylab.subplot(111)
pylab.suptitle('%s, Ref = %s'%(title,refstation))
else:
pylab.subplot(212)
# Set the arguments for the pylab plot
plotArgs = []
for corr in corrs:
plotArgs.extend([times,((valsphase[corr]-refphase[corr]+numpy.pi)%(2.*numpy.pi)-numpy.pi),CORR_COLOR_CODE[corr]+'.'])
pylab.plot(*tuple(plotArgs))
# Set labels
if opts.quicklook:
pylab.ylabel(station,rotation='horizontal')
if c<len(stations)-1: pylab.gca().set_xticklabels([])
else: pylab.xlabel('Time [sec]')
else:
pylab.ylabel('Phase [rad]')
pylab.xlabel('Time [sec]')
pylab.ylim(ymin=-numpy.pi,ymax=numpy.pi)
pylab.subplots_adjust(hspace=0.)
# Show the plot (if not quicklook or displayall) or save it in a file
print 'Plotting %s'%(station)
if opts.out == '' and not opts.quicklook and not opts.displayall:
print 'Close figure to proceed'
pylab.show()
elif not opts.quicklook and opts.out != '':
print '-> %s'%(opts.out+'-%s.%s'%(station,opts.ext))
pylab.savefig(opts.out+'-%s.%s'%(station,opts.ext))
# If we are in quicklook mode and no files have been generated we show all the plots or save tgem now
if (opts.quicklook or opts.displayall) and opts.out == '':
pylab.show()
elif opts.quicklook:
if pltamp != None:
pylab.figure(num=pltamp.number)
print '-> '+opts.out+'-amp.%s'%(opts.ext)
pylab.savefig(opts.out+'-amp.%s'%(opts.ext))
if pltphs != None:
pylab.figure(num=pltphs.number)
print '-> '+opts.out+'-phs.%s'%(opts.ext)
pylab.savefig(opts.out+'-phs.%s'%(opts.ext))
p = 0 # close the parmdb
# Close the stats file if it is present
if ofile != None:
print '-> %s'%(opts.stats)
ofile.close()
def solplot_ampphase(parmdb, imageroot, refstationi, norm_amp_lim=False, median_amp=False, plot_international=False):
parmdbmtable = lp.parmdb(parmdb)
soldict = parmdbmtable.getValuesGrid('*')
names = parmdbmtable.getNames()
'Gain:1:1:Phase:RS508HBA'
stationsnames = np.array([name.split(':')[-1] for name in names])
stationsnames = np.unique(stationsnames)
if not plot_international:
stationsnames = np.array([name for name in stationsnames if name[0] in ['C','R'] ])
Nstat = len(stationsnames)
refstation = stationsnames[refstationi]
times = soldict['Gain:1:1:Real:{s}'.format(s=refstation)]['times']
real11_ref = soldict['Gain:1:1:Real:{s}'.format(s=refstation)]['values']
real00_ref = soldict['Gain:0:0:Real:{s}'.format(s=refstation)]['values']
imag11_ref = soldict['Gain:1:1:Imag:{s}'.format(s=refstation)]['values']
imag00_ref = soldict['Gain:0:0:Imag:{s}'.format(s=refstation)]['values']
valscorr00 = real00_ref +1.j*imag00_ref
valscorr11 = real11_ref +1.j*imag11_ref
amp00_ref = np.abs(valscorr00)
phase00_ref = np.angle(valscorr00)
amp11_ref = np.abs(valscorr11)
phase11_ref = np.angle(valscorr11)
Nr = int(np.ceil(np.sqrt(Nstat)))
Nc = int(np.ceil(np.float(Nstat)/Nr))
fp, axp = plt.subplots(Nr, Nc, sharex=True, sharey=True, figsize=(16,12))
axsp = axp.reshape((Nr*Nc,1))
fa, axa = plt.subplots(Nr, Nc, sharex=True, sharey=True, figsize=(16,12))
axsa = axa.reshape((Nr*Nc,1))
ymin = 2
ymax = 0
for istat, station in enumerate(stationsnames):
real11 = soldict['Gain:1:1:Real:{s}'.format(s=station)]['values']
real00 = soldict['Gain:0:0:Real:{s}'.format(s=station)]['values']
imag11 = soldict['Gain:1:1:Imag:{s}'.format(s=station)]['values']
imag00 = soldict['Gain:0:0:Imag:{s}'.format(s=station)]['values']
valscorr00 = real00 +1.j*imag00
valscorr11 = real11 +1.j*imag11
if len(valscorr11) > 1000:
fmt = ','
else:
fmt = '.'
amp00 = np.abs(valscorr00)
phase00 = np.angle(valscorr00)
amp11 = np.abs(valscorr11)
phase11 = np.angle(valscorr11)
#phase11 = soldict['Gain:1:1:Phase:{s}'.format(s=station)]
#phase00 = soldict['Gain:0:0:Phase:{s}'.format(s=station)]
## for y scale: check max and min values
amp00m = np.ma.masked_where(amp00==1, amp00).compressed()
amp11m = np.ma.masked_where(amp11==1, amp11).compressed()
if len(amp00m) > 0:
ymax = max(np.max(amp00m),ymax)
if len(amp11m) > 0:
ymax = max(np.max(amp11m),ymax)
if len(amp00m) > 0:
ymin = min(np.min(amp00m),ymin)
if len(amp11m) > 0:
ymin = min(np.min(amp11m),ymin)
# don't plot flagged amplitudes
amp00 = np.ma.masked_where(amp00==1, amp00)
amp11 = np.ma.masked_where(amp11==1, amp11)
# don't plot flagged phases
phase00 = np.ma.masked_where(phase00==0, phase00)
phase11 = np.ma.masked_where(phase11==0, phase11)
axsp[istat][0].plot(times, normalize(phase00-phase00_ref), color='b', marker=fmt, label='Gain:0:0:Phase')
axsp[istat][0].plot(times, normalize(phase11-phase11_ref), color='g', marker=fmt, label='Gain:1:1:Phase')
axsp[istat][0].set_ylim(-3.2, 3.2)
axsp[istat][0].set_xlim(times.min(), times.max())
axsp[istat][0].set_title(station)
axsa[istat][0].plot(times, amp00, color='b', marker=fmt, label='Gain:0:0:Amp')
axsa[istat][0].plot(times, amp11, color='g', marker=fmt, label='Gain:1:1:Amp')
if median_amp:
median_amp00 = np.median(amp00)
median_amp11 = np.median(amp11)
#if abs(median_amp00) < 1.0e-10:
#if np.sum(amp00==median_amp00) != len(amp00):
#amp00 = np.ma.masked_where(amp00==1, amp00).compressed()
#median_amp00 = np.median(amp00)
#if abs(median_amp11) < 1.0e-10:
#if np.sum(amp11==median_amp11) != len(amp11):
#amp11 = np.ma.masked_where(amp11==1, amp11).compressed()
#median_amp11 = np.median(amp11)
#print median_amp00,median_amp11
axsa[istat][0].plot([times[0], times[-1]], [median_amp00,median_amp00], color='b', label='<Gain:0:0:Amp>')
axsa[istat][0].plot([times[0], times[-1]], [median_amp11,median_amp11], color='g', label='<Gain:1:1:Amp>')
if norm_amp_lim:
axsa[istat][0].set_ylim(0,2 )
else:
axsa[istat][0].set_ylim(ymin,ymax)
#print istat, station,ymin, ymax
axsa[istat][0].set_xlim(times.min(), times.max())
axsa[istat][0].set_title(station)
fp.savefig(imageroot+"_phase.png",dpi=100)
fa.savefig(imageroot+"_amp.png",dpi=100)
return
def solplot_amp(parmdb, imageroot, refstationi, norm_amp_lim=False, median_amp=False, plot_international=False):
parmdbmtable = lp.parmdb(parmdb)
soldict = parmdbmtable.getValuesGrid('*')
names = parmdbmtable.getNames()
'Gain:1:1:Phase:RS508HBA'
stationsnames = np.array([name.split(':')[-1] for name in names])
stationsnames = np.unique(stationsnames)
if not plot_international:
stationsnames = np.array([name for name in stationsnames if name[0] in ['C','R'] ])
Nstat = len(stationsnames)
refstation = stationsnames[refstationi]
times = soldict['Gain:1:1:Real:{s}'.format(s=refstation)]['times']
times = scaletimes(times)
real11_ref = soldict['Gain:1:1:Real:{s}'.format(s=refstation)]['values']
real00_ref = soldict['Gain:0:0:Real:{s}'.format(s=refstation)]['values']
imag11_ref = soldict['Gain:1:1:Imag:{s}'.format(s=refstation)]['values']
imag00_ref = soldict['Gain:0:0:Imag:{s}'.format(s=refstation)]['values']
num_channels = real11_ref.shape[1]
valscorr00 = real00_ref +1.j*imag00_ref
valscorr11 = real11_ref +1.j*imag11_ref
amp00_ref = np.abs(valscorr00)
amp11_ref = np.abs(valscorr11)
Nr = int(np.ceil(np.sqrt(Nstat)))
Nc = int(np.ceil(np.float(Nstat)/Nr))
for chan_indx in range(num_channels):
fa, axa = plt.subplots(Nr, Nc, sharex=True, sharey=True, figsize=(16,12))
axsa = axa.reshape((Nr*Nc,1))
ymin = 2
ymax = 0
for istat, station in enumerate(stationsnames):
real11 = soldict['Gain:1:1:Real:{s}'.format(s=station)]['values'][:, chan_indx]
real00 = soldict['Gain:0:0:Real:{s}'.format(s=station)]['values'][:, chan_indx]
imag11 = soldict['Gain:1:1:Imag:{s}'.format(s=station)]['values'][:, chan_indx]
imag00 = soldict['Gain:0:0:Imag:{s}'.format(s=station)]['values'][:, chan_indx]
valscorr00 = real00 +1.j*imag00
valscorr11 = real11 +1.j*imag11
amp00_ref_chan = amp00_ref[:, chan_indx]
amp11_ref_chan = amp11_ref[:, chan_indx]
if len(np.unique(real11)) > 500:
fmt = ','
else:
fmt = '.'
ls='none'
amp00 = np.abs(valscorr00)
amp11 = np.abs(valscorr11)
## for y scale: check max and min values
amp00m = np.ma.masked_where(amp00==1, amp00).compressed()
amp11m = np.ma.masked_where(amp11==1, amp11).compressed()
if len(amp00m) > 0:
ymax = max(np.max(amp00m),ymax)
if len(amp11m) > 0:
ymax = max(np.max(amp11m),ymax)
if len(amp00m) > 0:
ymin = min(np.min(amp00m),ymin)
if len(amp11m) > 0:
ymin = min(np.min(amp11m),ymin)
# don't plot flagged amplitudes
amp00 = np.ma.masked_where(amp00==1, amp00)
amp11 = np.ma.masked_where(amp11==1, amp11)
axsa[istat][0].plot(times, amp00, color='b', marker=fmt, ls=ls, label='Gain:0:0:Amp',mec='b')
axsa[istat][0].plot(times, amp11, color='g', marker=fmt, ls=ls, label='Gain:1:1:Amp',mec='g')
if median_amp:
median_amp00 = np.median(amp00)
median_amp11 = np.median(amp11)
axsa[istat][0].plot([times[0], times[-1]], [median_amp00,median_amp00], color='b', label='<Gain:0:0:Amp>')
axsa[istat][0].plot([times[0], times[-1]], [median_amp11,median_amp11], color='g', label='<Gain:1:1:Amp>')
if norm_amp_lim:
axsa[istat][0].set_ylim(0,2 )
else:
axsa[istat][0].set_ylim(ymin,ymax)
axsa[istat][0].set_xlim(times.min(), times.max())
axsa[istat][0].set_title(station)
fa.savefig(imageroot+"_amp_channel{}.png".format(chan_indx),dpi=100)
plt.close(fa)
parmdbmtable = False
del(soldict)
import os
if len(sys.argv) < 2:
print 'Please give a parmdb name.'
print 'Usage: python fixparmdb_zero_median.py <parmdbname>'
filename = sys.argv[1]
outfilename = sys.argv[2]
if os.path.isdir(outfilename):
print 'output file exists'
sys.exit()
os.system('cp -r %s %s' %(filename, outfilename))
filename = outfilename
parmdbmtable = lp.parmdb(filename)
dictionary = parmdbmtable.getValuesGrid('*')
real_names = parmdbmtable.getNames('*Real*')
imaginary_names = parmdbmtable.getNames('*Imag*')
names = parmdbmtable.getNames()
for name in real_names:
imaginary_name = name.replace('Real','Imag')
real_values = dictionary[name]['values']
imaginary_values = dictionary[imaginary_name]['values']
new_real_values = numpy.sqrt(real_values**2. + imaginary_values**2.)
#new_imaginary_values = numpy.zeros(dictionary[imaginary_name]['values'].shape)
new_real_value = numpy.median(new_real_values)
def main(input_mslist, parmdb_name, outparmdb, clobber=True):
"""
Merges parmdbs in time into a single parmdb
The parmdbs are assumed to be located in the input MS with name
parmdb_name
Parameters
----------
input_mslist : list
List of input MS file names
parmdb_name : str
Name of parmdb (relative to MS files)
outparmdb : str
Name of output merged parmdb
clobber : bool, optional
If True, overwrite existing output file
"""
if type(input_mslist) is str:
input_mslist = input_mslist.strip('[]').split(',')
input_mslist = [f.strip() for f in input_mslist]
inparmdbs = [os.path.join(ms, parmdb_name) for ms in input_mslist]
if type(clobber) is str:
if clobber.lower() == 'true':
clobber = True
else:
clobber = False
if os.path.exists(outparmdb):
if clobber:
os.system('rm -rf {0}'.format(outparmdb))
else:
return
os.system('cp -r {0} {1}'.format(inparmdbs[0], outparmdb))
### old copy, which puts all values into the same domain (i.e. ignoring the timestamps)
# if len(inparmdbs) > 1:
# pdb_concat = lofar.parmdb.parmdb(outparmdb)
# for inparmdb in inparmdbs[1:]:
# pdb = lofar.parmdb.parmdb(inparmdb)
# for parmname in pdb.getNames():
# v = pdb.getValuesGrid(parmname)
# pdb_concat.addValues(v.copy())
# pdb_concat.flush()
### version with TAQL, which avoids the previous problem, but may introduce new ones
# if len(inparmdbs) > 1:
# for inparmdb in inparmdbs[1:]:
# pt.taql('insert into '+outparmdb+' select from '+inparmdb)
# outdb = pt.table(outparmdb,readonly=False,ack=False)
# errdesc = outdb.getcoldesc('ERRORS')
# errdesc['name'] = 'ERRORS'
# outdb.removecols('ERRORS')
# outdb.addcols(errdesc)
# outdb.close()
### version which adds new data by generating a new data-holder first
if len(inparmdbs) > 1:
pdb_concat = pdb.parmdb(outparmdb)
for inparmdb in inparmdbs[1:]:
pdb_add = pdb.parmdb(inparmdb)
for parmname in pdb_add.getNames():
parms = pdb_add.getValuesGrid(parmname)
ValueHolder = pdb_concat.makeValue(values=parms[parmname]['values'],
sfreq=parms[parmname]['freqs'],
efreq=parms[parmname]['freqwidths'],
stime=parms[parmname]['times'],
etime=parms[parmname]['timewidths'],
asStartEnd=False)
pdb_concat.addValues(parmname,ValueHolder)
pdb_concat.flush()
pdb_add = False
pdb_concat = False
def getStructure(pdname,antennas,nr_grid=1,doplot=True,outbasename='ionosphere',dofilter=True):
outfile = open(outbasename +'_structure.txt','w')
myt=pd.parmdb(pdname)
val=myt.getValuesGrid("*")
stat=pt.table(antennas)
names=stat.getcol('NAME')
pos=stat.getcol('POSITION')
stations={}
for i,j in zip(names,pos):
stations[i]=j
phx=[]
phy=[]
allposx=[]
allposy=[]
nrtimes=val[val.keys()[0]]['times'].shape[0]
timestep=int(nrtimes/nr_grid)
for i in sorted(val.keys()):
if not "CS" in i:
continue
#if 'CS201' in i:
# continue
#if 'CS030' in i:
# continue
if "0:0" in i:
phx.append(val[i]['values'].flatten())
allposx.append(stations[i.split(":")[-1]])
if "1:1" in i:
phy.append(val[i]['values'].flatten())
allposy.append(stations[i.split(":")[-1]])
phx=np.array(phx)
phy=np.array(phy)
allposx=np.array(allposx)
D=allposx[:,np.newaxis,:]-allposx[np.newaxis,:]
D2=np.sqrt(np.sum(D**2,axis=-1))
allposy=np.array(allposy)
Dy=allposy[:,np.newaxis,:]-allposy[np.newaxis,:]
D2y=np.sqrt(np.sum(Dy**2,axis=-1))
S0s=[]
betas=[]
for itime in xrange(nr_grid+1):
tm=[0,int(1e9)]
if itime<nr_grid:
tm[0]=itime*timestep
tm[1]=tm[0]+timestep
dphx=phx[:,np.newaxis,tm[0]:tm[1]]-phx[np.newaxis,:,tm[0]:tm[1]]
dphy=phy[:,np.newaxis,tm[0]:tm[1]]-phy[np.newaxis,:,tm[0]:tm[1]]
dphx=np.unwrap(np.remainder(dphx-dphx[:,:,0][:,:,np.newaxis]+np.pi,2*np.pi))
dvarx=np.var(dphx,axis=-1)
dphy=np.unwrap(np.remainder(dphy-dphy[:,:,0][:,:,np.newaxis]+np.pi,2*np.pi))
dvary=np.var(dphy,axis=-1)
myselect=np.logical_and(D2>0,np.logical_and(np.any(np.logical_and(dvarx>1e-7,dvarx<.1),axis=0)[np.newaxis],np.any(np.logical_and(dvarx>1e-7,dvarx<.1),axis=1)[:,np.newaxis]))
if dofilter:
x=D2[myselect]
y=dvarx[myselect]
# Seems like real values dont occur above 1.0
flagselect = np.where(y > 1.0)
xplot = np.delete(x,flagselect)
yplot = np.delete(y,flagselect)
bins = np.logspace(np.log10(np.min(xplot)),np.log10(np.max(xplot)),10)
binys = []
binxs = []
for i in range(0,len(bins)-1):
binmin = bins[i]
binmax = bins[i+1]
inbin = np.intersect1d(np.where(xplot > binmin),np.where(xplot < binmax))
binxs.append((binmin+binmax)/2.0)
#binys.append(np.percentile(y[inbin],90))
binys.append(np.median(yplot[inbin])+10*mad(yplot[inbin]))
x0 = [0.1,1.0,1.0]
xfit, flag = scipy.optimize.leastsq(residuals, x0, args=(binys,binxs))
flagselect = np.where(y > model(x,xfit))
print xfit,'fitting'
if doplot:
x=D2[myselect]
y=dvarx[myselect]
subplot(2,1,1)
scatter(x,y,color='b')
if dofilter:
y2 = y[flagselect]
x2 = x[flagselect]
scatter(x2,y2,color='r')
scatter(x,model(x,xfit),color='gray',linestyle='-')#,'gray',linestyle='-',linewidth=3)
x=D2[np.logical_and(D2>1e3,myselect)]
y=dvarx[np.logical_and(D2>1e3,myselect)]
if dofilter:
flagselect = np.where(y > model(x,xfit))
x = np.delete(x,flagselect)
y = np.delete(y,flagselect)
A=np.ones((2,x.shape[0]),dtype=float)
A[1,:]=np.log10(x)
par=np.dot(np.linalg.inv(np.dot(A,A.T)),np.dot(A,np.log10(y)))
S0=10**(-1*np.array(par)[0]/np.array(par)[1])
if doplot:
plot(x,10**(par[0]+par[1]*np.log10(x)),'r-')
xscale("log")
yscale("log")
#ylim(1e-3,2)
xlim(30,4000)
myselect=np.logical_and(D2y>0,np.logical_and(np.any(np.logical_and(dvary>1e-7,dvary<.1),axis=0)[np.newaxis],np.any(np.logical_and(dvary>1e-7,dvary<.1),axis=1)[:,np.newaxis]))
if dofilter:
x=D2y[myselect]
y=dvary[myselect]
# seems like real values dont occur above 1.0
flagselect = np.where(y > 1.0)
xplot = np.delete(x,flagselect)
yplot = np.delete(y,flagselect)
bins = np.logspace(np.log10(np.min(xplot)),np.log10(np.max(xplot)),20)
binys = []
binxs = []
for i in range(0,len(bins)-1):
binmin = bins[i]
binmax = bins[i+1]
inbin = np.intersect1d(np.where(xplot > binmin),np.where(xplot < binmax))
binxs.append((binmin+binmax)/2.0)
#binys.append(np.percentile(y[inbin],90))
binys.append(np.median(yplot[inbin])+10*mad(yplot[inbin]))
x0 = [0.1,1.0,1.0]
xfit, flag = scipy.optimize.leastsq(residuals, x0, args=(binys,binxs))
flagselect = np.where(y > model(x,xfit))
print xfit,'fitting'
if doplot:
x=D2y[myselect]
y=dvary[myselect]
subplot(2,1,2)
scatter(x,y,color='b')
if dofilter:
y2 = y[flagselect]
x2 = x[flagselect]
scatter(x2,y2,color='r')
scatter(x,model(x,xfit),color='gray',linestyle='-')#,'gray',linestyle='-',linewidth=3)
x=D2y[np.logical_and(D2y>1e3,myselect)]
y=dvary[np.logical_and(D2y>1e3,myselect)]
if dofilter:
flagselect = np.where(y > model(x,xfit))
x = np.delete(x,flagselect)
y = np.delete(y,flagselect)
A=np.ones((2,x.shape[0]),dtype=float)
A[1,:]=np.log10(x)
pary=np.dot(np.linalg.inv(np.dot(A,A.T)),np.dot(A,np.log10(y)))
S0y=10**(-1*np.array(pary)[0]/np.array(pary)[1])
if doplot:
plot(x,10**(pary[0]+pary[1]*np.log10(x)),'r-')
xscale("log")
yscale("log")
#ylim(1e-3,2)
xlim(30,4000)
savefig('%s_structure.png'%outbasename)
close()
cla()
S0s.append([S0,S0y])
betas.append([par[1],pary[1]])
outfile.write('S0s ****%s**** %s beta %s %s\n'%(S0,S0y,par[1],pary[1]))
return S0s,betas
def main(input_mslist, parmdb_name, outparmdb, clobber=True):
"""
Merges parmdbs in time into a single parmdb
The parmdbs are assumed to be located in the input MS with name
parmdb_name
Parameters
----------
input_mslist : list
List of input MS file names
parmdb_name : str
Name of parmdb (relative to MS files)
outparmdb : str
Name of output merged parmdb
clobber : bool, optional
If True, overwrite existing output file
"""
if type(input_mslist) is str:
input_mslist = input_mslist.strip('[]').split(',')
input_mslist = [f.strip() for f in input_mslist]
inparmdbs = [os.path.join(ms, parmdb_name) for ms in input_mslist]
if type(clobber) is str:
if clobber.lower() == 'true':
clobber = True
else:
clobber = False
if os.path.exists(outparmdb):
if clobber:
os.system('rm -rf {0}'.format(outparmdb))
else:
return
os.system('cp -r {0} {1}'.format(inparmdbs[0], outparmdb))
### old copy, which puts all values into the same domain (i.e. ignoring the timestamps)
# if len(inparmdbs) > 1:
# pdb_concat = lofar.parmdb.parmdb(outparmdb)
# for inparmdb in inparmdbs[1:]:
# pdb = lofar.parmdb.parmdb(inparmdb)
# for parmname in pdb.getNames():
# v = pdb.getValuesGrid(parmname)
# pdb_concat.addValues(v.copy())
# pdb_concat.flush()
### version with TAQL, which avoids the previous problem, but may introduce new ones
# if len(inparmdbs) > 1:
# for inparmdb in inparmdbs[1:]:
# pt.taql('insert into '+outparmdb+' select from '+inparmdb)
# outdb = pt.table(outparmdb,readonly=False,ack=False)
# errdesc = outdb.getcoldesc('ERRORS')
# errdesc['name'] = 'ERRORS'
# outdb.removecols('ERRORS')
# outdb.addcols(errdesc)
# outdb.close()
### version which adds new data by generating a new data-holder first
if len(inparmdbs) > 1:
pdb_concat = pdb.parmdb(outparmdb)
for inparmdb in inparmdbs[1:]:
pdb_add = pdb.parmdb(inparmdb)
for parmname in pdb_add.getNames():
parms = pdb_add.getValuesGrid(parmname)
ValueHolder = pdb_concat.makeValue(
values=parms[parmname]['values'],
sfreq=parms[parmname]['freqs'],
efreq=parms[parmname]['freqwidths'],
stime=parms[parmname]['times'],
etime=parms[parmname]['timewidths'],
asStartEnd=False)
pdb_concat.addValues(parmname, ValueHolder)
pdb_concat.flush()
pdb_add = False
pdb_concat = False
def addSourceTable(image, sourcedbName, minTime, maxTime):
# Create the table using TaQL.
tab = pt.taql("create table '" + image.name() + "/LOFAR_SOURCE' " +
"SOURCE_ID int, \TIME double, INTERVAL double, " +
"NUM_LINES int, NAME string, " +
"DIRECTION double shape=[2], " +
"PROPER_MOTION double shape=[2], " +
"FLUX double shape=[4], " +
"SPINX double, REF_FREQUENCY double, " +
"SHAPE double shape=[3]")
tab.putcolkeyword("TIME", "QuantumUnits", ["s"])
tab.putcolkeyword("INTERVAL", "QuantumUnits", ["s"])
tab.putcolkeyword("DIRECTION", "QuantumUnits", ["rad"])
tab.putcolkeyword("PROPER_MOTION", "QuantumUnits", ["rad/s"])
tab.putcolkeyword("FLUX", "QuantumUnits", ["Jy"])
tab.putcolkeyword("REF_FREQUENCY", "QuantumUnits", ["MHz"])
tab.putcolkeyword("SHAPE", "QuantumUnits", ["rad", "rad", "rad"])
tab.putcolkeyword("TIME", "MEASINFO", {"Ref": "UTC", "type": "epoch"})
tab.putcolkeyword("DIRECTION", "MEASINFO", {
"Ref": "J2000",
"type": "direction"
})
tab.flush()
image.putkeyword("ATTRGROUPS." + "LOFAR_SOURCE", tab)
# Get all parameters from the source parmdb.
midtime = (minTime + maxTime) / 2
inttime = maxTime - minTime
sourcedb = pdb.parmdb(sourcedbName)
# Get all source names by getting the Ra parms from DEFAULTVALUES
names = [name[3:] for name in sourcedb.getDefNames("Ra:*")]
values = sourcedb.getDefValues()
sourcedb = 0 # close
row = 0
tab.addrows(len(names))
# Add the info of all sources.
# The field names below are as used in SourceDB.
fldnames = [
"Ra", "Dec", "I", "Q", "U", "V", "SpectralIndex:0",
"ReferenceFrequency", "Orientation", "MajorAxis", "MinorAxis"
]
vals = [0. for fld in fldnames]
for name in names:
for i in range(len(fldnames)):
key = fldnames[i] + ":" + name
if values.has_key(key):
vals[i] = values[key][0][0]
else:
vals[i] = 0.
tab.putcell("SOURCE_ID", row, row)
tab.putcell("TIME", row, midtime)
tab.putcell("INTERVAL", row, inttime)
tab.putcell("NUM_LINES", row, 0)
tab.putcell("NAME", row, name)
tab.putcell("DIRECTION", row, vals[:2])
tab.putcell("PROPER_MOTION", row, (0., 0.))
tab.putcell("FLUX", row, vals[2:6])
tab.putcell("SPINX", row, vals[6])
tab.putcell("REF_FREQUENCY", row, vals[7])
tab.putcell("SHAPE", row, vals[8:11])
row += 1
# Ready.
tab.close()
print "Added subtable LOFAR_SOURCE containing", row, "rows"
def handle_close(self):
self.close_all_figures()
def closeEvent(self, event):
self.close_all_figures()
event.accept()
if __name__ == "__main__":
if len(sys.argv) <= 1 or sys.argv[1] == "--help":
print "usage: parmdbplot.py <parmdb>"
sys.exit(1)
try:
db = parmdb.parmdb(sys.argv[1])
except:
print "ERROR:", sys.argv[1], "is not a valid parmdb."
sys.exit(1)
app = QApplication(sys.argv)
# show parmdbname in title (with msname if it is inside an MS)
splitpath = string.split(string.rstrip(sys.argv[1],"/"),"/")
if len(splitpath)>1 and splitpath[-2][-2:]=="MS":
parmdbname = "/".join(splitpath[-2:])
else:
parmdbname = splitpath[-1]
window = MainWindow(db, parmdbname)
window.show()
def main(parmdbfile, targetms, phaseonly = True):
if not os.path.exists(parmdbfile):
print "Parmdb file %s doesn't exist!" % parmdbfile
return(1)
if not os.path.exists(targetms):
print "Target ms %s doesn't exist!" % targetms
return(1)
msinfo = ReadMs(targetms)
# Open up the parmdb, get an example value
parmdb = pdb.parmdb(parmdbfile)
parnames = parmdb.getNames()
examplevalue = None
for name in parnames:
if "CS" in name:
examplevalue = parmdb.getValuesGrid(name)[name]
break
#print examplevalue.keys()
# Zero the phases of the example entry
if examplevalue == None:
print "Couldn't find an example entry"
return(1)
examplevalue['values'] = np.zeros(examplevalue['values'].shape)
if not(phaseonly):
examplevalue_ones = copy.deepcopy(examplevalue)
examplevalue_ones['values'] = np.ones(examplevalue_ones['values'].shape)
# Add the necessary stations
for antenna_id, antenna in enumerate(msinfo.stations):
if not "CS" in antenna and not "RS" in antenna:
ValueHolder = parmdb.makeValue(values=examplevalue['values'],
sfreq=examplevalue['freqs'],
efreq=examplevalue['freqwidths'],
stime=examplevalue['times'],
etime=examplevalue['timewidths'],
asStartEnd=False)
if phaseonly:
## in case the values already exist (and may be NaN) they need to be deleted first
parmdb.deleteValues("Gain:0:0:Phase:" + antenna)
parmdb.deleteValues("Gain:1:1:Phase:" + antenna)
parmdb.addValues("Gain:0:0:Phase:" + antenna,ValueHolder)
parmdb.addValues("Gain:1:1:Phase:" + antenna,ValueHolder)
else:
ValueHolder_ones = parmdb.makeValue(values=examplevalue_ones['values'],
sfreq=examplevalue_ones['freqs'],
efreq=examplevalue_ones['freqwidths'],
stime=examplevalue_ones['times'],
etime=examplevalue_ones['timewidths'],
asStartEnd=False)
## in case the values already exist (and may be NaN) they need to be deleted first
parmdb.deleteValues("Gain:0:0:Real:" + antenna)
parmdb.deleteValues("Gain:1:1:Real:" + antenna)
parmdb.deleteValues("Gain:0:0:Imag:" + antenna)
parmdb.deleteValues("Gain:1:1:Imag:" + antenna)
parmdb.addValues("Gain:0:0:Real:" + antenna,ValueHolder_ones)
parmdb.addValues("Gain:0:0:Imag:" + antenna,ValueHolder)
parmdb.addValues("Gain:1:1:Real:" + antenna,ValueHolder_ones)
parmdb.addValues("Gain:1:1:Imag:" + antenna,ValueHolder)
parmdb.flush()
parmdb = 0
return(0)
