def run(soltab, axesInPlot, axisInTable='', axisInCol='', axisDiff='', NColFig=0, figSize=[0,0], markerSize=2, minmax=[0,0], log='', \
plotFlag=False, doUnwrap=False, refAnt='', soltabsToAdd='', makeAntPlot=False, makeMovie=False, prefix='', ncpu=0):
"""
This operation for LoSoTo implements basic plotting
WEIGHT: flag-only compliant, no need for weight
Parameters
----------
axesInPlot : array of str
1- or 2-element array which says the coordinates to plot (2 for 3D plots).
axisInTable : str, optional
the axis to plot on a page - e.g. ant to get all antenna's on one file. By default ''.
axisInCol : str, optional
The axis to plot in different colours - e.g. pol to get correlations with different colors. By default ''.
axisDiff : str, optional
This must be a len=2 axis and the plot will have the differential value - e.g. 'pol' to plot XX-YY. By default ''.
NColFig : int, optional
Number of columns in a multi-table image. By default is automatically chosen.
figSize : array of int, optional
Size of the image [x,y], if one of the values is 0, then it is automatically chosen. By default automatic set.
markerSize : int, optional
Size of the markers in the 2D plot. By default 2.
minmax : array of float, optional
Min max value for the independent variable (0 means automatic). By default 0.
log : bool, optional
Use Log='XYZ' to set which axes to put in Log. By default ''.
plotFlag : bool, optional
Whether to plot also flags as red points in 2D plots. By default False.
doUnwrap : bool, optional
Unwrap phases. By default False.
refAnt : str, optional
Reference antenna for phases. By default None.
soltabsToAdd : str, optional
Tables to "add" (e.g. 'sol000/tec000'), it works only for tec and clock to be added to phases. By default None.
makeAntPlot : bool, optional
Make a plot containing antenna coordinates in x,y and in color the value to plot, axesInPlot must be [ant]. By default False.
makeMovie : bool, optional
Make a movie summing up all the produced plots, by default False.
prefix : str, optional
Prefix to add before the self-generated filename, by default None.
ncpu : int, optional
Number of cpus, by default all available.
"""
import os, random
import numpy as np
from losoto.lib_unwrap import unwrap, unwrap_2d
logging.info("Plotting soltab: " + soltab.name)
# input check
# str2list
if axisInTable == '': axisInTable = []
else: axisInTable = [axisInTable]
if axisInCol == '': axisInCol = []
else: axisInCol = [axisInCol]
if axisDiff == '': axisDiff = []
else: axisDiff = [axisDiff]
if len(set(axisInTable + axesInPlot + axisInCol +
axisDiff)) != len(axisInTable + axesInPlot + axisInCol +
axisDiff):
logging.error('Axis defined multiple times.')
return 1
# just because we use lists, check that they are 1-d
if len(axisInTable) > 1 or len(axisInCol) > 1 or len(axisDiff) > 1:
logging.error(
'Too many TableAxis/ColAxis/DiffAxis, they must be at most one each.'
)
return 1
for axis in axesInPlot + axisInCol + axisDiff:
if axis not in soltab.getAxesNames():
logging.error('Axis \"' + axis + '\" not found.')
return 1
if makeMovie:
prefix = prefix + '__tmp__'
if os.path.dirname(prefix) != '' and not os.path.exists(
os.path.dirname(prefix)):
logging.debug('Creating ' + os.path.dirname(prefix) + '.')
os.makedirs(os.path.dirname(prefix))
if refAnt == '': refAnt = None
elif refAnt != 'closest' and not refAnt in soltab.getAxisValues(
'ant', ignoreSelection=True):
logging.error('Reference antenna ' + refAnt + ' not found. Using: ' +
soltab.getAxisValues('ant')[1])
refAnt = soltab.getAxisValues('ant')[1]
minZ, maxZ = minmax
solset = soltab.getSolset()
soltabsToAdd = [
solset.getSoltab(soltabName) for soltabName in soltabsToAdd
]
cmesh = False
if len(axesInPlot) == 2:
cmesh = True
# not color possible in 3D
axisInCol = []
elif len(axesInPlot) != 1:
logging.error('Axes must be a len 1 or 2 array.')
return 1
# end input check
# all axes that are not iterated by anything else
axesInFile = soltab.getAxesNames()
for axis in axisInTable + axesInPlot + axisInCol + axisDiff:
axesInFile.remove(axis)
# set subplots scheme
if axisInTable != []:
Nplots = soltab.getAxisLen(axisInTable[0])
else:
Nplots = 1
# prepare antennas coord in makeAntPlot case
if makeAntPlot:
if axesInPlot != ['ant']:
logging.error(
'If makeAntPlot is selected the "Axes" values must be "ant"')
return 1
antCoords = [[], []]
for ant in soltab.getAxisValues(
'ant'): # select only user-selected antenna in proper order
antCoords[0].append(+1 * soltab.getSolset().getAnt()[ant][1])
antCoords[1].append(-1 * soltab.getSolset().getAnt()[ant][0])
else:
antCoords = []
datatype = soltab.getType()
# start processes for multi-thread
mpm = multiprocManager(ncpu, _plot)
# compute dataCube size
shape = []
if axisInTable != []: shape.append(soltab.getAxisLen(axisInTable[0]))
else: shape.append(1)
if axisInCol != []: shape.append(soltab.getAxisLen(axisInCol[0]))
else: shape.append(1)
if cmesh:
shape.append(soltab.getAxisLen(axesInPlot[1]))
shape.append(soltab.getAxisLen(axesInPlot[0]))
else:
shape.append(soltab.getAxisLen(axesInPlot[0]))
# will contain the data to pass to each thread to make 1 image
dataCube = np.ma.zeros(shape=shape, fill_value=np.nan)
# cycle on files
if makeMovie: pngs = [] # store png filenames
for vals, coord, selection in soltab.getValuesIter(
returnAxes=axisDiff + axisInTable + axisInCol + axesInPlot):
# set filename
filename = ''
for axis in axesInFile:
filename += axis + str(coord[axis]) + '_'
filename = filename[:-1] # remove last _
if prefix + filename == '': filename = 'plot'
# axis vals (they are always the same, regulat arrays)
xvals = coord[axesInPlot[0]]
# if plotting antenna - convert to number
if axesInPlot[0] == 'ant':
xvals = np.arange(len(xvals))
# if plotting time - convert in h/min/s
xlabelunit = ''
if axesInPlot[0] == 'time':
if xvals[-1] - xvals[0] > 3600:
xvals = (xvals - xvals[0]) / 3600. # hrs
xlabelunit = ' [hr]'
elif xvals[-1] - xvals[0] > 60:
xvals = (xvals - xvals[0]) / 60. # mins
xlabelunit = ' [min]'
else:
xvals = (xvals - xvals[0]) # sec
xlabelunit = ' [s]'
# if plotting freq convert in MHz
elif axesInPlot[0] == 'freq':
xvals = xvals / 1.e6 # MHz
xlabelunit = ' [MHz]'
if cmesh:
# axis vals (they are always the same, regular arrays)
yvals = coord[axesInPlot[1]]
# same as above but for y-axis
if axesInPlot[1] == 'ant':
yvals = np.arange(len(yvals))
if len(xvals) <= 1 or len(yvals) <= 1:
logging.error(
'3D plot must have more then one value per axes.')
mpm.wait()
return 1
ylabelunit = ''
if axesInPlot[1] == 'time':
if yvals[-1] - yvals[0] > 3600:
yvals = (yvals - yvals[0]) / 3600. # hrs
ylabelunit = ' [hr]'
elif yvals[-1] - yvals[0] > 60:
yvals = (yvals - yvals[0]) / 60. # mins
ylabelunit = ' [min]'
else:
yvals = (yvals - yvals[0]) # sec
ylabelunit = ' [s]'
elif axesInPlot[1] == 'freq': # Mhz
yvals = yvals / 1.e6
ylabelunit = ' [MHz]'
else:
yvals = None
if datatype == 'clock':
datatype = 'Clock'
ylabelunit = ' (s)'
elif datatype == 'tec':
datatype = 'dTEC'
ylabelunit = ' (TECU)'
elif datatype == 'rotationmeasure':
datatype = 'dRM'
ylabelunit = r' (rad m$^{-2}$)'
elif datatype == 'tec3rd':
datatype = r'dTEC$_3$'
ylabelunit = r' (rad m$^{-3}$)'
else:
ylabelunit = ''
# cycle on tables
soltab1Selection = soltab.selection # save global selection and subselect only axex to iterate
soltab.selection = selection
titles = []
for Ntab, (vals, coord, selection) in enumerate(
soltab.getValuesIter(returnAxes=axisDiff + axisInCol +
axesInPlot)):
# set tile
titles.append('')
for axis in coord:
if axis in axesInFile + axesInPlot + axisInCol: continue
titles[Ntab] += axis + ':' + str(coord[axis]) + ' '
titles[Ntab] = titles[Ntab][:-1] # remove last ' '
# cycle on colors
soltab2Selection = soltab.selection
soltab.selection = selection
for Ncol, (vals, weight, coord, selection) in enumerate(
soltab.getValuesIter(returnAxes=axisDiff + axesInPlot,
weight=True,
reference=refAnt)):
# differential plot
if axisDiff != []:
# find ordered list of axis
names = [
axis for axis in soltab.getAxesNames()
if axis in axisDiff + axesInPlot
]
if axisDiff[0] not in names:
logging.error("Axis to differentiate (%s) not found." %
axisDiff[0])
mpm.wait()
return 1
if len(coord[axisDiff[0]]) != 2:
logging.error(
"Axis to differentiate (%s) has too many values, only 2 is allowed."
% axisDiff[0])
mpm.wait()
return 1
# find position of interesting axis
diff_idx = names.index(axisDiff[0])
# roll to first place
vals = np.rollaxis(vals, diff_idx, 0)
vals = vals[0] - vals[1]
weight = np.rollaxis(weight, diff_idx, 0)
weight[0][weight[1] == 0] = 0
weight = weight[0]
del coord[axisDiff[0]]
# add tables if required (e.g. phase/tec)
for soltabToAdd in soltabsToAdd:
logging.warning('soltabsToAdd not implemented. Ignoring.')
# newCoord = {}
# for axisName in coord.keys():
# # prepare selected on present axes
# if axisName in soltabToAdd.getAxesNames():
# if type(coord[axisName]) is np.ndarray:
# newCoord[axisName] = coord[axisName]
# else:
# newCoord[axisName] = [coord[axisName]] # avoid being interpreted as regexp, faster
#
# soltabToAdd.setSelection(**newCoord)
# valsAdd = np.squeeze(soltabToAdd.getValues(retAxesVals=False, weight=False, reference=refAnt))
#
# # add missing axes
# print ('shape:', vals.shape)
# for axisName in coord.keys():
# if not axisName in soltabToAdd.getAxesNames():
# # find axis positions
# axisPos = soltab.getAxesNames().index(axisName)
# # create a new axes for the table to add and duplicate the values
# valsAdd = np.expand_dims(valsAdd, axisPos)
# print ('shape to add:', valsAdd.shape)
#
# if soltabToAdd.getType() == 'clock':
# valsAdd = 2. * np.pi * valsAdd * coord['freq']
# elif soltabToAdd.getType() == 'tec':
# valsAdd = -8.44797245e9 * valsAdd / coord['freq']
# else:
# logging.warning('Only Clock or TEC can be added to solutions. Ignoring: '+soltabToAdd.getType()+'.')
# continue
#
# if valsAdd.shape != vals.shape:
# logging.error('Cannot combine the table '+soltabToAdd.getType()+' with '+soltab.getType()+'. Wrong shape.')
# mpm.wait()
# return 1
#
# vals += valsAdd
# normalize
if (soltab.getType() == 'phase'
or soltab.getType() == 'scalarphase'):
vals = normalize_phase(vals)
if (soltab.getType() == 'rotation'):
vals = np.mod(vals + np.pi / 2., np.pi) - np.pi / 2.
# is user requested axis in an order that is different from h5parm, we need to transpose
if cmesh:
if soltab.getAxesNames().index(
axesInPlot[0]) < soltab.getAxesNames().index(
axesInPlot[1]):
vals = vals.T
weight = weight.T
# unwrap if required
if (soltab.getType() == 'phase'
or soltab.getType() == 'scalarphase') and doUnwrap:
if len(axesInPlot) == 1:
vals = unwrap(vals)
else:
flags = np.array((weight == 0), dtype=bool)
if not (flags == True).all():
vals = unwrap_2d(vals, flags, coord[axesInPlot[0]],
coord[axesInPlot[1]])
dataCube[Ntab, Ncol] = vals
sel1 = np.where(weight == 0.)
sel2 = np.where(np.isnan(vals))
if cmesh:
dataCube[Ntab, Ncol, sel1[0], sel1[1]] = np.ma.masked
dataCube[Ntab, Ncol, sel2[0], sel2[1]] = np.ma.masked
else:
dataCube[Ntab, Ncol, sel1[0]] = np.ma.masked
dataCube[Ntab, Ncol, sel2[0]] = np.ma.masked
soltab.selection = soltab2Selection
### end cycle on colors
# if dataCube too large (> 500 MB) do not go parallel
if np.array(dataCube).nbytes > 1024 * 1024 * 500:
logging.debug('Big plot, parallel not possible.')
_plot(Nplots, NColFig, figSize, markerSize, cmesh, axesInPlot,
axisInTable, xvals, yvals, xlabelunit, ylabelunit, datatype,
prefix + filename, titles, log, dataCube, minZ, maxZ,
plotFlag, makeMovie, antCoords, None)
else:
mpm.put([
Nplots, NColFig, figSize, markerSize, cmesh, axesInPlot,
axisInTable, xvals, yvals, xlabelunit, ylabelunit, datatype,
prefix + filename, titles, log,
np.ma.copy(dataCube), minZ, maxZ, plotFlag, makeMovie,
antCoords
])
if makeMovie: pngs.append(prefix + filename + '.png')
soltab.selection = soltab1Selection
### end cycle on tables
mpm.wait()
if makeMovie:
def long_substr(strings):
"""
Find longest common substring
"""
substr = ''
if len(strings) > 1 and len(strings[0]) > 0:
for i in range(len(strings[0])):
for j in range(len(strings[0]) - i + 1):
if j > len(substr) and all(strings[0][i:i + j] in x
for x in strings):
substr = strings[0][i:i + j]
return substr
movieName = long_substr(pngs)
assert movieName != '' # need a common prefix, use prefix keyword in case
logging.info('Making movie: ' + movieName)
# make every movie last 20 sec, min one second per slide
fps = np.ceil(len(pngs) / 200.)
ss="mencoder -ovc lavc -lavcopts vcodec=mpeg4:vpass=1:vbitrate=6160000:mbd=2:keyint=132:v4mv:vqmin=3:lumi_mask=0.07:dark_mask=0.2:"+\
"mpeg_quant:scplx_mask=0.1:tcplx_mask=0.1:naq -mf type=png:fps="+str(fps)+" -nosound -o "+movieName.replace('__tmp__','')+".mpg mf://"+movieName+"* > mencoder.log 2>&1"
os.system(ss)
#for png in pngs: os.system('rm '+png)
return 0
def run( soltab, doUnwrap=False, refAnt='', plotName='', ndiv=1 ):
"""
Find the structure function from phase solutions of core stations.
Parameters
----------
doUnwrap : bool, optional
refAnt : str, optional
Reference antenna, by default the first.
plotName : str, optional
Plot file name, by default no plot.
ndiv : int, optional
"""
import numpy as np
from losoto.lib_unwrap import unwrap, unwrap_2d
logging.info("Find structure function for soltab: "+soltab.name)
# input check
solType = soltab.getType()
if solType != 'phase':
logging.warning("Soltab type of "+soltab._v_name+" is of type "+solType+", should be phase.")
return 1
ants = soltab.getAxisValues('ant')
if refAnt != '' and refAnt != 'closest' and not refAnt in soltab.getAxisValues('ant', ignoreSelection = True):
logging.error('Reference antenna '+refAnt+' not found. Using: '+ants[1])
refAnt = ants[1]
if refAnt == '' and doUnwrap:
logging.error('Unwrap requires reference antenna. Using: '+ants[1])
refAnt = ants[1]
if refAnt == '': refAnt = None
soltab.setSelection(ant='CS*', update=True)
posAll = soltab.getSolset().getAnt()
for vals, weights, coord, selection in soltab.getValuesIter(returnAxes=['freq','pol','ant','time'], weight=True, reference=refAnt):
# reorder axes
vals = reorderAxes( vals, soltab.getAxesNames(), ['pol','ant','freq','time'] )
weights = reorderAxes( weights, soltab.getAxesNames(), ['pol','ant','freq','time'] )
# order positions
pos = np.array([list(posAll[ant]) for ant in coord['ant']])
# avg pols
vals = np.cos(vals) + 1.j * np.sin(vals)
vals = np.nansum(vals, axis=0)
vals = np.angle(vals)
flags = np.array((weights[0]==0)|(weights[1]==0), dtype=bool)
# unwrap
if doUnwrap:
# remove mean to facilitate unwrapping
for a, ant in enumerate(coord['ant']):
if not (flags[a,:,:] == True).all() and ant != refAnt:
logging.debug('Unwrapping: '+ant)
mean = np.angle( np.nanmean( np.exp(1j*vals[a].flatten()) ))
vals[a] -= mean
vals[a] = np.mod(vals[a]+np.pi, 2*np.pi) - np.pi
vals[a,:,:] = unwrap_2d(vals[a,:,:], flags[a,:,:], coord['freq'], coord['time'])
logging.debug('Computing differential values...')
t1 = np.ma.array( vals, mask=flags ) # mask flagged data
dph = t1[np.newaxis]-t1[:,np.newaxis] # ant x ant x freq x time
D = pos[np.newaxis]-pos[:,np.newaxis] # ant x ant x 3
D2 = np.triu(np.sqrt(np.sum(D**2,axis=-1))) # calc distance and keep only uppoer triangle larger than 0
myselect = (D2>0)
if not doUnwrap:
logging.debug('Re-normalising...')
dph = np.mod(dph+np.pi, 2*np.pi) - np.pi
avgdph = np.ma.average(dph, axis=2) # avg in freq (can do because is between -pi and pi)
#one extra step to remove most(all) phase wraps, phase wraps disturbe the averaging...
dph = np.remainder(dph - np.ma.average(avgdph,axis=-1)[:,:,np.newaxis,np.newaxis]+np.pi,2*np.pi) + np.ma.average(avgdph,axis=-1)[:,:,np.newaxis,np.newaxis]-np.pi #center around the avg value
logging.debug('Computing sructure function...')
avgdph = np.ma.average(dph,axis=2) # avg in freq to reduce noise
variances = []; pars = []
avgdph = avgdph[...,avgdph.shape[-1]%ndiv:] # remove a few timeslots to make the array divisible by np.split
for i, avgdphSplit in enumerate( np.split(avgdph, ndiv, axis=-1) ):
variance = np.ma.var(avgdphSplit, axis=-1)*(np.average(coord['freq'])/150.e6)**2 # get time variance and rescale to 150 MHz
# linear regression
#A = np.ones((2,D2[myselect].shape[0]),dtype=float)
#A[1,:] = np.log10(D2[myselect][~variance.mask])
#par = np.dot(np.linalg.inv(np.dot(A,A.T)),np.dot(A,np.log10(variance[myselect])))
mask = variance[myselect].mask
A = np.vstack([np.log10(D2[myselect][~mask]), np.ones(len(D2[myselect][~mask]))])
par = np.linalg.lstsq( A.T, np.log10(variance[myselect][~mask]) )[0]
S0 = 10**(-1*par[1]/par[0])
logging.info(r't%i: beta=%.2f - R_diff=%.2f km' % (i, par[0], S0/1.e3))
variances.append(variance)
pars.append(par)
if plotName != '':
if plotName.split('.')[-1] != 'png': plotName += '.png' # add png
if not 'matplotlib' in sys.modules:
import matplotlib as mpl
mpl.use("Agg")
import matplotlib.pyplot as plt
fig = plt.figure()
fig.subplots_adjust(wspace=0)
ax = fig.add_subplot(111)
ax1 = ax.twinx()
for i, variance in enumerate(variances):
if len(variances) > 1:
color = plt.cm.jet(i/float(len(variances)-1)) # from 0 to 1
else: color = 'black'
ax.plot(D2[myselect]/1.e3,variance[myselect],marker='o',linestyle='', color=color, markeredgecolor='none', label='T')
# regression
par = pars[i]
x = D2[myselect]
S0 = 10**(-1*par[1]/par[0])
if color == 'black': color = 'red' # in case of single color, use red line that is more visible
ax1.plot(x.flatten()/1.e3, par[0]*np.log10(x.flatten()) + par[1], linestyle='-', color=color, label=r'$\beta=%.2f$ - $R_{\rm diff}=%.2f$ km' % (par[0], S0/1.e3))
ax.set_xlabel('Distance (km)')
ax.set_ylabel(r'Phase variance @150 MHz (rad$^2$)')
ax.set_xscale('log')
ax.set_yscale('log')
ymin = np.min(variance[myselect])
ymax = np.max(variance[myselect])
ax.set_xlim(xmin=0.1,xmax=3)
ax.set_ylim(ymin,ymax)
ax1.set_ylim(np.log10(ymin),np.log10(ymax))
ax1.legend(loc='lower right', frameon=False)
ax1.set_yticks([])
logging.warning('Save pic: %s' % plotName)
plt.savefig(plotName, bbox_inches='tight')
return 0
def run(soltab, doUnwrap=False, refAnt='', plotName='', ndiv=1):
"""
Find the structure function from phase solutions of core stations.
Parameters
----------
doUnwrap : bool, optional
refAnt : str, optional
Reference antenna, by default the first.
plotName : str, optional
Plot file name, by default no plot.
ndiv : int, optional
"""
import numpy as np
from losoto.lib_unwrap import unwrap, unwrap_2d
logging.info("Find structure function for soltab: " + soltab.name)
# input check
solType = soltab.getType()
if solType != 'phase':
logging.warning("Soltab type of " + soltab._v_name + " is of type " +
solType + ", should be phase.")
return 1
ants = soltab.getAxisValues('ant')
if refAnt != '' and refAnt != 'closest' and not refAnt in soltab.getAxisValues(
'ant', ignoreSelection=True):
logging.error('Reference antenna ' + refAnt + ' not found. Using: ' +
ants[1])
refAnt = ants[1]
if refAnt == '' and doUnwrap:
logging.error('Unwrap requires reference antenna. Using: ' + ants[1])
refAnt = ants[1]
if refAnt == '': refAnt = None
soltab.setSelection(ant='CS*', update=True)
posAll = soltab.getSolset().getAnt()
for vals, weights, coord, selection in soltab.getValuesIter(
returnAxes=['freq', 'pol', 'ant', 'time'],
weight=True,
reference=refAnt):
# reorder axes
vals = reorderAxes(vals, soltab.getAxesNames(),
['pol', 'ant', 'freq', 'time'])
weights = reorderAxes(weights, soltab.getAxesNames(),
['pol', 'ant', 'freq', 'time'])
# order positions
pos = np.array([list(posAll[ant]) for ant in coord['ant']])
# avg pols
vals = np.cos(vals) + 1.j * np.sin(vals)
vals = np.nansum(vals, axis=0)
vals = np.angle(vals)
flags = np.array((weights[0] == 0) | (weights[1] == 0), dtype=bool)
# unwrap
if doUnwrap:
# remove mean to facilitate unwrapping
for a, ant in enumerate(coord['ant']):
if not (flags[a, :, :] == True).all() and ant != refAnt:
logging.debug('Unwrapping: ' + ant)
mean = np.angle(np.nanmean(np.exp(1j * vals[a].flatten())))
vals[a] -= mean
vals[a] = np.mod(vals[a] + np.pi, 2 * np.pi) - np.pi
vals[a, :, :] = unwrap_2d(vals[a, :, :], flags[a, :, :],
coord['freq'], coord['time'])
logging.debug('Computing differential values...')
t1 = np.ma.array(vals, mask=flags) # mask flagged data
dph = t1[np.newaxis] - t1[:, np.newaxis] # ant x ant x freq x time
D = pos[np.newaxis] - pos[:, np.newaxis] # ant x ant x 3
D2 = np.triu(
np.sqrt(np.sum(D**2, axis=-1))
) # calc distance and keep only uppoer triangle larger than 0
myselect = (D2 > 0)
if not doUnwrap:
logging.debug('Re-normalising...')
dph = np.mod(dph + np.pi, 2 * np.pi) - np.pi
avgdph = np.ma.average(
dph,
axis=2) # avg in freq (can do because is between -pi and pi)
#one extra step to remove most(all) phase wraps, phase wraps disturbe the averaging...
dph = np.remainder(
dph -
np.ma.average(avgdph, axis=-1)[:, :, np.newaxis, np.newaxis] +
np.pi, 2 * np.pi) + np.ma.average(
avgdph,
axis=-1)[:, :, np.newaxis,
np.newaxis] - np.pi #center around the avg value
logging.debug('Computing sructure function...')
avgdph = np.ma.average(dph, axis=2) # avg in freq to reduce noise
variances = []
pars = []
avgdph = avgdph[
..., avgdph.shape[-1] %
ndiv:] # remove a few timeslots to make the array divisible by np.split
for i, avgdphSplit in enumerate(np.split(avgdph, ndiv, axis=-1)):
variance = np.ma.var(avgdphSplit, axis=-1) * (
np.average(coord['freq']) /
150.e6)**2 # get time variance and rescale to 150 MHz
# linear regression
#A = np.ones((2,D2[myselect].shape[0]),dtype=float)
#A[1,:] = np.log10(D2[myselect][~variance.mask])
#par = np.dot(np.linalg.inv(np.dot(A,A.T)),np.dot(A,np.log10(variance[myselect])))
mask = variance[myselect].mask
A = np.vstack([
np.log10(D2[myselect][~mask]),
np.ones(len(D2[myselect][~mask]))
])
par = np.linalg.lstsq(A.T, np.log10(variance[myselect][~mask]))[0]
S0 = 10**(-1 * par[1] / par[0])
logging.info(r't%i: beta=%.2f - R_diff=%.2f km' %
(i, par[0], S0 / 1.e3))
variances.append(variance)
pars.append(par)
if plotName != '':
if plotName.split('.')[-1] != 'png': plotName += '.png' # add png
if not 'matplotlib' in sys.modules:
import matplotlib as mpl
mpl.use("Agg")
import matplotlib.pyplot as plt
fig = plt.figure()
fig.subplots_adjust(wspace=0)
ax = fig.add_subplot(111)
ax1 = ax.twinx()
for i, variance in enumerate(variances):
if len(variances) > 1:
color = plt.cm.jet(
i / float(len(variances) - 1)) # from 0 to 1
else:
color = 'black'
ax.plot(D2[myselect] / 1.e3,
variance[myselect],
marker='o',
linestyle='',
color=color,
markeredgecolor='none',
label='T')
# regression
par = pars[i]
x = D2[myselect]
S0 = 10**(-1 * par[1] / par[0])
if color == 'black':
color = 'red' # in case of single color, use red line that is more visible
ax1.plot(x.flatten() / 1.e3,
par[0] * np.log10(x.flatten()) + par[1],
linestyle='-',
color=color,
label=r'$\beta=%.2f$ - $R_{\rm diff}=%.2f$ km' %
(par[0], S0 / 1.e3))
ax.set_xlabel('Distance (km)')
ax.set_ylabel(r'Phase variance @150 MHz (rad$^2$)')
ax.set_xscale('log')
ax.set_yscale('log')
ymin = np.min(variance[myselect])
ymax = np.max(variance[myselect])
ax.set_xlim(xmin=0.1, xmax=3)
ax.set_ylim(ymin, ymax)
ax1.set_ylim(np.log10(ymin), np.log10(ymax))
ax1.legend(loc='lower right', frameon=False)
ax1.set_yticks([])
logging.warning('Save pic: %s' % plotName)
plt.savefig(plotName, bbox_inches='tight')
return 0
def run(soltab, axesInPlot, axisInTable='', axisInCol='', axisDiff='', NColFig=0, figSize=[0,0], markerSize=2, minmax=[0,0], log='', \
plotFlag=False, doUnwrap=False, refAnt='', soltabsToAdd='', makeAntPlot=False, makeMovie=False, prefix='', ncpu=0):
"""
This operation for LoSoTo implements basic plotting
WEIGHT: flag-only compliant, no need for weight
Parameters
----------
axesInPlot : array of str
1- or 2-element array which says the coordinates to plot (2 for 3D plots).
axisInTable : str, optional
the axis to plot on a page - e.g. ant to get all antenna's on one file. By default ''.
axisInCol : str, optional
The axis to plot in different colours - e.g. pol to get correlations with different colors. By default ''.
axisDiff : str, optional
This must be a len=2 axis and the plot will have the differential value - e.g. 'pol' to plot XX-YY. By default ''.
NColFig : int, optional
Number of columns in a multi-table image. By default is automatically chosen.
figSize : array of int, optional
Size of the image [x,y], if one of the values is 0, then it is automatically chosen. By default automatic set.
markerSize : int, optional
Size of the markers in the 2D plot. By default 2.
minmax : array of float, optional
Min max value for the independent variable (0 means automatic). By default 0.
log : bool, optional
Use Log='XYZ' to set which axes to put in Log. By default ''.
plotFlag : bool, optional
Whether to plot also flags as red points in 2D plots. By default False.
doUnwrap : bool, optional
Unwrap phases. By default False.
refAnt : str, optional
Reference antenna for phases. By default None.
soltabsToAdd : str, optional
Tables to "add" (e.g. 'sol000/tec000'), it works only for tec and clock to be added to phases. By default None.
makeAntPlot : bool, optional
Make a plot containing antenna coordinates in x,y and in color the value to plot, axesInPlot must be [ant]. By default False.
makeMovie : bool, optional
Make a movie summing up all the produced plots, by default False.
prefix : str, optional
Prefix to add before the self-generated filename, by default None.
ncpu : int, optional
Number of cpus, by default all available.
"""
import os, random
import numpy as np
from losoto.lib_unwrap import unwrap, unwrap_2d
logging.info("Plotting soltab: "+soltab.name)
# input check
# str2list
if axisInTable == '': axisInTable = []
else: axisInTable = [axisInTable]
if axisInCol == '': axisInCol = []
else: axisInCol = [axisInCol]
if axisDiff == '': axisDiff = []
else: axisDiff = [axisDiff]
if len(set(axisInTable+axesInPlot+axisInCol+axisDiff)) != len(axisInTable+axesInPlot+axisInCol+axisDiff):
logging.error('Axis defined multiple times.')
return 1
# just because we use lists, check that they are 1-d
if len(axisInTable) > 1 or len(axisInCol) > 1 or len(axisDiff) > 1:
logging.error('Too many TableAxis/ColAxis/DiffAxis, they must be at most one each.')
return 1
for axis in axesInPlot+axisInCol+axisDiff:
if axis not in soltab.getAxesNames():
logging.error('Axis \"'+axis+'\" not found.')
return 1
if makeMovie:
prefix = prefix+'__tmp__'
if os.path.dirname(prefix) != '' and not os.path.exists(os.path.dirname(prefix)):
logging.debug('Creating '+os.path.dirname(prefix)+'.')
os.makedirs(os.path.dirname(prefix))
if refAnt == '': refAnt = None
elif refAnt != 'closest' and not refAnt in soltab.getAxisValues('ant', ignoreSelection = True):
logging.error('Reference antenna '+refAnt+' not found. Using: '+soltab.getAxisValues('ant')[1])
refAnt = soltab.getAxisValues('ant')[1]
minZ, maxZ = minmax
solset = soltab.getSolset()
soltabsToAdd = [ solset.getSoltab(soltabName) for soltabName in soltabsToAdd ]
cmesh = False
if len(axesInPlot) == 2:
cmesh = True
# not color possible in 3D
axisInCol = []
elif len(axesInPlot) != 1:
logging.error('Axes must be a len 1 or 2 array.')
return 1
# end input check
# all axes that are not iterated by anything else
axesInFile = soltab.getAxesNames()
for axis in axisInTable+axesInPlot+axisInCol+axisDiff:
axesInFile.remove(axis)
# set subplots scheme
if axisInTable != []:
Nplots = soltab.getAxisLen(axisInTable[0])
else:
Nplots = 1
# prepare antennas coord in makeAntPlot case
if makeAntPlot:
if axesInPlot != ['ant']:
logging.error('If makeAntPlot is selected the "Axes" values must be "ant"')
return 1
antCoords = [[],[]]
for ant in soltab.getAxisValues('ant'): # select only user-selected antenna in proper order
antCoords[0].append(+1*soltab.getSolset().getAnt()[ant][1])
antCoords[1].append(-1*soltab.getSolset().getAnt()[ant][0])
else:
antCoords = []
datatype = soltab.getType()
# start processes for multi-thread
mpm = multiprocManager(ncpu, _plot)
# compute dataCube size
shape = []
if axisInTable != []: shape.append(soltab.getAxisLen(axisInTable[0]))
else: shape.append(1)
if axisInCol != []: shape.append(soltab.getAxisLen(axisInCol[0]))
else: shape.append(1)
if cmesh:
shape.append(soltab.getAxisLen(axesInPlot[1]))
shape.append(soltab.getAxisLen(axesInPlot[0]))
else:
shape.append(soltab.getAxisLen(axesInPlot[0]))
# will contain the data to pass to each thread to make 1 image
dataCube = np.ma.zeros( shape=shape, fill_value=np.nan )
# cycle on files
if makeMovie: pngs = [] # store png filenames
for vals, coord, selection in soltab.getValuesIter(returnAxes=axisDiff+axisInTable+axisInCol+axesInPlot):
# set filename
filename = ''
for axis in axesInFile:
filename += axis+str(coord[axis])+'_'
filename = filename[:-1] # remove last _
if prefix+filename == '': filename = 'plot'
# axis vals (they are always the same, regulat arrays)
xvals = coord[axesInPlot[0]]
# if plotting antenna - convert to number
if axesInPlot[0] == 'ant':
xvals = np.arange(len(xvals))
# if plotting time - convert in h/min/s
xlabelunit=''
if axesInPlot[0] == 'time':
if xvals[-1] - xvals[0] > 3600:
xvals = (xvals-xvals[0])/3600. # hrs
xlabelunit = ' [hr]'
elif xvals[-1] - xvals[0] > 60:
xvals = (xvals-xvals[0])/60. # mins
xlabelunit = ' [min]'
else:
xvals = (xvals-xvals[0]) # sec
xlabelunit = ' [s]'
# if plotting freq convert in MHz
elif axesInPlot[0] == 'freq':
xvals = xvals/1.e6 # MHz
xlabelunit = ' [MHz]'
if cmesh:
# axis vals (they are always the same, regular arrays)
yvals = coord[axesInPlot[1]]
# same as above but for y-axis
if axesInPlot[1] == 'ant':
yvals = np.arange(len(yvals))
if len(xvals) <= 1 or len(yvals) <=1:
logging.error('3D plot must have more then one value per axes.')
mpm.wait()
return 1
ylabelunit=''
if axesInPlot[1] == 'time':
if yvals[-1] - yvals[0] > 3600:
yvals = (yvals-yvals[0])/3600. # hrs
ylabelunit = ' [hr]'
elif yvals[-1] - yvals[0] > 60:
yvals = (yvals-yvals[0])/60. # mins
ylabelunit = ' [min]'
else:
yvals = (yvals-yvals[0]) # sec
ylabelunit = ' [s]'
elif axesInPlot[1] == 'freq': # Mhz
yvals = yvals/1.e6
ylabelunit = ' [MHz]'
else:
yvals = None
if datatype == 'clock':
datatype = 'Clock'
ylabelunit = ' (s)'
elif datatype == 'tec':
datatype = 'dTEC'
ylabelunit = ' (TECU)'
elif datatype == 'rotationmeasure':
datatype = 'dRM'
ylabelunit = r' (rad m$^{-2}$)'
elif datatype == 'tec3rd':
datatype = r'dTEC$_3$'
ylabelunit = r' (rad m$^{-3}$)'
else:
ylabelunit = ''
# cycle on tables
soltab1Selection = soltab.selection # save global selection and subselect only axex to iterate
soltab.selection = selection
titles = []
for Ntab, (vals, coord, selection) in enumerate(soltab.getValuesIter(returnAxes=axisDiff+axisInCol+axesInPlot)):
# set tile
titles.append('')
for axis in coord:
if axis in axesInFile+axesInPlot+axisInCol: continue
titles[Ntab] += axis+':'+str(coord[axis])+' '
titles[Ntab] = titles[Ntab][:-1] # remove last ' '
# cycle on colors
soltab2Selection = soltab.selection
soltab.selection = selection
for Ncol, (vals, weight, coord, selection) in enumerate(soltab.getValuesIter(returnAxes=axisDiff+axesInPlot, weight=True, reference=refAnt)):
# differential plot
if axisDiff != []:
# find ordered list of axis
names = [axis for axis in soltab.getAxesNames() if axis in axisDiff+axesInPlot]
if axisDiff[0] not in names:
logging.error("Axis to differentiate (%s) not found." % axisDiff[0])
mpm.wait()
return 1
if len(coord[axisDiff[0]]) != 2:
logging.error("Axis to differentiate (%s) has too many values, only 2 is allowed." % axisDiff[0])
mpm.wait()
return 1
# find position of interesting axis
diff_idx = names.index(axisDiff[0])
# roll to first place
vals = np.rollaxis(vals,diff_idx,0)
vals = vals[0] - vals[1]
weight = np.rollaxis(weight,diff_idx,0)
weight[0][ weight[1]==0 ] = 0
weight = weight[0]
del coord[axisDiff[0]]
# add tables if required (e.g. phase/tec)
for soltabToAdd in soltabsToAdd:
logging.warning('soltabsToAdd not implemented. Ignoring.')
# newCoord = {}
# for axisName in coord.keys():
# # prepare selected on present axes
# if axisName in soltabToAdd.getAxesNames():
# if type(coord[axisName]) is np.ndarray:
# newCoord[axisName] = coord[axisName]
# else:
# newCoord[axisName] = [coord[axisName]] # avoid being interpreted as regexp, faster
#
# soltabToAdd.setSelection(**newCoord)
# valsAdd = np.squeeze(soltabToAdd.getValues(retAxesVals=False, weight=False, reference=refAnt))
#
# # add missing axes
# print ('shape:', vals.shape)
# for axisName in coord.keys():
# if not axisName in soltabToAdd.getAxesNames():
# # find axis positions
# axisPos = soltab.getAxesNames().index(axisName)
# # create a new axes for the table to add and duplicate the values
# valsAdd = np.expand_dims(valsAdd, axisPos)
# print ('shape to add:', valsAdd.shape)
#
# if soltabToAdd.getType() == 'clock':
# valsAdd = 2. * np.pi * valsAdd * coord['freq']
# elif soltabToAdd.getType() == 'tec':
# valsAdd = -8.44797245e9 * valsAdd / coord['freq']
# else:
# logging.warning('Only Clock or TEC can be added to solutions. Ignoring: '+soltabToAdd.getType()+'.')
# continue
#
# if valsAdd.shape != vals.shape:
# logging.error('Cannot combine the table '+soltabToAdd.getType()+' with '+soltab.getType()+'. Wrong shape.')
# mpm.wait()
# return 1
#
# vals += valsAdd
# normalize
if (soltab.getType() == 'phase' or soltab.getType() == 'scalarphase'):
vals = normalize_phase(vals)
if (soltab.getType() == 'rotation'):
vals = np.mod(vals + np.pi/2., np.pi) - np.pi/2.
# is user requested axis in an order that is different from h5parm, we need to transpose
if cmesh:
if soltab.getAxesNames().index(axesInPlot[0]) < soltab.getAxesNames().index(axesInPlot[1]):
vals = vals.T
weight = weight.T
# unwrap if required
if (soltab.getType() == 'phase' or soltab.getType() == 'scalarphase') and doUnwrap:
if len(axesInPlot) == 1:
vals = unwrap(vals)
else:
flags = np.array((weight == 0), dtype=bool)
if not (flags == True).all():
vals = unwrap_2d(vals, flags, coord[axesInPlot[0]], coord[axesInPlot[1]])
dataCube[Ntab,Ncol] = vals
sel1 = np.where(weight == 0.)
sel2 = np.where(np.isnan(vals))
if cmesh:
dataCube[Ntab,Ncol,sel1[0],sel1[1]] = np.ma.masked
dataCube[Ntab,Ncol,sel2[0],sel2[1]] = np.ma.masked
else:
dataCube[Ntab,Ncol,sel1[0]] = np.ma.masked
dataCube[Ntab,Ncol,sel2[0]] = np.ma.masked
soltab.selection = soltab2Selection
### end cycle on colors
# if dataCube too large (> 500 MB) do not go parallel
if np.array(dataCube).nbytes > 1024*1024*500:
logging.debug('Big plot, parallel not possible.')
_plot(Nplots, NColFig, figSize, markerSize, cmesh, axesInPlot, axisInTable, xvals, yvals, xlabelunit, ylabelunit, datatype, prefix+filename, titles, log, dataCube, minZ, maxZ, plotFlag, makeMovie, antCoords, None)
else:
mpm.put([Nplots, NColFig, figSize, markerSize, cmesh, axesInPlot, axisInTable, xvals, yvals, xlabelunit, ylabelunit, datatype, prefix+filename, titles, log, np.ma.copy(dataCube), minZ, maxZ, plotFlag, makeMovie, antCoords])
if makeMovie: pngs.append(prefix+filename+'.png')
soltab.selection = soltab1Selection
### end cycle on tables
mpm.wait()
if makeMovie:
def long_substr(strings):
"""
Find longest common substring
"""
substr = ''
if len(strings) > 1 and len(strings[0]) > 0:
for i in range(len(strings[0])):
for j in range(len(strings[0])-i+1):
if j > len(substr) and all(strings[0][i:i+j] in x for x in strings):
substr = strings[0][i:i+j]
return substr
movieName = long_substr(pngs)
assert movieName != '' # need a common prefix, use prefix keyword in case
logging.info('Making movie: '+movieName)
# make every movie last 20 sec, min one second per slide
fps = np.ceil(len(pngs)/200.)
ss="mencoder -ovc lavc -lavcopts vcodec=mpeg4:vpass=1:vbitrate=6160000:mbd=2:keyint=132:v4mv:vqmin=3:lumi_mask=0.07:dark_mask=0.2:"+\
"mpeg_quant:scplx_mask=0.1:tcplx_mask=0.1:naq -mf type=png:fps="+str(fps)+" -nosound -o "+movieName.replace('__tmp__','')+".mpg mf://"+movieName+"* > mencoder.log 2>&1"
os.system(ss)
#for png in pngs: os.system('rm '+png)
return 0