def streamPowerSpecTest(ds, outputFolder="", vmin=-1, vmax=5,
prefilterSigma=0, *args, **kwargs):
if outputFolder =="":
outputFolder=ds.outputFolder
N = len(ds)
Ztotal=0
Zmax =0
for a in ds:
a.load()
a1 = a.getWRFwindow()
#debug
a1.show()
time.sleep(5)
try:
XYZs = powerSpecTest(a1, outputFolder=outputFolder, prefilterSigma=prefilterSigma, *args, **kwargs)
except:
print 'ERROR! "XYZs = powerSpecTest(a1, outputFolder=outputFolder, *args, **kwargs)" <-- ' + a.name
XYZmax = XYZs['XYZmax']
XYZtotal= XYZs['XYZtotal']
Zmax += XYZmax['Z']
Ztotal += XYZtotal['Z']
a.matrix = np.ma.array([0]) # unload
Zmax /= N
Ztotal /= N
XYZmax['Z'] = Zmax
XYZtotal['Z'] = Ztotal
plt.close()
print "***************************************************************************************************************"
print "***************************************************************************************************************"
print "* * Zmax, Ztotal:", Zmax, Ztotal
print "***************************************************************************************************************"
print "***************************************************************************************************************"
print "sleep 5 seconds"
time.sleep(5)
XYZ3dMax = spectrum3d.spectrum3d(XYZ=XYZmax, outputFolder=outputFolder, fileName=str(time.time())+ ds.name+ "_mean_maxSpec3d.png",
title= ("Mean Max 3d Spectrum from %d Images: " %N) + ds.name, display=True, vmin=vmin, vmax=vmax,**kwargs)
plt.close()
XYZ3dTotal = spectrum3d.spectrum3d(XYZ=XYZtotal, outputFolder=outputFolder, fileName=str(time.time())+ds.name+ "_mean_totalSpec3d.png",
title= ("Mean Total 3d Spectrum from %d Images: " % N) + ds.name, display=True, vmin=vmin, vmax=vmax,**kwargs)
# contourplots
plt.close()
XYZcontourMax = specContour.specContour(XYZ=XYZmax, outputFolder=outputFolder, fileName=str(time.time())+ds.name+ "_mean_maxSpecContour.png",
title= "Mean Max Spectrum: " + ds.name, display=True, vmin=vmin, vmax=vmax,**kwargs)
plt.close()
XYZcontourTotal = specContour.specContour(XYZ=XYZtotal, outputFolder=outputFolder, fileName=str(time.time())+ds.name+ "_mean_totalSpecContour.png",
title= "Mean Total Spectrum: " + ds.name, display=True,vmin=vmin, vmax=vmax, **kwargs)
plt.close()
returnValues= {'XYZmax' : XYZmax,
'XYZtotal' : XYZtotal,
'ds' : ds.dataFolder,
}
pickle.dump(returnValues, open(outputFolder+str(time.time())+'dbzstreamSpec_returnValues.pydump','w'))
return returnValues
def crossStreamsPowerSpecTest(ds1, ds2, outputFolder="", crossContourVmax=1, vmin=-1, vmax=5,crossContourVmin=-1,
prefilterSigma1=0, prefilterSigma2=0, #2014-08-05
*args, **kwargs):
""" 2014-07-17
from armor.initialise import *; march.list=[v for v in march.list if '0311.1200' in v.dataTime or '0311.1230' in v.dataTime] ; marchwrf.list=[v for v in marchwrf.list if '0311.12' in v.dataTime and ('WRF01' in v.name or 'WRF02' in v.name)] ; from armor import analysis as an; res = an.crossStreamsPowerSpecTest(marchwrf,march, outputFolder='testing/')
"""
plt.close()
res1 = streamPowerSpecTest(ds1, outputFolder=outputFolder, vmin=vmin, vmax=vmax, prefilterSigma=prefilterSigma1, *args, **kwargs)
plt.close()
res2 = streamPowerSpecTest(ds2, outputFolder=outputFolder, vmin=vmin, vmax=vmax, prefilterSigma=prefilterSigma2, *args, **kwargs)
XYZmax1 = res1['XYZmax']
XYZmax2 = res2['XYZmax']
XYZtotal1 = res1['XYZtotal']
XYZtotal2 = res2['XYZtotal']
if outputFolder =="":
outputFolder = ds1.outputFolder
# contourplots
plt.close()
crossContourMax = specContour.specContour(XYZmax1,XYZmax2 ,outputFolder=outputFolder, fileName=str(time.time())+ds1.name+ "_versus_" + ds2.name + "_maxSpecContour.png",
title= "Max Spectrum: " + ds2.name+ " (Red) - " +ds1.name , vmax=crossContourVmax, vmin=crossContourVmin, display=True)
plt.close()
crossContourTotal = specContour.specContour(XYZtotal1, XYZtotal2, outputFolder=outputFolder, fileName=str(time.time())+ds1.name+ "_versus_" + ds2.name + "_totalSpecContour.png",
title= "Total Spectrum: " + ds2.name+ "(Red) - " +ds1.name, vmax=crossContourVmax, vmin=crossContourVmin, display=True)
plt.close()
returnValues= {'crossContourMax':crossContourMax,
'crossContourTotal':crossContourTotal,
'ds1': ds1.dataFolder,
'ds2': ds2.dataFolder,
}
pickle.dump(returnValues, open(outputFolder+str(time.time())+'crossSpec_returnValues.pydump','w'))
return returnValues
def crossStreamsPowerSpecTest2(ds1, ds2, numberOfShuffles=0, numberOfTrials=100, randomise=True, outputFolder="",
prefilterSigmas=(0,0), #2014-08-05
crossContourVmax=1, vmin=-1, vmax=5,crossContourVmin=-1, *args, **kwargs):
""" 2014-07-31
rewriting crossStreamsPowerSpecTest
with streamPowerSpecTest
with intermediate results outputted as soon as possible
"""
timeString = str(int(time.time()))
plt.close()
if outputFolder =="":
outputFolder=ds.outputFolder
#res1 = streamPowerSpecTest(ds1, outputFolder=outputFolder, vmin=vmin, vmax=vmax,*args, **kwargs)
#plt.close()
#res2 = streamPowerSpecTest(ds2, outputFolder=outputFolder, vmin=vmin, vmax=vmax,*args, **kwargs)
####################
ds = [ds1, ds2]
Ns = [len(ds1), len(ds2)]
Ztotals = [0,0]
Zmaxes = [0,0]
Ztotals = [0,0]
XYZmaxes= [{},{}]
XYZtotals =[{},{}]
ds1.shuffle(numberOfShuffles)
ds2.shuffle(numberOfShuffles)
for i in range(numberOfTrials):
if randomise:
rns = [int(np.random.random() * v )for v in Ns]
else:
rns = [i % v for v in Ns]
print '\n......................\n'
print i
for j in [0, 1]:
a = ds[j][rns[j]]
print a.name
a.load()
a1 = a.getWRFwindow()
try:
XYZs = powerSpecTest(a1, outputFolder=outputFolder, prefilterSigma=prefilterSigmas[j], *args, **kwargs)
except AttributeError:
print 'ERROR! "XYZs = powerSpecTest(a1, outputFolder=outputFolder, *args, **kwargs)" <-- ' + a.name
continue
XYZmaxes[j] = XYZs['XYZmax']
XYZtotals[j] = XYZs['XYZtotal']
Zmaxes[j] += XYZmaxes[j]['Z']
Ztotals[j] += XYZtotals[j]['Z']
a.matrix = np.ma.array([0]) # unload
print "***************************************************************************************************************"
print "***************************************************************************************************************"
print ds[j][rns[j]].name
print "* * Zmax, Ztotal:", Zmaxes[j], Ztotals[j]
print "***************************************************************************************************************"
print "***************************************************************************************************************"
time.sleep(1)
XYZmaxes[j]['Z'] = Zmaxes[j].copy() /(i+1)
XYZtotals[j]['Z'] = Ztotals[j].copy() /(i+1)
np.savetxt(outputFolder+"Average"+ds[j].name + "_maxSpec(%d).csv" %(i+1), XYZmaxes[j]['Z'], delimiter=",")
np.savetxt(outputFolder+"Average"+ds[j].name + "_totalSpec(%d).csv"%(i+1), XYZtotals[j]['Z'] , delimiter=",")
# contourplots
try:
plt.close()
crossContourMax = specContour.specContour(XYZmaxes[0],XYZmaxes[1] ,outputFolder=outputFolder, fileName= "Average"+ds1.name+ "_versus_" + ds2.name + "_maxSpecContour(%d).png" %(i+1),
title= "Max Spectrum(%d): " %(i+1) + ds2.name+ " (Red) - " +ds1.name , vmax=crossContourVmax, vmin=crossContourVmin, display=True)
except KeyError:
print "KeyError!"
try:
plt.close()
crossContourTotal = specContour.specContour(XYZtotals[0], XYZtotals[1], outputFolder=outputFolder, fileName="Average"+ds1.name+ "_versus_" + ds2.name + "_totalSpecContour(%d).png" %(i+1),
title= "Total Spectrum(%d): "%(i+1) + ds2.name+ "(Red) - " +ds1.name, vmax=crossContourVmax, vmin=crossContourVmin, display=True)
except KeyError:
print "KeyError!"
plt.close()
returnValues= {'crossContourMax':crossContourMax,
'crossContourTotal':crossContourTotal,
'ds1': ds1.dataFolder,
'ds2': ds2.dataFolder,
}
pickle.dump(returnValues, open(outputFolder+ timeString +'crossSpec_returnValues.pydump','w'))
return returnValues
def powerSpecTest(a, outputFolder="",
sigmas = [1, 2, 4, 5, 8 ,10 ,16, 20, 32, 40, 64, 80, 128],
bins=[0.01, 0.03, 0.1, 0.3, 1., 3., 10., 30.,100.],
vmin=-1, vmax=5,
prefilterSigma=0, #2014-08-05
*args, **kwargs):
"""
2014-07-17
"""
from graphics import spectrum3d
from graphics import specContour
timeStamp = str(int(time.time()))
if outputFolder =="":
outputFolder = a.outputFolder
if not os.path.exists(outputFolder):
os.makedirs(outputFolder)
plt.close()
# save the original image
##################
# debug
print a.name, a.matrix.shape
a.show()
#
###################
a.saveImage(outputFolder+str(time.time())+a.name+'.png')
if prefilterSigma > 0:
print "Prefiltering", a.name, "with sigma =", prefilterSigma
a.gaussianFilter(prefilterSigma) #2014-08-05
psResults = a.powerSpec(outputFolder=outputFolder, toPlot3d=True, toPlotContours=True, toReload=True,
sigmas=sigmas, bins=bins,
*args, **kwargs)
# save the response for each gaussian filter with each sigma
responseImages= psResults['responseImages']
sigmas = psResults['sigmas']
maxSpec = psResults['maxSpec']
XYZmax = psResults['XYZmax']
XYZtotal = psResults['XYZtotal']
height, width, depth = responseImages.shape
for i in range(depth):
resp = responseImages[:,:,i]
plt.close()
plt.imshow(resp, origin='lower', cmap='jet',)
plt.colorbar()
plt.title("Filter Intensity for sigma=%d" % sigmas[i])
plt.savefig(outputFolder+ str(time.time()) + a.name + "_LOG_sigma%d.png" %sigmas[i])
# save the max response sigma (2d maxspec) -DONE ("~LOG_numerical_spec.png")
# save the max responses for each max response sigma for each point - DONE ("~_max_response.png")
# save all the responses - DONE ("~responseImagesList.pydump")
# save the max spec data
pickle.dump(maxSpec, open(outputFolder+ str(time.time()) + a.name + "maxSpec.pydump", 'w'))
# save the totalspec data -pass (too big)
# 3d plots
plt.close()
XYZ3dMax = spectrum3d.spectrum3d(XYZ=XYZmax, outputFolder=outputFolder, fileName=str(time.time())+a.name + "_maxSpec3d.png",
title= a.name+"Max 3d Spectrum", display=True, vmin=vmin, vmax=vmax,**kwargs)
plt.close()
XYZ3dTotal = spectrum3d.spectrum3d(XYZ=XYZtotal, outputFolder=outputFolder, fileName=str(time.time())+a.name + "_totalSpec3d.png",
title= a.name+"Total 3d Spectrum", display=True,vmin=vmin, vmax=vmax, **kwargs)
# contourplots
plt.close()
XYZcontourMax = specContour.specContour(XYZ=XYZmax, outputFolder=outputFolder, fileName=str(time.time())+a.name + "_maxSpecContour.png",
title= a.name+"Max Spectrum Contours", display=True, vmin=vmin, vmax=vmax,**kwargs)
plt.close()
XYZcontourTotal = specContour.specContour(XYZ=XYZtotal, outputFolder=outputFolder, fileName=str(time.time())+a.name + "_totalSpecContour.png",
title= a.name+"Total Spectrum Contours", display=True,vmin=vmin, vmax=vmax, **kwargs)
# create tables
plt.close()
np.savetxt(outputFolder+str(time.time())+a.name + "_maxSpec.csv", XYZmax['Z'], delimiter=",")
np.savetxt(outputFolder+str(time.time())+a.name + "_totalSpec.csv", XYZtotal['Z'],delimiter=",")
# XYZ max spec dump - DONE ("~XYZmax.pydump")
# XYZ total spec dump - DONE ("~XYZ.pydump")
return {'XYZ3dMax' : XYZ3dMax,
'XYZ3dTotal' : XYZ3dTotal,
'XYZcontourMax' : XYZcontourMax,
'XYZcontourTotal':XYZcontourTotal,
'XYZmax':XYZcontourMax,
'XYZtotal':XYZcontourTotal,
}
def powerSpec(a, b="", thres=0, outputFolder="", toReload=False,
toPlotContours=True,
toPlot3d=True,
#spectrumType = "numerical",
vmin="",
vmax="",
**kwargs):
"""
updated 2014-07-03
including the new 3dplotting function from lin yen ting
armor.graphics.spectrum3d
new pipeline:
WRF/RADAR -> response layers for various sigmas -> 1. max spec map
2. max internsity map
3. convolution intensity range for each sigma
-> 1. 3D max spec chart
2. 3D total spec chart
"""
plt.close()
if outputFolder=="":
outputFolder= a.outputFolder
from armor.spectral import powerSpec1 as ps1
#getLaplacianOfGaussianSpectrum(a, sigmas=sigmas, thres=thresPreprocessing, outputFolder=outputFolder, toReload=True)
psResults = ps1.getLaplacianOfGaussianSpectrum(a, thres=thres, outputFolder=outputFolder,
toReload=toReload,
#spectrumType=spectrumType,
**kwargs)
# all convolution results stored in a.responseImages
# max spectrum: a.LOGspec
# convolution intensity corresponding to a.LOGspec: a.responseMax
print "Results stored in file:", outputFolder
print "Results stored in attribute: a.maxSpec"
maxSpec = psResults['maxSpec']
XYZmax = psResults['XYZmax']
XYZtotal= psResults['XYZtotal']
if toPlot3d:
spectrum3d.spectrum3d(XYZmax, outputFolder=outputFolder, fileName = str(time.time())+ 'maxSpec3d_' + a.name+ '.png')
spectrum3d.spectrum3d(XYZtotal, outputFolder=outputFolder, fileName= str(time.time())+ 'totalSpec3d_' + a.name+'.png')
if b != "":
psResults_b = powerSpec(b, thres=thres, outputFolder=outputFolder, toReload=toReload,
#spectrumType = "numerical",
toPlotContours=toPlotContours, #2014-07-08
toPlot3d=toPlot3d,
vmin=vmin,
vmax=vmax,
**kwargs)
XYZmax2 = psResults_b['XYZmax']
XYZtotal2 = psResults_b['XYZtotal']
fileName1 = str(time.time())+ "maxSpec_" + a.name + "_" + b.name + ".png"
fileName2 = str(time.time())+ "totalSpec_" + a.name + "_" + b.name + ".png"
if toPlotContours:
try:
plt.close()
XYZ1 = XYZmax
XYZ2 = XYZmax2
if not XYZmax['Z'].max() <= 0 or not XYZmax2['Z'].max() <= 0:
specContour.specContour(XYZ1, XYZ2, outputFolder=outputFolder, fileName=fileName1,
vmin=vmin,
vmax=vmax,)
else:
pass
XYZ1 = XYZtotal
XYZ2 = XYZtotal2
plt.close()
if not XYZtotal['Z'].max() <= 0 or not XYZtotal2['Z'].max() <= 0:
specContour.specContour(XYZ1, XYZ2, outputFolder=outputFolder, fileName=fileName2,
vmin=vmin,
vmax=vmax,)
else:
pass
except:
print "Contour plot failure due to input data max = %f" % XYZmax['Z'].max()
os.system("pause")
# debug
a.XYZmax = XYZmax
a.XYZtotal = XYZtotal
# end debug
fileName1 = str(time.time())+ "maxSpec_" + a.name + ".png"
fileName2 = str(time.time())+ "totalSpec_" + a.name + ".png"
plt.close()
specContour.specContour(XYZmax, outputFolder=outputFolder, fileName=fileName1, vmin=vmin, vmax=vmax)
plt.close()
specContour.specContour(XYZtotal, outputFolder=outputFolder, fileName=fileName2, vmin=vmin, vmax=vmax)
if toPlotContours:
try:
plt.close()
XYZ1=XYZtotal
XYZ2=XYZmax
specContour.specContour(XYZ1, XYZ2, outputFolder=outputFolder, fileName=fileName2,
vmin=vmin,
vmax=vmax,)
except:
print "function specContour.specContour() failed!!"
return {'XYZtotal': XYZtotal, 'XYZmax': XYZmax}
#specContour.specContour(XYZ=XYZmax, display=True)
#specContour.specContour(XYZ=XYZmax, display=True)
return psResults