Example #1
0
def streamPowerSpecTest(ds,  outputFolder="", vmin=-1, vmax=5, 
                        prefilterSigma=0, *args, **kwargs):
    if outputFolder =="":
        outputFolder=ds.outputFolder
    N = len(ds)
    Ztotal=0
    Zmax  =0
    for a in ds:
        a.load()
        a1 = a.getWRFwindow()
        #debug
        a1.show()
        time.sleep(5)

        try:
            XYZs = powerSpecTest(a1, outputFolder=outputFolder, prefilterSigma=prefilterSigma, *args, **kwargs)
        except:
            print 'ERROR!  "XYZs = powerSpecTest(a1, outputFolder=outputFolder, *args, **kwargs)" <-- ' + a.name
        XYZmax  = XYZs['XYZmax']
        XYZtotal= XYZs['XYZtotal']
        Zmax   += XYZmax['Z']
        Ztotal += XYZtotal['Z']
        a.matrix = np.ma.array([0]) # unload
    Zmax    /= N
    Ztotal  /= N

    XYZmax['Z']     = Zmax
    XYZtotal['Z']   = Ztotal
    plt.close()
    print "***************************************************************************************************************"    
    print "***************************************************************************************************************"    
    print "* *    Zmax, Ztotal:", Zmax, Ztotal
    print "***************************************************************************************************************"
    print "***************************************************************************************************************"
    print "sleep 5 seconds"
    time.sleep(5)

    XYZ3dMax = spectrum3d.spectrum3d(XYZ=XYZmax, outputFolder=outputFolder, fileName=str(time.time())+ ds.name+ "_mean_maxSpec3d.png",
                                 title= ("Mean Max 3d Spectrum from %d Images: " %N) + ds.name, display=True, vmin=vmin, vmax=vmax,**kwargs)
    plt.close()
    XYZ3dTotal = spectrum3d.spectrum3d(XYZ=XYZtotal, outputFolder=outputFolder, fileName=str(time.time())+ds.name+ "_mean_totalSpec3d.png",
                                 title= ("Mean Total 3d Spectrum from %d Images: " % N) + ds.name, display=True, vmin=vmin, vmax=vmax,**kwargs)

    #   contourplots
    plt.close()
    XYZcontourMax = specContour.specContour(XYZ=XYZmax, outputFolder=outputFolder, fileName=str(time.time())+ds.name+ "_mean_maxSpecContour.png",
                                 title= "Mean Max Spectrum: " + ds.name, display=True, vmin=vmin, vmax=vmax,**kwargs)
    plt.close()

    XYZcontourTotal = specContour.specContour(XYZ=XYZtotal, outputFolder=outputFolder, fileName=str(time.time())+ds.name+ "_mean_totalSpecContour.png",
                                 title= "Mean Total Spectrum: " + ds.name, display=True,vmin=vmin, vmax=vmax, **kwargs)
    plt.close()

    returnValues= {'XYZmax'    :   XYZmax,
                    'XYZtotal'  :   XYZtotal,
                    'ds'        : ds.dataFolder,
                    }
        
    pickle.dump(returnValues, open(outputFolder+str(time.time())+'dbzstreamSpec_returnValues.pydump','w'))
    return returnValues
Example #2
0
def crossStreamsPowerSpecTest(ds1, ds2, outputFolder="", crossContourVmax=1, vmin=-1, vmax=5,crossContourVmin=-1, 
                              prefilterSigma1=0, prefilterSigma2=0, #2014-08-05
                              *args, **kwargs):
    """ 2014-07-17
    from armor.initialise import *; march.list=[v for v in march.list if '0311.1200' in v.dataTime or '0311.1230' in v.dataTime] ; marchwrf.list=[v for v in marchwrf.list if '0311.12' in v.dataTime and ('WRF01' in v.name or 'WRF02' in v.name)] ; from armor import analysis as an; res = an.crossStreamsPowerSpecTest(marchwrf,march, outputFolder='testing/')

    """
    plt.close()
    res1 = streamPowerSpecTest(ds1,  outputFolder=outputFolder, vmin=vmin, vmax=vmax, prefilterSigma=prefilterSigma1, *args, **kwargs)
    plt.close()
    res2 = streamPowerSpecTest(ds2,  outputFolder=outputFolder, vmin=vmin, vmax=vmax, prefilterSigma=prefilterSigma2, *args, **kwargs)


    XYZmax1 = res1['XYZmax']
    XYZmax2 = res2['XYZmax']
    XYZtotal1 = res1['XYZtotal']
    XYZtotal2 = res2['XYZtotal']

    if outputFolder =="":
        outputFolder = ds1.outputFolder
    
    #   contourplots
    plt.close()

    crossContourMax = specContour.specContour(XYZmax1,XYZmax2 ,outputFolder=outputFolder, fileName=str(time.time())+ds1.name+ "_versus_" + ds2.name + "_maxSpecContour.png",
                                 title= "Max Spectrum: " + ds2.name+ " (Red) - " +ds1.name , vmax=crossContourVmax, vmin=crossContourVmin, display=True)
    plt.close()

    crossContourTotal = specContour.specContour(XYZtotal1, XYZtotal2, outputFolder=outputFolder, fileName=str(time.time())+ds1.name+ "_versus_" + ds2.name + "_totalSpecContour.png",
                                 title= "Total Spectrum: " + ds2.name+ "(Red) - " +ds1.name, vmax=crossContourVmax, vmin=crossContourVmin, display=True)
    plt.close()

    returnValues= {'crossContourMax':crossContourMax, 
                'crossContourTotal':crossContourTotal,
                'ds1': ds1.dataFolder,
                'ds2': ds2.dataFolder,
                }

    pickle.dump(returnValues, open(outputFolder+str(time.time())+'crossSpec_returnValues.pydump','w'))
    return returnValues
Example #3
0
def crossStreamsPowerSpecTest2(ds1, ds2, numberOfShuffles=0, numberOfTrials=100, randomise=True, outputFolder="", 
                                prefilterSigmas=(0,0),               #2014-08-05
                                crossContourVmax=1, vmin=-1, vmax=5,crossContourVmin=-1, *args, **kwargs):
    """ 2014-07-31
    rewriting crossStreamsPowerSpecTest
        with streamPowerSpecTest
    with intermediate results outputted as soon as possible
    """
    timeString = str(int(time.time()))
    plt.close()
    if outputFolder =="":
        outputFolder=ds.outputFolder
    #res1 = streamPowerSpecTest(ds1,  outputFolder=outputFolder, vmin=vmin, vmax=vmax,*args, **kwargs)
    #plt.close()
    #res2 = streamPowerSpecTest(ds2,  outputFolder=outputFolder, vmin=vmin, vmax=vmax,*args, **kwargs)
####################
    ds = [ds1, ds2]
    Ns = [len(ds1), len(ds2)]
    Ztotals = [0,0]
    Zmaxes  = [0,0]
    Ztotals = [0,0]
    XYZmaxes= [{},{}]
    XYZtotals =[{},{}]

    ds1.shuffle(numberOfShuffles)
    ds2.shuffle(numberOfShuffles)

    for i in range(numberOfTrials):
        if randomise:
            rns = [int(np.random.random() * v )for v in Ns]
        else:
            rns = [i % v for v in Ns]
        print '\n......................\n'
        print i
        for j in [0, 1]:
            a = ds[j][rns[j]]
            print a.name
            a.load()
            a1 = a.getWRFwindow()
            try:
                XYZs = powerSpecTest(a1, outputFolder=outputFolder, prefilterSigma=prefilterSigmas[j], *args, **kwargs)
            except AttributeError:
                print 'ERROR!  "XYZs = powerSpecTest(a1, outputFolder=outputFolder, *args, **kwargs)" <-- ' + a.name
                continue
            XYZmaxes[j]  = XYZs['XYZmax']
            XYZtotals[j] = XYZs['XYZtotal']
            Zmaxes[j]  += XYZmaxes[j]['Z']
            Ztotals[j] += XYZtotals[j]['Z']
            a.matrix = np.ma.array([0]) # unload
            print "***************************************************************************************************************"    
            print "***************************************************************************************************************"    
            print ds[j][rns[j]].name
            print "* *    Zmax, Ztotal:", Zmaxes[j], Ztotals[j]
            print "***************************************************************************************************************"
            print "***************************************************************************************************************"
            time.sleep(1)
            XYZmaxes[j]['Z']     = Zmaxes[j].copy() /(i+1)
            XYZtotals[j]['Z']    = Ztotals[j].copy() /(i+1)
            np.savetxt(outputFolder+"Average"+ds[j].name + "_maxSpec(%d).csv" %(i+1), XYZmaxes[j]['Z'], delimiter=",")
            np.savetxt(outputFolder+"Average"+ds[j].name + "_totalSpec(%d).csv"%(i+1), XYZtotals[j]['Z'] , delimiter=",")
            
        #   contourplots
        try:
            plt.close()
            crossContourMax = specContour.specContour(XYZmaxes[0],XYZmaxes[1] ,outputFolder=outputFolder, fileName= "Average"+ds1.name+ "_versus_" + ds2.name + "_maxSpecContour(%d).png" %(i+1),
                                         title= "Max Spectrum(%d): " %(i+1) + ds2.name+ " (Red) - " +ds1.name , vmax=crossContourVmax, vmin=crossContourVmin, display=True)
        except KeyError:
            print "KeyError!"

        try:
            plt.close()
            crossContourTotal = specContour.specContour(XYZtotals[0], XYZtotals[1], outputFolder=outputFolder, fileName="Average"+ds1.name+ "_versus_" + ds2.name + "_totalSpecContour(%d).png" %(i+1),
                                         title= "Total Spectrum(%d): "%(i+1) + ds2.name+ "(Red) - " +ds1.name, vmax=crossContourVmax, vmin=crossContourVmin, display=True)
        except KeyError:
            print "KeyError!"

        plt.close()

    
        returnValues= {'crossContourMax':crossContourMax, 
                    'crossContourTotal':crossContourTotal,
                    'ds1': ds1.dataFolder,
                    'ds2': ds2.dataFolder,
                    }
    
        pickle.dump(returnValues, open(outputFolder+ timeString +'crossSpec_returnValues.pydump','w'))
    return returnValues
Example #4
0
def powerSpecTest(a, outputFolder="", 
                sigmas  = [1, 2, 4, 5, 8 ,10 ,16, 20, 32, 40, 64, 80, 128],
                bins=[0.01, 0.03, 0.1, 0.3, 1., 3., 10., 30.,100.],
                vmin=-1, vmax=5,
                prefilterSigma=0, #2014-08-05
                *args, **kwargs):
    """
    2014-07-17
    
    """
    from graphics import spectrum3d
    from graphics import specContour
    timeStamp = str(int(time.time()))
    if outputFolder =="":
        outputFolder = a.outputFolder
    if not os.path.exists(outputFolder):
        os.makedirs(outputFolder)

    plt.close()
    #   save the original image
    ##################
    #   debug
    print a.name, a.matrix.shape
    a.show()
    #
    ###################
    a.saveImage(outputFolder+str(time.time())+a.name+'.png')
    if prefilterSigma > 0:
        print "Prefiltering", a.name, "with sigma =", prefilterSigma
        a.gaussianFilter(prefilterSigma)   #2014-08-05    
    psResults = a.powerSpec(outputFolder=outputFolder, toPlot3d=True, toPlotContours=True, toReload=True, 
                            sigmas=sigmas, bins=bins,
                            *args, **kwargs)
    #   save the response for each gaussian filter with each sigma
    responseImages= psResults['responseImages']
    sigmas        = psResults['sigmas']
    maxSpec       = psResults['maxSpec']
    XYZmax        = psResults['XYZmax']
    XYZtotal      = psResults['XYZtotal']
    height, width, depth = responseImages.shape
    for i in range(depth):
        resp = responseImages[:,:,i]
        plt.close()
        plt.imshow(resp, origin='lower', cmap='jet',)
        plt.colorbar()
        plt.title("Filter Intensity for sigma=%d" % sigmas[i])
        plt.savefig(outputFolder+ str(time.time()) + a.name + "_LOG_sigma%d.png" %sigmas[i])

    #   save the max response sigma (2d maxspec)                            -DONE ("~LOG_numerical_spec.png")
    #   save the max responses for each max response sigma for each point  - DONE ("~_max_response.png")
    #   save all the responses                                           - DONE ("~responseImagesList.pydump")
    #   save the max spec data
    pickle.dump(maxSpec, open(outputFolder+ str(time.time()) + a.name + "maxSpec.pydump", 'w'))
    
    #   save the totalspec data                                         -pass (too big)
    #   3d plots
    plt.close()
    XYZ3dMax = spectrum3d.spectrum3d(XYZ=XYZmax, outputFolder=outputFolder, fileName=str(time.time())+a.name + "_maxSpec3d.png",
                                 title= a.name+"Max 3d Spectrum", display=True, vmin=vmin, vmax=vmax,**kwargs)
    plt.close()
    XYZ3dTotal = spectrum3d.spectrum3d(XYZ=XYZtotal, outputFolder=outputFolder, fileName=str(time.time())+a.name + "_totalSpec3d.png",
                                 title= a.name+"Total 3d Spectrum", display=True,vmin=vmin, vmax=vmax, **kwargs)

    #   contourplots

    plt.close()
    XYZcontourMax = specContour.specContour(XYZ=XYZmax, outputFolder=outputFolder, fileName=str(time.time())+a.name + "_maxSpecContour.png",
                                 title= a.name+"Max Spectrum Contours", display=True, vmin=vmin, vmax=vmax,**kwargs)
    plt.close()
    XYZcontourTotal = specContour.specContour(XYZ=XYZtotal, outputFolder=outputFolder, fileName=str(time.time())+a.name + "_totalSpecContour.png",
                                 title= a.name+"Total Spectrum Contours", display=True,vmin=vmin, vmax=vmax, **kwargs)

    #   create tables
    plt.close()
    np.savetxt(outputFolder+str(time.time())+a.name + "_maxSpec.csv", XYZmax['Z'], delimiter=",")
    np.savetxt(outputFolder+str(time.time())+a.name + "_totalSpec.csv", XYZtotal['Z'],delimiter=",")


    #   XYZ max spec dump                                               - DONE ("~XYZmax.pydump")
    #   XYZ total spec dump                                              - DONE ("~XYZ.pydump")

    return {'XYZ3dMax'      : XYZ3dMax,
            'XYZ3dTotal'    : XYZ3dTotal,
            'XYZcontourMax' : XYZcontourMax,
            'XYZcontourTotal':XYZcontourTotal,
            'XYZmax':XYZcontourMax,
            'XYZtotal':XYZcontourTotal,
            }
Example #5
0
def powerSpec(a, b="", thres=0, outputFolder="", toReload=False, 
             toPlotContours=True,
             toPlot3d=True,
            #spectrumType = "numerical", 
            vmin="",
            vmax="",
            **kwargs):
    """
    updated 2014-07-03 
        including the new 3dplotting function from lin yen ting
        armor.graphics.spectrum3d
    new pipeline:
        WRF/RADAR  ->   response layers for various sigmas -> 1. max spec map
                                                              2. max internsity map
                                                              3. convolution intensity range for each sigma

                                                           -> 1.    3D max spec chart
                                                              2.    3D total spec chart
                                                          
    
    """
    plt.close()
    if outputFolder=="":
        outputFolder= a.outputFolder
    from armor.spectral import powerSpec1 as ps1
    #getLaplacianOfGaussianSpectrum(a, sigmas=sigmas, thres=thresPreprocessing, outputFolder=outputFolder, toReload=True)
    psResults = ps1.getLaplacianOfGaussianSpectrum(a, thres=thres, outputFolder=outputFolder,
                                                     toReload=toReload, 
                                                     #spectrumType=spectrumType, 
                                                     **kwargs)

    
    # all convolution results stored in a.responseImages
    #   max spectrum:  a.LOGspec
    #   convolution intensity corresponding to a.LOGspec: a.responseMax
    
    print "Results stored in file:", outputFolder
    print "Results stored in attribute:  a.maxSpec"
    maxSpec = psResults['maxSpec']
    XYZmax  = psResults['XYZmax']
    XYZtotal= psResults['XYZtotal']
    if toPlot3d:
        spectrum3d.spectrum3d(XYZmax, outputFolder=outputFolder, fileName  = str(time.time())+ 'maxSpec3d_' + a.name+ '.png')
        spectrum3d.spectrum3d(XYZtotal, outputFolder=outputFolder, fileName= str(time.time())+ 'totalSpec3d_' + a.name+'.png')

    if b != "":
        psResults_b = powerSpec(b, thres=thres, outputFolder=outputFolder, toReload=toReload, 
            #spectrumType = "numerical", 
            toPlotContours=toPlotContours, #2014-07-08
            toPlot3d=toPlot3d,
            vmin=vmin,
            vmax=vmax,
            **kwargs)
        XYZmax2     = psResults_b['XYZmax']
        XYZtotal2   = psResults_b['XYZtotal']
        fileName1   = str(time.time())+ "maxSpec_" + a.name + "_" + b.name + ".png"
        fileName2   = str(time.time())+ "totalSpec_" + a.name + "_" + b.name + ".png"
        if toPlotContours:
            try:
                plt.close()
                XYZ1 = XYZmax
                XYZ2 = XYZmax2
                if not XYZmax['Z'].max() <= 0 or not XYZmax2['Z'].max() <= 0:
                    specContour.specContour(XYZ1, XYZ2, outputFolder=outputFolder, fileName=fileName1,
                                                        vmin=vmin,
                                                        vmax=vmax,)
                else:
                    pass
                XYZ1 = XYZtotal
                XYZ2 = XYZtotal2
                plt.close()
                if not XYZtotal['Z'].max() <= 0 or not XYZtotal2['Z'].max() <= 0:
                    specContour.specContour(XYZ1, XYZ2, outputFolder=outputFolder, fileName=fileName2,
                                                        vmin=vmin,
                                                        vmax=vmax,)
                else:
                    pass
            except:
                print "Contour plot failure due to input data max = %f" % XYZmax['Z'].max()
                os.system("pause")
    # debug
    a.XYZmax = XYZmax
    a.XYZtotal = XYZtotal
    # end debug
    fileName1   = str(time.time())+ "maxSpec_" + a.name + ".png"
    fileName2   = str(time.time())+ "totalSpec_" + a.name + ".png"
    plt.close()
    specContour.specContour(XYZmax,  outputFolder=outputFolder, fileName=fileName1, vmin=vmin, vmax=vmax)   
    plt.close() 
    specContour.specContour(XYZtotal,  outputFolder=outputFolder, fileName=fileName2, vmin=vmin, vmax=vmax)    
    
    if toPlotContours:
        try:
            plt.close()
            XYZ1=XYZtotal
            XYZ2=XYZmax

            specContour.specContour(XYZ1, XYZ2, outputFolder=outputFolder, fileName=fileName2,
                                                        vmin=vmin,
                                                        vmax=vmax,)
        except:
            print "function specContour.specContour() failed!!"
            return {'XYZtotal': XYZtotal, 'XYZmax': XYZmax}
    #specContour.specContour(XYZ=XYZmax,  display=True)
    #specContour.specContour(XYZ=XYZmax,  display=True)
    return psResults