Пример #1
0
def print_map(cs2cs_args, glon, glat, datm, sonpath, p, vmin, vmax): #humlon, humlat, merge, 

    try:
      # =========================================================
      print("creating kmz file ...")
      ## new way to create kml file  
      pixels = 1024 * 10
 
      fig, ax = humutils.gearth_fig(llcrnrlon=glon.min(),
                     llcrnrlat=glat.min(),
                     urcrnrlon=glon.max(),
                     urcrnrlat=glat.max(),
                     pixels=pixels)
      cs = ax.pcolormesh(glon, glat, datm, cmap='pink', vmin=vmin, vmax=vmax)
      ax.set_axis_off()
      fig.savefig(os.path.normpath(os.path.join(sonpath,'class_map'+str(p)+'.png')), transparent=True, format='png')    
    
      # =========================================================
      fig = plt.figure(figsize=(1.0, 4.0), facecolor=None, frameon=False)
      ax = fig.add_axes([0.0, 0.05, 0.2, 0.9])
      cb = fig.colorbar(cs, cax=ax)
      cb.set_label('Texture Lengthscale [m]', rotation=-90, color='k', labelpad=20)
      fig.savefig(os.path.normpath(os.path.join(sonpath,'class_legend'+str(p)+'.png')), transparent=False, format='png')  

      # =========================================================
      humutils.make_kml(llcrnrlon=glon.min(), llcrnrlat=glat.min(),
         urcrnrlon=glon.max(), urcrnrlat=glat.max(),
         figs=[os.path.normpath(os.path.join(sonpath,'class_map'+str(p)+'.png'))], 
         colorbar=os.path.normpath(os.path.join(sonpath,'class_legend'+str(p)+'.png')),
         kmzfile=os.path.normpath(os.path.join(sonpath,'class_GroundOverlay'+str(p)+'.kmz')), 
         name='Texture Lengthscale')

    except:
       print("error: map could not be created...")
Пример #2
0
def print_map(cs2cs_args, glon, glat, datm, sonpath, p, vmin, vmax): #humlon, humlat, merge, 

    try:
      # =========================================================
      print "creating kmz file ..."
      ## new way to create kml file  
      pixels = 1024 * 10
 
      fig, ax = humutils.gearth_fig(llcrnrlon=glon.min(),
                     llcrnrlat=glat.min(),
                     urcrnrlon=glon.max(),
                     urcrnrlat=glat.max(),
                     pixels=pixels)
      cs = ax.pcolormesh(glon, glat, datm, cmap='pink', vmin=vmin, vmax=vmax)
      ax.set_axis_off()
      fig.savefig(os.path.normpath(os.path.join(sonpath,'class_map'+str(p)+'.png')), transparent=True, format='png')    
    
      # =========================================================
      fig = plt.figure(figsize=(1.0, 4.0), facecolor=None, frameon=False)
      ax = fig.add_axes([0.0, 0.05, 0.2, 0.9])
      cb = fig.colorbar(cs, cax=ax)
      cb.set_label('Texture Lengthscale [m]', rotation=-90, color='k', labelpad=20)
      fig.savefig(os.path.normpath(os.path.join(sonpath,'class_legend'+str(p)+'.png')), transparent=False, format='png')  

      # =========================================================
      humutils.make_kml(llcrnrlon=glon.min(), llcrnrlat=glat.min(),
         urcrnrlon=glon.max(), urcrnrlat=glat.max(),
         figs=[os.path.normpath(os.path.join(sonpath,'class_map'+str(p)+'.png'))], 
         colorbar=os.path.normpath(os.path.join(sonpath,'class_legend'+str(p)+'.png')),
         kmzfile=os.path.normpath(os.path.join(sonpath,'class_GroundOverlay'+str(p)+'.kmz')), 
         name='Texture Lengthscale')

    except:
       print "error: map could not be created..."
Пример #3
0
def mosaic_texture(humfile,
                   sonpath,
                   cs2cs_args="epsg:26949",
                   res=99,
                   nn=5,
                   weight=1):
    '''
    Create mosaics of the spatially referenced sidescan echograms

    Syntax
    ----------
    [] = PyHum.mosaic_texture(humfile, sonpath, cs2cs_args, res, nn, weight)

    Parameters
    ----------
    humfile : str
       path to the .DAT file
    sonpath : str
       path where the *.SON files are
    cs2cs_args : int, *optional* [Default="epsg:26949"]
       arguments to create coordinates in a projected coordinate system
       this argument gets given to pyproj to turn wgs84 (lat/lon) coordinates
       into any projection supported by the proj.4 libraries
    res : float, *optional* [Default=0]
       grid resolution of output gridded texture map
       if res=99, res will be determined automatically from the spatial resolution of 1 pixel
    nn: int, *optional* [Default=5]
       number of nearest neighbours for gridding
    weight: int, *optional* [Default=1]
       specifies the type of pixel weighting in the gridding process
       weight = 1, based on grazing angle and inverse distance weighting
       weight = 2, based on grazing angle only
       weight = 3, inverse distance weighting only
       weight = 4, no weighting
    
    Returns
    -------

    sonpath+'GroundOverlay.kml': kml file
        contains gridded (or point cloud) sidescan intensity map for importing into google earth
        of the pth chunk

    sonpath+'map.png' : 
        image overlay associated with the kml file

    '''

    # prompt user to supply file if no input file given
    if not humfile:
        print('An input file is required!!!!!!')
        Tk().withdraw(
        )  # we don't want a full GUI, so keep the root window from appearing
        humfile = askopenfilename(filetypes=[("DAT files", "*.DAT")])

    # prompt user to supply directory if no input sonpath is given
    if not sonpath:
        print('A *.SON directory is required!!!!!!')
        Tk().withdraw(
        )  # we don't want a full GUI, so keep the root window from appearing
        sonpath = askdirectory()

    # print given arguments to screen and convert data type where necessary
    if humfile:
        print('Input file is %s' % (humfile))

    if sonpath:
        print('Sonar file path is %s' % (sonpath))

    if cs2cs_args:
        print('cs2cs arguments are %s' % (cs2cs_args))

    if res:
        res = np.asarray(res, float)
        print('Gridding resolution: %s' % (str(res)))

    if nn:
        nn = int(nn)
        print('Number of nearest neighbours for gridding: %s' % (str(nn)))

    if weight:
        weight = int(weight)
        print('Weighting for gridding: %s' % (str(weight)))

    ##nn = 5 #number of nearest neighbours in gridding
    ##noisefloor=10 # noise threshold in dB W

    # start timer
    if os.name == 'posix':  # true if linux/mac or cygwin on windows
        start = time.time()
    else:  # windows
        start = time.clock()

    trans = pyproj.Proj(init=cs2cs_args)

    # if son path name supplied has no separator at end, put one on
    if sonpath[-1] != os.sep:
        sonpath = sonpath + os.sep

    base = humfile.split('.DAT')  # get base of file name for output
    base = base[0].split(os.sep)[-1]

    # remove underscores, negatives and spaces from basename
    base = humutils.strip_base(base)

    meta = loadmat(os.path.normpath(os.path.join(sonpath, base + 'meta.mat')))

    esi = np.squeeze(meta['e'])
    nsi = np.squeeze(meta['n'])

    theta = np.squeeze(meta['heading']) / (180 / np.pi)

    # load memory mapped scans
    shape_port = np.squeeze(meta['shape_port'])
    if shape_port != '':
        if os.path.isfile(
                os.path.normpath(
                    os.path.join(sonpath, base + '_data_port_lar.dat'))):
            port_fp = io.get_mmap_data(sonpath, base, '_data_port_lar.dat',
                                       'float32', tuple(shape_port))
        else:
            port_fp = io.get_mmap_data(sonpath, base, '_data_port_la.dat',
                                       'float32', tuple(shape_port))

    shape_star = np.squeeze(meta['shape_star'])
    if shape_star != '':
        if os.path.isfile(
                os.path.normpath(
                    os.path.join(sonpath, base + '_data_star_lar.dat'))):
            star_fp = io.get_mmap_data(sonpath, base, '_data_star_lar.dat',
                                       'float32', tuple(shape_star))
        else:
            star_fp = io.get_mmap_data(sonpath, base, '_data_star_la.dat',
                                       'float32', tuple(shape_star))

    # time varying gain
    tvg = ((8.5 * 10**-5) + (3 / 76923) + ((8.5 * 10**-5) / 4)) * meta['c']

    # depth correction
    dist_tvg = np.squeeze((
        (np.tan(np.radians(25))) * np.squeeze(meta['dep_m'])) - (tvg))

    # read in range data
    R_fp = io.get_mmap_data(sonpath, base, '_data_range.dat', 'float32',
                            tuple(shape_star))

    dx = np.arcsin(meta['c'] / (1000 * meta['t'] * meta['f']))
    pix_m = meta['pix_m']
    c = meta['c']

    if not os.path.isfile(os.path.normpath(os.path.join(sonpath,
                                                        base + "S.p"))):
        #if 2 > 1:
        inputfiles = []
        if len(shape_star) > 2:
            for p in range(len(star_fp)):
                e = esi[shape_port[-1] * p:shape_port[-1] * (p + 1)]
                n = nsi[shape_port[-1] * p:shape_port[-1] * (p + 1)]
                t = theta[shape_port[-1] * p:shape_port[-1] * (p + 1)]
                d = dist_tvg[shape_port[-1] * p:shape_port[-1] * (p + 1)]
                dat_port = port_fp[p]
                dat_star = star_fp[p]
                data_R = R_fp[p]
                print("writing chunk %s " % (str(p)))
                write_points(e, n, t, d, dat_port, dat_star, data_R, pix_m,
                             res, cs2cs_args, sonpath, p, c, dx)
                inputfiles.append(
                    os.path.normpath(
                        os.path.join(sonpath, 'x_y_class' + str(p) + '.asc')))
        else:
            p = 0
            print("writing chunk %s " % (str(p)))
            write_points(esi, nsi, theta, dist_tvg, port_fp, star_fp, R_fp,
                         meta['pix_m'], res, cs2cs_args, sonpath, 0, c, dx)
            inputfiles.append(
                os.path.normpath(
                    os.path.join(sonpath, 'x_y_class' + str(p) + '.asc')))

        #trans =  pyproj.Proj(init=cs2cs_args)

        # D, R, h, t
        print("reading points from %s files" % (str(len(inputfiles))))
        X, Y, S, D, R, h, t, i = getxys(inputfiles)

        print("%s points read from %s files" %
              (str(len(S)), str(len(inputfiles))))

        # remove values where sidescan intensity is zero
        ind = np.where(np.logical_not(S == 0))[0]

        X = X[ind]
        Y = Y[ind]
        S = S[ind]
        D = D[ind]
        R = R[ind]
        h = h[ind]
        t = t[ind]
        i = i[ind]
        del ind

        # save to file for temporary storage
        pickle.dump(
            S, open(os.path.normpath(os.path.join(sonpath, base + "S.p")),
                    "wb"))
        del S
        pickle.dump(
            D, open(os.path.normpath(os.path.join(sonpath, base + "D.p")),
                    "wb"))
        del D
        pickle.dump(
            t, open(os.path.normpath(os.path.join(sonpath, base + "t.p")),
                    "wb"))
        del t
        pickle.dump(
            i, open(os.path.normpath(os.path.join(sonpath, base + "i.p")),
                    "wb"))
        del i

        pickle.dump(
            X, open(os.path.normpath(os.path.join(sonpath, base + "X.p")),
                    "wb"))
        del X
        pickle.dump(
            Y, open(os.path.normpath(os.path.join(sonpath, base + "Y.p")),
                    "wb"))
        del Y
        pickle.dump(
            R, open(os.path.normpath(os.path.join(sonpath, base + "R.p")),
                    "wb"))
        pickle.dump(
            h, open(os.path.normpath(os.path.join(sonpath, base + "h.p")),
                    "wb"))

        #grazing angle
        g = np.arctan(R.flatten(), h.flatten())
        pickle.dump(
            g, open(os.path.normpath(os.path.join(sonpath, base + "g.p")),
                    "wb"))
        del g, R, h

    print("creating grids ...")

    if res == 0:
        res = 99

    if res == 99:

        #### prepare grids
        R = pickle.load(
            open(os.path.normpath(os.path.join(sonpath, base + "R.p")), "rb"))

        ## actual along-track resolution is this: dx times dy = Af
        tmp = R * dx * (c * 0.007 / 2)
        del R

        resg = np.min(tmp[tmp > 0])
        del tmp
    else:
        resg = res

    X = pickle.load(
        open(os.path.normpath(os.path.join(sonpath, base + "X.p")), "rb"))
    Y = pickle.load(
        open(os.path.normpath(os.path.join(sonpath, base + "Y.p")), "rb"))

    humlon, humlat = trans(X, Y, inverse=True)

    grid_x, grid_y = np.meshgrid(np.arange(np.min(X), np.max(X), resg),
                                 np.arange(np.min(Y), np.max(Y), resg))

    shape = np.shape(grid_x)

    tree = KDTree(zip(X.flatten(), Y.flatten()))
    del X, Y

    print("mosaicking ...")
    #k nearest neighbour
    try:
        dist, inds = tree.query(zip(grid_x.flatten(), grid_y.flatten()),
                                k=nn,
                                n_jobs=-1)
    except:
        #print ".... update your scipy installation to use faster kd-tree"
        dist, inds = tree.query(zip(grid_x.flatten(), grid_y.flatten()), k=nn)

    #del grid_x, grid_y

    if weight == 1:
        g = pickle.load(
            open(os.path.normpath(os.path.join(sonpath, base + "g.p")), "rb"))
        w = g[inds] + 1.0 / dist**2
        del g
    elif weight == 2:
        g = pickle.load(
            open(os.path.normpath(os.path.join(sonpath, base + "g.p")), "rb"))
        w = g[inds]
        del g
    elif weight == 3:
        w = 1.0 / dist**2
    elif weight == 4:
        w = 1.0

    #g = pickle.load( open( os.path.normpath(os.path.join(sonpath,base+"g.p")), "rb" ) )
    #w = g[inds] + 1.0 / dist**2
    #del g

    if weight < 4:
        w[np.isinf(w)] = 1
        w[np.isnan(w)] = 1
        w[w > 10000] = 10000
        w[w <= 0] = 1

    # load in sidescan intensity
    S = pickle.load(
        open(os.path.normpath(os.path.join(sonpath, base + "S.p")), "rb"))
    # filter out noise pixels
    S[S < noisefloor] = np.nan

    if nn == 1:
        Sdat_g = (w * S.flatten()[inds]).reshape(shape)
        del w
        dist = dist.reshape(shape)
    else:
        if weight < 4:
            Sdat_g = (np.nansum(w * S.flatten()[inds], axis=1) /
                      np.nansum(w, axis=1)).reshape(shape)
        else:
            Sdat_g = (np.nansum(S.flatten()[inds], axis=1)).reshape(shape)
        del w
        dist = np.nanmean(dist, axis=1).reshape(shape)

    del S

    Sdat_g[dist > 1] = np.nan
    Sdat_g[Sdat_g < noisefloor] = np.nan

    dat = Sdat_g.copy()
    dat[dist > 1] = 0
    dat2 = replace_nans.RN(dat.astype('float64'), 1000, 0.01, 2,
                           'localmean').getdata()
    dat2[dat == 0] = np.nan
    del dat

    dat2[dat2 < noisefloor] = np.nan

    Sdat_g = dat2.copy()
    del dat2

    Sdat_g[Sdat_g == 0] = np.nan
    Sdat_g[np.isinf(Sdat_g)] = np.nan
    Sdat_gm = np.ma.masked_invalid(Sdat_g)
    del Sdat_g

    glon, glat = trans(grid_x, grid_y, inverse=True)
    del grid_x, grid_y

    # =========================================================
    print("creating kmz file ...")
    ## new way to create kml file
    pixels = 1024 * 10

    fig, ax = humutils.gearth_fig(llcrnrlon=glon.min(),
                                  llcrnrlat=glat.min(),
                                  urcrnrlon=glon.max(),
                                  urcrnrlat=glat.max(),
                                  pixels=pixels)
    cs = ax.pcolormesh(glon, glat, Sdat_gm)
    ax.set_axis_off()
    fig.savefig(os.path.normpath(os.path.join(sonpath, 'class_overlay1.png')),
                transparent=True,
                format='png')

    fig = plt.figure(figsize=(1.0, 4.0), facecolor=None, frameon=False)
    ax = fig.add_axes([0.0, 0.05, 0.2, 0.9])
    cb = fig.colorbar(cs, cax=ax)
    cb.set_label('Texture lengthscale [m]',
                 rotation=-90,
                 color='k',
                 labelpad=20)
    fig.savefig(os.path.normpath(os.path.join(sonpath, 'class_legend.png')),
                transparent=False,
                format='png')

    humutils.make_kml(
        llcrnrlon=glon.min(),
        llcrnrlat=glat.min(),
        urcrnrlon=glon.max(),
        urcrnrlat=glat.max(),
        figs=[os.path.normpath(os.path.join(sonpath, 'class_overlay1.png'))],
        colorbar=os.path.normpath(os.path.join(sonpath, 'class_legend.png')),
        kmzfile=os.path.normpath(
            os.path.join(sonpath, 'class_GroundOverlay.kmz')),
        name='Sidescan Intensity')

    # =========================================================
    print("drawing and printing map ...")
    fig = plt.figure(frameon=False)
    map = Basemap(
        projection='merc',
        epsg=cs2cs_args.split(':')[1],
        resolution='i',  #h #f
        llcrnrlon=np.min(humlon) - 0.001,
        llcrnrlat=np.min(humlat) - 0.001,
        urcrnrlon=np.max(humlon) + 0.001,
        urcrnrlat=np.max(humlat) + 0.001)

    gx, gy = map.projtran(glon, glat)

    try:
        map.arcgisimage(server='http://server.arcgisonline.com/ArcGIS',
                        service='ESRI_Imagery_World_2D',
                        xpixels=1000,
                        ypixels=None,
                        dpi=300)
    except:
        map.arcgisimage(server='http://server.arcgisonline.com/ArcGIS',
                        service='World_Imagery',
                        xpixels=1000,
                        ypixels=None,
                        dpi=300)
    #finally:
    #   print "error: map could not be created..."

    ax = plt.Axes(
        fig,
        [0., 0., 1., 1.],
    )
    ax.set_axis_off()
    fig.add_axes(ax)

    if Sdat_gm.size > 25000000:
        print(
            "matrix size > 25,000,000 - decimating by factor of 5 for display")
        map.pcolormesh(gx[::5, ::5],
                       gy[::5, ::5],
                       Sdat_gm[::5, ::5],
                       vmin=np.nanmin(Sdat_gm),
                       vmax=np.nanmax(Sdat_gm))
    else:
        map.pcolormesh(gx,
                       gy,
                       Sdat_gm,
                       vmin=np.nanmin(Sdat_gm),
                       vmax=np.nanmax(Sdat_gm))

    custom_save2(sonpath, 'class_map_imagery')
    del fig

    if os.name == 'posix':  # true if linux/mac
        elapsed = (time.time() - start)
    else:  # windows
        elapsed = (time.clock() - start)
    print("Processing took " + str(elapsed) + "seconds to analyse")

    print("Done!")
Пример #4
0
def make_map(e, n, t, d, dat_port, dat_star, data_R, pix_m, res, cs2cs_args, sonpath, p, mode, nn, numstdevs, c, dx, use_uncorrected, scalemax): #dogrid, influence,dowrite,

   thres=5

   trans =  pyproj.Proj(init=cs2cs_args)

   mp = np.nanmean(dat_port)
   ms = np.nanmean(dat_star)
   if mp>ms:
      merge = np.vstack((dat_port,dat_star*(mp/ms)))      
   else:
      merge = np.vstack((dat_port*(ms/mp),dat_star))
   del dat_port, dat_star

   merge[np.isnan(merge)] = 0
   merge = merge[:,:len(n)]

   ## actual along-track resolution is this: dx times dy = Af
   tmp = data_R * dx * (c*0.007 / 2) #dx = np.arcsin(c/(1000*meta['t']*meta['f']))
   res_grid = np.sqrt(np.vstack((tmp, tmp)))
   del tmp
   res_grid = res_grid[:np.shape(merge)[0],:np.shape(merge)[1]]
   
   #if use_uncorrected != 1:
   #   merge = merge - 10*np.log10(res_grid)
   
   res_grid = res_grid.astype('float32')

   merge[np.isnan(merge)] = 0
   merge[merge<0] = 0

   merge = merge.astype('float32')

   merge = denoise_tv_chambolle(merge.copy(), weight=.2, multichannel=False).astype('float32')

   R = np.vstack((np.flipud(data_R),data_R))
   del data_R
   R = R[:np.shape(merge)[0],:np.shape(merge)[1]]

   # get number pixels in scan line
   extent = int(np.shape(merge)[0]/2)

   yvec = np.squeeze(np.linspace(np.squeeze(pix_m),extent*np.squeeze(pix_m),extent))

   X, Y, D, h, t  = getXY(e,n,yvec,np.squeeze(d),t,extent)

   X = X.astype('float32')
   Y = Y.astype('float32')
   D = D.astype('float32')
   h = h.astype('float32')
   t = t.astype('float32')
   X = X.astype('float32')

   D[np.isnan(D)] = 0
   h[np.isnan(h)] = 0
   t[np.isnan(t)] = 0

   X = X[np.where(np.logical_not(np.isnan(Y)))]
   merge = merge.flatten()[np.where(np.logical_not(np.isnan(Y)))]
   res_grid = res_grid.flatten()[np.where(np.logical_not(np.isnan(Y)))]
   Y = Y[np.where(np.logical_not(np.isnan(Y)))]
   D = D[np.where(np.logical_not(np.isnan(Y)))]
   R = R.flatten()[np.where(np.logical_not(np.isnan(Y)))]
   h = h[np.where(np.logical_not(np.isnan(Y)))]
   t = t[np.where(np.logical_not(np.isnan(Y)))]

   Y = Y[np.where(np.logical_not(np.isnan(X)))]
   merge = merge.flatten()[np.where(np.logical_not(np.isnan(X)))]
   res_grid = res_grid.flatten()[np.where(np.logical_not(np.isnan(X)))]
   X = X[np.where(np.logical_not(np.isnan(X)))]
   D = D[np.where(np.logical_not(np.isnan(X)))]
   R = R.flatten()[np.where(np.logical_not(np.isnan(X)))]
   h = h[np.where(np.logical_not(np.isnan(X)))]
   t = t[np.where(np.logical_not(np.isnan(X)))]

   X = X[np.where(np.logical_not(np.isnan(merge)))]
   Y = Y[np.where(np.logical_not(np.isnan(merge)))]
   merge = merge[np.where(np.logical_not(np.isnan(merge)))]
   res_grid = res_grid.flatten()[np.where(np.logical_not(np.isnan(merge)))]
   D = D[np.where(np.logical_not(np.isnan(merge)))]
   R = R[np.where(np.logical_not(np.isnan(merge)))]
   h = h[np.where(np.logical_not(np.isnan(merge)))]
   t = t[np.where(np.logical_not(np.isnan(merge)))]

   X = X[np.where(np.logical_not(np.isinf(merge)))]
   Y = Y[np.where(np.logical_not(np.isinf(merge)))]
   merge = merge[np.where(np.logical_not(np.isinf(merge)))]
   res_grid = res_grid.flatten()[np.where(np.logical_not(np.isinf(merge)))]
   D = D[np.where(np.logical_not(np.isinf(merge)))]
   R = R[np.where(np.logical_not(np.isinf(merge)))]
   h = h[np.where(np.logical_not(np.isinf(merge)))]
   t = t[np.where(np.logical_not(np.isinf(merge)))]



   print("writing point cloud")
   #if dowrite==1:
   ## write raw bs to file
   outfile = os.path.normpath(os.path.join(sonpath,'x_y_ss_raw'+str(p)+'.asc'))
   ##write.txtwrite( outfile, np.hstack((humutils.ascol(X.flatten()),humutils.ascol(Y.flatten()), humutils.ascol(merge.flatten()), humutils.ascol(D.flatten()), humutils.ascol(R.flatten()), humutils.ascol(h.flatten()), humutils.ascol(t.flatten())  )) )
   np.savetxt(outfile, np.hstack((humutils.ascol(X.flatten()),humutils.ascol(Y.flatten()), humutils.ascol(merge.flatten()), humutils.ascol(D.flatten()), humutils.ascol(R.flatten()), humutils.ascol(h.flatten()), humutils.ascol(t.flatten())  )) , fmt="%8.6f %8.6f %8.6f %8.6f %8.6f %8.6f %8.6f") 

   del D, R, h, t

   sigmas = 0.1 #m
   eps = 2

   print("gridding ...")
   #if dogrid==1:
   if 2>1:

      if res==99:
         resg = np.min(res_grid[res_grid>0])/2
         print('Gridding at resolution of %s' % str(resg))
      else:
         resg = res

      tree = KDTree(np.c_[X.flatten(),Y.flatten()])
      complete=0
      while complete==0:
         try:
            grid_x, grid_y, res = getmesh(np.min(X), np.max(X), np.min(Y), np.max(Y), resg)
            longrid, latgrid = trans(grid_x, grid_y, inverse=True)
            longrid = longrid.astype('float32')
            latgrid = latgrid.astype('float32')
            shape = np.shape(grid_x)

            ## create mask for where the data is not
            if pykdtree==1:
               dist, _ = tree.query(np.c_[grid_x.ravel(), grid_y.ravel()], k=1)
            else:
               try:
                  dist, _ = tree.query(np.c_[grid_x.ravel(), grid_y.ravel()], k=1, n_jobs=cpu_count())
               except:
                  #print ".... update your scipy installation to use faster kd-tree queries"
                  dist, _ = tree.query(np.c_[grid_x.ravel(), grid_y.ravel()], k=1)

            dist = dist.reshape(grid_x.shape)

            targ_def = pyresample.geometry.SwathDefinition(lons=longrid.flatten(), lats=latgrid.flatten())
            del longrid, latgrid

            humlon, humlat = trans(X, Y, inverse=True)
            orig_def = pyresample.geometry.SwathDefinition(lons=humlon.flatten(), lats=humlat.flatten())
            del humlon, humlat
            if 'orig_def' in locals():
               complete=1
         except:
            print("memory error: trying grid resolution of %s" % (str(resg*2)))
            resg = resg*2

      if mode==1:

         complete=0
         while complete==0:
            try:
               try:
                  dat = pyresample.kd_tree.resample_nearest(orig_def, merge.flatten(), targ_def, radius_of_influence=res*20, fill_value=None, nprocs = cpu_count(), reduce_data=1)
               except:
                  dat = pyresample.kd_tree.resample_nearest(orig_def, merge.flatten(), targ_def, radius_of_influence=res*20, fill_value=None, nprocs = 1, reduce_data=1)

               try:
                  r_dat = pyresample.kd_tree.resample_nearest(orig_def, res_grid.flatten(), targ_def, radius_of_influence=res*20, fill_value=None, nprocs = cpu_count(), reduce_data=1)
               except:
                  r_dat = pyresample.kd_tree.resample_nearest(orig_def, res_grid.flatten(), targ_def, radius_of_influence=res*20, fill_value=None, nprocs = 1, reduce_data=1)

               stdev = None
               counts = None
               if 'dat' in locals():
                  complete=1
            except:
               del grid_x, grid_y, targ_def, orig_def

               wf = None
               humlon, humlat = trans(X, Y, inverse=True)
               dat, stdev, counts, resg, complete, shape = getgrid_lm(humlon, humlat, merge, res*10, min(X), max(X), min(Y), max(Y), resg*2, mode, trans, nn, wf, sigmas, eps)
               r_dat, stdev, counts, resg, complete, shape = getgrid_lm(humlon, humlat, res_grid, res*10, min(X), max(X), min(Y), max(Y), resg*2, mode, trans, nn, wf, sigmas, eps)
               del humlon, humlat

      elif mode==2:

         # custom inverse distance
         wf = lambda r: 1/r**2

         complete=0
         while complete==0:
            try:
               try:
                  dat, stdev, counts = pyresample.kd_tree.resample_custom(orig_def, merge.flatten(),targ_def, radius_of_influence=res*20, neighbours=nn, weight_funcs=wf, fill_value=None, with_uncert = True, nprocs = cpu_count(), reduce_data=1)
               except:
                  dat, stdev, counts = pyresample.kd_tree.resample_custom(orig_def, merge.flatten(),targ_def, radius_of_influence=res*20, neighbours=nn, weight_funcs=wf, fill_value=None, with_uncert = True, nprocs = 1, reduce_data=1)

               try:
                  r_dat = pyresample.kd_tree.resample_custom(orig_def, res_grid.flatten(), targ_def, radius_of_influence=res*20, neighbours=nn, weight_funcs=wf, fill_value=None, with_uncert = False, nprocs = cpu_count(), reduce_data=1)
               except:
                  r_dat = pyresample.kd_tree.resample_custom(orig_def, res_grid.flatten(), targ_def, radius_of_influence=res*20, neighbours=nn, weight_funcs=wf, fill_value=None, with_uncert = False, nprocs = 1, reduce_data=1)

               if 'dat' in locals():
                  complete=1
            except:
               del grid_x, grid_y, targ_def, orig_def
               humlon, humlat = trans(X, Y, inverse=True)
               dat, stdev, counts, resg, complete, shape = getgrid_lm(humlon, humlat, merge, res*2, min(X), max(X), min(Y), max(Y), resg*2, mode, trans, nn, wf, sigmas, eps)
               r_dat, stdev, counts, resg, complete, shape = getgrid_lm(humlon, humlat, res_grid, res*2, min(X), max(X), min(Y), max(Y), resg*2, mode, trans, nn, wf, sigmas, eps)
               del humlat, humlon
               del stdev_null, counts_null

      elif mode==3:
         wf = None

         complete=0
         while complete==0:
            try:
               try:
                  dat, stdev, counts = pyresample.kd_tree.resample_gauss(orig_def, merge.flatten(), targ_def, radius_of_influence=res*20, neighbours=nn, sigmas=sigmas, fill_value=None, with_uncert = True, nprocs = cpu_count(), epsilon = eps, reduce_data=1)
               except:
                  dat, stdev, counts = pyresample.kd_tree.resample_gauss(orig_def, merge.flatten(), targ_def, radius_of_influence=res*20, neighbours=nn, sigmas=sigmas, fill_value=None, with_uncert = True, nprocs = 1, epsilon = eps, reduce_data=1)

               try:
                  r_dat = pyresample.kd_tree.resample_gauss(orig_def, res_grid.flatten(), targ_def, radius_of_influence=res*20, neighbours=nn, sigmas=sigmas, fill_value=None, with_uncert = False, nprocs = cpu_count(), epsilon = eps, reduce_data=1)
               except:
                  r_dat = pyresample.kd_tree.resample_gauss(orig_def, res_grid.flatten(), targ_def, radius_of_influence=res*20, neighbours=nn, sigmas=sigmas, fill_value=None, with_uncert = False, nprocs = 1, epsilon = eps, reduce_data=1)

               if 'dat' in locals():
                  complete=1
            except:
               del grid_x, grid_y, targ_def, orig_def
               humlon, humlat = trans(X, Y, inverse=True)
               dat, stdev, counts, resg, complete, shape = getgrid_lm(humlon, humlat, merge, res*10, min(X), max(X), min(Y), max(Y), resg*2, mode, trans, nn, wf, sigmas, eps)
               r_dat, stdev_null, counts_null, resg, complete, shape = getgrid_lm(humlon, humlat, res_grid, res*10, min(X), max(X), min(Y), max(Y), resg*2, mode, trans, nn, wf, sigmas, eps)
               del humlat, humlon
               del stdev_null, counts_null

      humlon, humlat = trans(X, Y, inverse=True)
      del X, Y, res_grid, merge

      dat = dat.reshape(shape)

      dat[dist>res*30] = np.nan
      del dist

      r_dat = r_dat.reshape(shape)
      r_dat[r_dat<1] = 1
      r_dat[r_dat > 2*np.pi] = 1
      r_dat[np.isnan(dat)] = np.nan

      dat = dat + r_dat #np.sqrt(np.cos(np.deg2rad(r_dat))) #dat*np.sqrt(r_dat) + dat

      del r_dat

      if mode>1:
         stdev = stdev.reshape(shape)
         counts = counts.reshape(shape)

      mask = dat.mask.copy()

      dat[mask==1] = np.nan
      #dat[mask==1] = 0

      if mode>1:
         dat[(stdev>numstdevs) & (mask!=0)] = np.nan
         dat[(counts<nn) & (counts>0)] = np.nan


   #if dogrid==1:

   dat[dat==0] = np.nan
   dat[np.isinf(dat)] = np.nan

   dat[dat<thres] = np.nan

   datm = np.ma.masked_invalid(dat)

   glon, glat = trans(grid_x, grid_y, inverse=True)
   #del grid_x, grid_y

   try:
      from osgeo import gdal,ogr,osr
      proj = osr.SpatialReference()
      proj.ImportFromEPSG(int(cs2cs_args.split(':')[-1])) #26949)
      datout = np.squeeze(np.ma.filled(dat))#.astype('int16')
      datout[np.isnan(datout)] = -99
      driver = gdal.GetDriverByName('GTiff')
      #rows,cols = np.shape(datout)
      cols,rows = np.shape(datout)    
      outFile = os.path.normpath(os.path.join(sonpath,'geotiff_map'+str(p)+'.tif'))
      ds = driver.Create( outFile, rows, cols, 1, gdal.GDT_Float32, [ 'COMPRESS=LZW' ] )        
      if proj is not None:  
        ds.SetProjection(proj.ExportToWkt()) 

      xmin, ymin, xmax, ymax = [grid_x.min(), grid_y.min(), grid_x.max(), grid_y.max()]

      xres = (xmax - xmin) / float(rows)
      yres = (ymax - ymin) / float(cols)
      geotransform = (xmin, xres, 0, ymax, 0, -yres)

      ds.SetGeoTransform(geotransform)
      ss_band = ds.GetRasterBand(1)
      ss_band.WriteArray(np.flipud(datout)) #datout)
      ss_band.SetNoDataValue(-99)
      ss_band.FlushCache()
      ss_band.ComputeStatistics(False)
      del ds   
   
   except:
      print("error: geotiff could not be created... check your gdal/ogr install")


   try:

      # =========================================================
      print("creating kmz file ...")
      ## new way to create kml file
      pixels = 1024 * 10

      fig, ax = humutils.gearth_fig(llcrnrlon=glon.min(),
                     llcrnrlat=glat.min(),
                     urcrnrlon=glon.max(),
                     urcrnrlat=glat.max(),
                     pixels=pixels)
      cs = ax.pcolormesh(glon, glat, datm, vmax=scalemax, cmap='gray')
      ax.set_axis_off()
      fig.savefig(os.path.normpath(os.path.join(sonpath,'map'+str(p)+'.png')), transparent=True, format='png')
      del fig, ax

      # =========================================================
      fig = plt.figure(figsize=(1.0, 4.0), facecolor=None, frameon=False)
      ax = fig.add_axes([0.0, 0.05, 0.2, 0.9])
      cb = fig.colorbar(cs, cax=ax)
      cb.set_label('Intensity [dB W]', rotation=-90, color='k', labelpad=20)
      fig.savefig(os.path.normpath(os.path.join(sonpath,'legend'+str(p)+'.png')), transparent=False, format='png')
      del fig, ax, cs, cb

      # =========================================================
      humutils.make_kml(llcrnrlon=glon.min(), llcrnrlat=glat.min(),
         urcrnrlon=glon.max(), urcrnrlat=glat.max(),
         figs=[os.path.normpath(os.path.join(sonpath,'map'+str(p)+'.png'))],
         colorbar=os.path.normpath(os.path.join(sonpath,'legend'+str(p)+'.png')),
         kmzfile=os.path.normpath(os.path.join(sonpath,'GroundOverlay'+str(p)+'.kmz')),
         name='Sidescan Intensity')

   except:
      print("error: map could not be created...")


   #y1 = np.min(glat)-0.001
   #x1 = np.min(glon)-0.001
   #y2 = np.max(glat)+0.001
   #x2 = np.max(glon)+0.001

   print("drawing and printing map ...")
   fig = plt.figure(frameon=False)
   map = Basemap(projection='merc', epsg=cs2cs_args.split(':')[1],
    resolution = 'i', #h #f
    llcrnrlon=np.min(humlon)-0.001, llcrnrlat=np.min(glat)-0.001,
    urcrnrlon=np.max(humlon)+0.001, urcrnrlat=np.max(glat)+0.001)

   try:
      map.arcgisimage(server='http://server.arcgisonline.com/ArcGIS', service='World_Imagery', xpixels=1000, ypixels=None, dpi=300)
   except:
      map.arcgisimage(server='http://server.arcgisonline.com/ArcGIS', service='ESRI_Imagery_World_2D', xpixels=1000, ypixels=None, dpi=300)
   #finally:
   #   print "error: map could not be created..."

   #if dogrid==1:
   gx,gy = map.projtran(glon, glat)

   ax = plt.Axes(fig, [0., 0., 1., 1.], )
   ax.set_axis_off()
   fig.add_axes(ax)

   #if dogrid==1:
   if 2>1:
      if datm.size > 25000000:
         print("matrix size > 25,000,000 - decimating by factor of 5 for display")
         map.pcolormesh(gx[::5,::5], gy[::5,::5], datm[::5,::5], cmap='gray', vmin=np.nanmin(datm), vmax=scalemax) #vmax=np.nanmax(datm)
      else:
         map.pcolormesh(gx, gy, datm, cmap='gray', vmin=np.nanmin(datm), vmax=scalemax) #vmax=np.nanmax(datm)
      del datm, dat
   else:
      ## draw point cloud
      x,y = map.projtran(humlon, humlat)
      map.scatter(x.flatten(), y.flatten(), 0.5, merge.flatten(), cmap='gray', linewidth = '0')

   #map.drawmapscale(x1+0.001, y1+0.001, x1, y1, 200., units='m', barstyle='fancy', labelstyle='simple', fontcolor='k') #'#F8F8FF')
   #map.drawparallels(np.arange(y1-0.001, y2+0.001, 0.005),labels=[1,0,0,1], linewidth=0.0, rotation=30, fontsize=8)
   #map.drawmeridians(np.arange(x1, x2, 0.002),labels=[1,0,0,1], linewidth=0.0, rotation=30, fontsize=8)

   custom_save2(sonpath,'map_imagery'+str(p))
   del fig


   del humlat, humlon
   return res #return the new resolution
Пример #5
0
def mosaic_texture(humfile, sonpath, cs2cs_args = "epsg:26949", res = 99, nn = 5, weight = 1):
         
    '''
    Create mosaics of the spatially referenced sidescan echograms

    Syntax
    ----------
    [] = PyHum.mosaic_texture(humfile, sonpath, cs2cs_args, res, nn, weight)

    Parameters
    ----------
    humfile : str
       path to the .DAT file
    sonpath : str
       path where the *.SON files are
    cs2cs_args : int, *optional* [Default="epsg:26949"]
       arguments to create coordinates in a projected coordinate system
       this argument gets given to pyproj to turn wgs84 (lat/lon) coordinates
       into any projection supported by the proj.4 libraries
    res : float, *optional* [Default=0]
       grid resolution of output gridded texture map
       if res=99, res will be determined automatically from the spatial resolution of 1 pixel
    nn: int, *optional* [Default=5]
       number of nearest neighbours for gridding
    weight: int, *optional* [Default=1]
       specifies the type of pixel weighting in the gridding process
       weight = 1, based on grazing angle and inverse distance weighting
       weight = 2, based on grazing angle only
       weight = 3, inverse distance weighting only
       weight = 4, no weighting
    
    Returns
    -------

    sonpath+'GroundOverlay.kml': kml file
        contains gridded (or point cloud) sidescan intensity map for importing into google earth
        of the pth chunk

    sonpath+'map.png' : 
        image overlay associated with the kml file

    '''

    # prompt user to supply file if no input file given
    if not humfile:
       print('An input file is required!!!!!!')
       Tk().withdraw() # we don't want a full GUI, so keep the root window from appearing
       humfile = askopenfilename(filetypes=[("DAT files","*.DAT")]) 

    # prompt user to supply directory if no input sonpath is given
    if not sonpath:
       print('A *.SON directory is required!!!!!!')
       Tk().withdraw() # we don't want a full GUI, so keep the root window from appearing
       sonpath = askdirectory() 

    # print given arguments to screen and convert data type where necessary
    if humfile:
       print('Input file is %s' % (humfile))

    if sonpath:
       print('Sonar file path is %s' % (sonpath))

    if cs2cs_args:
       print('cs2cs arguments are %s' % (cs2cs_args))  

    if res:
       res = np.asarray(res,float)
       print('Gridding resolution: %s' % (str(res)))   
       
    if nn:
       nn = int(nn)
       print('Number of nearest neighbours for gridding: %s' % (str(nn)))
                    
    if weight:
       weight = int(weight)
       print('Weighting for gridding: %s' % (str(weight)))               


    ##nn = 5 #number of nearest neighbours in gridding
    ##noisefloor=10 # noise threshold in dB W

    # start timer
    if os.name=='posix': # true if linux/mac or cygwin on windows
       start = time.time()
    else: # windows
       start = time.clock()

    trans =  pyproj.Proj(init=cs2cs_args)

    # if son path name supplied has no separator at end, put one on
    if sonpath[-1]!=os.sep:
       sonpath = sonpath + os.sep

    base = humfile.split('.DAT') # get base of file name for output
    base = base[0].split(os.sep)[-1]

    # remove underscores, negatives and spaces from basename
    base = humutils.strip_base(base)

    meta = loadmat(os.path.normpath(os.path.join(sonpath,base+'meta.mat')))

    esi = np.squeeze(meta['e'])
    nsi = np.squeeze(meta['n']) 
    
    theta = np.squeeze(meta['heading'])/(180/np.pi)

    # load memory mapped scans
    shape_port = np.squeeze(meta['shape_port'])
    if shape_port!='':
       if os.path.isfile(os.path.normpath(os.path.join(sonpath,base+'_data_port_lar.dat'))):
          port_fp = io.get_mmap_data(sonpath, base, '_data_port_lar.dat', 'float32', tuple(shape_port))
       else:
          port_fp = io.get_mmap_data(sonpath, base, '_data_port_la.dat', 'float32', tuple(shape_port))

    shape_star = np.squeeze(meta['shape_star'])
    if shape_star!='':
       if os.path.isfile(os.path.normpath(os.path.join(sonpath,base+'_data_star_lar.dat'))):
             star_fp = io.get_mmap_data(sonpath, base, '_data_star_lar.dat', 'float32', tuple(shape_star))
       else:
          star_fp = io.get_mmap_data(sonpath, base, '_data_star_la.dat', 'float32', tuple(shape_star))

    # time varying gain
    tvg = ((8.5*10**-5)+(3/76923)+((8.5*10**-5)/4))*meta['c']
        
    # depth correction
    dist_tvg = np.squeeze(((np.tan(np.radians(25)))*np.squeeze(meta['dep_m']))-(tvg))

    # read in range data
    R_fp = io.get_mmap_data(sonpath, base, '_data_range.dat', 'float32', tuple(shape_star))

    dx = np.arcsin(meta['c']/(1000*meta['t']*meta['f']))
    pix_m = meta['pix_m']
    c = meta['c']

    if not os.path.isfile( os.path.normpath(os.path.join(sonpath,base+"S.p")) ):
    #if 2 > 1:
       inputfiles = []
       if len(shape_star)>2:    
          for p in range(len(star_fp)):
             e = esi[shape_port[-1]*p:shape_port[-1]*(p+1)]
             n = nsi[shape_port[-1]*p:shape_port[-1]*(p+1)]
             t = theta[shape_port[-1]*p:shape_port[-1]*(p+1)]
             d = dist_tvg[shape_port[-1]*p:shape_port[-1]*(p+1)]
             dat_port = port_fp[p]
             dat_star = star_fp[p]
             data_R = R_fp[p]
             print("writing chunk %s " % (str(p)))
             write_points(e, n, t, d, dat_port, dat_star, data_R, pix_m, res, cs2cs_args, sonpath, p, c, dx)
             inputfiles.append(os.path.normpath(os.path.join(sonpath,'x_y_class'+str(p)+'.asc')))
       else:
          p=0
          print("writing chunk %s " % (str(p)))
          write_points(esi, nsi, theta, dist_tvg, port_fp, star_fp, R_fp, meta['pix_m'], res, cs2cs_args, sonpath, 0, c, dx)
          inputfiles.append(os.path.normpath(os.path.join(sonpath,'x_y_class'+str(p)+'.asc')))         
          
       #trans =  pyproj.Proj(init=cs2cs_args)

       # D, R, h, t
       print("reading points from %s files" % (str(len(inputfiles))))
       X,Y,S,D,R,h,t,i = getxys(inputfiles)

       print("%s points read from %s files" % (str(len(S)), str(len(inputfiles))))

       # remove values where sidescan intensity is zero
       ind = np.where(np.logical_not(S==0))[0]

       X = X[ind]; Y = Y[ind]
       S = S[ind]; D = D[ind]
       R = R[ind]; h = h[ind]
       t = t[ind]; i = i[ind]
       del ind   
   
       # save to file for temporary storage
       pickle.dump( S, open( os.path.normpath(os.path.join(sonpath,base+"S.p")), "wb" ) ); del S
       pickle.dump( D, open( os.path.normpath(os.path.join(sonpath,base+"D.p")), "wb" ) ); del D
       pickle.dump( t, open( os.path.normpath(os.path.join(sonpath,base+"t.p")), "wb" ) ); del t
       pickle.dump( i, open( os.path.normpath(os.path.join(sonpath,base+"i.p")), "wb" ) ); del i

       pickle.dump( X, open( os.path.normpath(os.path.join(sonpath,base+"X.p")), "wb" ) ); del X
       pickle.dump( Y, open( os.path.normpath(os.path.join(sonpath,base+"Y.p")), "wb" ) ); del Y
       pickle.dump( R, open( os.path.normpath(os.path.join(sonpath,base+"R.p")), "wb" ) ); 
       pickle.dump( h, open( os.path.normpath(os.path.join(sonpath,base+"h.p")), "wb" ) ); 

       #grazing angle
       g = np.arctan(R.flatten(),h.flatten())
       pickle.dump( g, open( os.path.normpath(os.path.join(sonpath,base+"g.p")), "wb" ) ); del g, R, h
   
    print("creating grids ...") 

    if res==0:
       res=99

    if res==99:

       #### prepare grids
       R = pickle.load( open( os.path.normpath(os.path.join(sonpath,base+"R.p")), "rb" ) )

       ## actual along-track resolution is this: dx times dy = Af
       tmp = R * dx * (c*0.007 / 2)
       del R

       resg = np.min(tmp[tmp>0])
       del tmp
    else:
       resg = res

    X = pickle.load( open( os.path.normpath(os.path.join(sonpath,base+"X.p")), "rb" ) )
    Y = pickle.load( open( os.path.normpath(os.path.join(sonpath,base+"Y.p")), "rb" ) )
    
    humlon, humlat = trans(X, Y, inverse=True)

    grid_x, grid_y = np.meshgrid( np.arange(np.min(X), np.max(X), resg), np.arange(np.min(Y), np.max(Y), resg) )    
 
    shape = np.shape(grid_x)

    tree = KDTree(zip(X.flatten(), Y.flatten()))
    del X, Y

    print("mosaicking ...")  
    #k nearest neighbour
    try:
       dist, inds = tree.query(zip(grid_x.flatten(), grid_y.flatten()), k = nn, n_jobs=-1)
    except:
       #print ".... update your scipy installation to use faster kd-tree"   
       dist, inds = tree.query(zip(grid_x.flatten(), grid_y.flatten()), k = nn)    
    
    #del grid_x, grid_y
    
    if weight==1:
       g = pickle.load( open( os.path.normpath(os.path.join(sonpath,base+"g.p")), "rb" ) )
       w = g[inds] + 1.0 / dist**2
       del g
    elif weight==2:
       g = pickle.load( open( os.path.normpath(os.path.join(sonpath,base+"g.p")), "rb" ) )
       w = g[inds]
       del g
    elif weight==3:
       w = 1.0 / dist**2    
    elif weight==4:
       w = 1.0
    
    #g = pickle.load( open( os.path.normpath(os.path.join(sonpath,base+"g.p")), "rb" ) )
    #w = g[inds] + 1.0 / dist**2
    #del g

    if weight < 4:
       w[np.isinf(w)]=1
       w[np.isnan(w)]=1
       w[w>10000]=10000
       w[w<=0]=1
    
    # load in sidescan intensity
    S = pickle.load( open( os.path.normpath(os.path.join(sonpath,base+"S.p")), "rb" ) )
    # filter out noise pixels
    S[S<noisefloor] = np.nan

    if nn==1:
       Sdat_g = (w * S.flatten()[inds]).reshape(shape)
       del w
       dist = dist.reshape(shape)
    else:
       if weight < 4:
          Sdat_g = (np.nansum(w * S.flatten()[inds], axis=1) / np.nansum(w, axis=1)).reshape(shape)
       else:
          Sdat_g = (np.nansum(S.flatten()[inds], axis=1)).reshape(shape)
       del w
       dist = np.nanmean(dist,axis=1).reshape(shape)

    del S

    Sdat_g[dist>1] = np.nan
    Sdat_g[Sdat_g<noisefloor] = np.nan

    dat = Sdat_g.copy()
    dat[dist>1] = 0
    dat2 = replace_nans.RN(dat.astype('float64'),1000,0.01,2,'localmean').getdata()
    dat2[dat==0] = np.nan
    del dat

    dat2[dat2<noisefloor] = np.nan

    Sdat_g = dat2.copy()
    del dat2
   
    Sdat_g[Sdat_g==0] = np.nan
    Sdat_g[np.isinf(Sdat_g)] = np.nan
    Sdat_gm = np.ma.masked_invalid(Sdat_g)
    del Sdat_g

    glon, glat = trans(grid_x, grid_y, inverse=True)
    del grid_x, grid_y
    
    # =========================================================
    print("creating kmz file ...")
    ## new way to create kml file  
    pixels = 1024 * 10
 
    fig, ax = humutils.gearth_fig(llcrnrlon=glon.min(),
                     llcrnrlat=glat.min(),
                     urcrnrlon=glon.max(),
                     urcrnrlat=glat.max(),
                     pixels=pixels)
    cs = ax.pcolormesh(glon, glat, Sdat_gm)
    ax.set_axis_off()
    fig.savefig(os.path.normpath(os.path.join(sonpath,'class_overlay1.png')), transparent=True, format='png')    
    

    fig = plt.figure(figsize=(1.0, 4.0), facecolor=None, frameon=False)
    ax = fig.add_axes([0.0, 0.05, 0.2, 0.9])
    cb = fig.colorbar(cs, cax=ax)
    cb.set_label('Texture lengthscale [m]', rotation=-90, color='k', labelpad=20)
    fig.savefig(os.path.normpath(os.path.join(sonpath,'class_legend.png')), transparent=False, format='png')  


    humutils.make_kml(llcrnrlon=glon.min(), llcrnrlat=glat.min(),
         urcrnrlon=glon.max(), urcrnrlat=glat.max(),
         figs=[os.path.normpath(os.path.join(sonpath,'class_overlay1.png'))], 
         colorbar=os.path.normpath(os.path.join(sonpath,'class_legend.png')),
         kmzfile=os.path.normpath(os.path.join(sonpath,'class_GroundOverlay.kmz')), 
         name='Sidescan Intensity')


    # =========================================================
    print("drawing and printing map ...")
    fig = plt.figure(frameon=False)
    map = Basemap(projection='merc', epsg=cs2cs_args.split(':')[1], 
     resolution = 'i', #h #f
     llcrnrlon=np.min(humlon)-0.001, llcrnrlat=np.min(humlat)-0.001,
     urcrnrlon=np.max(humlon)+0.001, urcrnrlat=np.max(humlat)+0.001)

    gx,gy = map.projtran(glon, glat)
       
    try:
       map.arcgisimage(server='http://server.arcgisonline.com/ArcGIS', service='ESRI_Imagery_World_2D', xpixels=1000, ypixels=None, dpi=300)
    except:
       map.arcgisimage(server='http://server.arcgisonline.com/ArcGIS', service='World_Imagery', xpixels=1000, ypixels=None, dpi=300)
    #finally:
    #   print "error: map could not be created..."
      
    ax = plt.Axes(fig, [0., 0., 1., 1.], )
    ax.set_axis_off()
    fig.add_axes(ax)

    if Sdat_gm.size > 25000000:
       print("matrix size > 25,000,000 - decimating by factor of 5 for display")
       map.pcolormesh(gx[::5,::5], gy[::5,::5], Sdat_gm[::5,::5], vmin=np.nanmin(Sdat_gm), vmax=np.nanmax(Sdat_gm))
    else:
       map.pcolormesh(gx, gy, Sdat_gm, vmin=np.nanmin(Sdat_gm), vmax=np.nanmax(Sdat_gm))

    custom_save2(sonpath,'class_map_imagery')
    del fig 

   
    if os.name=='posix': # true if linux/mac
       elapsed = (time.time() - start)
    else: # windows
       elapsed = (time.clock() - start)
    print("Processing took "+str(elapsed)+"seconds to analyse")

    print("Done!")