def print_map(cs2cs_args, glon, glat, datm, sonpath, p, vmin, vmax): #humlon, humlat, merge, try: # ========================================================= print("creating kmz file ...") ## new way to create kml file pixels = 1024 * 10 fig, ax = humutils.gearth_fig(llcrnrlon=glon.min(), llcrnrlat=glat.min(), urcrnrlon=glon.max(), urcrnrlat=glat.max(), pixels=pixels) cs = ax.pcolormesh(glon, glat, datm, cmap='pink', vmin=vmin, vmax=vmax) ax.set_axis_off() fig.savefig(os.path.normpath(os.path.join(sonpath,'class_map'+str(p)+'.png')), transparent=True, format='png') # ========================================================= fig = plt.figure(figsize=(1.0, 4.0), facecolor=None, frameon=False) ax = fig.add_axes([0.0, 0.05, 0.2, 0.9]) cb = fig.colorbar(cs, cax=ax) cb.set_label('Texture Lengthscale [m]', rotation=-90, color='k', labelpad=20) fig.savefig(os.path.normpath(os.path.join(sonpath,'class_legend'+str(p)+'.png')), transparent=False, format='png') # ========================================================= humutils.make_kml(llcrnrlon=glon.min(), llcrnrlat=glat.min(), urcrnrlon=glon.max(), urcrnrlat=glat.max(), figs=[os.path.normpath(os.path.join(sonpath,'class_map'+str(p)+'.png'))], colorbar=os.path.normpath(os.path.join(sonpath,'class_legend'+str(p)+'.png')), kmzfile=os.path.normpath(os.path.join(sonpath,'class_GroundOverlay'+str(p)+'.kmz')), name='Texture Lengthscale') except: print("error: map could not be created...")
def print_map(cs2cs_args, glon, glat, datm, sonpath, p, vmin, vmax): #humlon, humlat, merge, try: # ========================================================= print "creating kmz file ..." ## new way to create kml file pixels = 1024 * 10 fig, ax = humutils.gearth_fig(llcrnrlon=glon.min(), llcrnrlat=glat.min(), urcrnrlon=glon.max(), urcrnrlat=glat.max(), pixels=pixels) cs = ax.pcolormesh(glon, glat, datm, cmap='pink', vmin=vmin, vmax=vmax) ax.set_axis_off() fig.savefig(os.path.normpath(os.path.join(sonpath,'class_map'+str(p)+'.png')), transparent=True, format='png') # ========================================================= fig = plt.figure(figsize=(1.0, 4.0), facecolor=None, frameon=False) ax = fig.add_axes([0.0, 0.05, 0.2, 0.9]) cb = fig.colorbar(cs, cax=ax) cb.set_label('Texture Lengthscale [m]', rotation=-90, color='k', labelpad=20) fig.savefig(os.path.normpath(os.path.join(sonpath,'class_legend'+str(p)+'.png')), transparent=False, format='png') # ========================================================= humutils.make_kml(llcrnrlon=glon.min(), llcrnrlat=glat.min(), urcrnrlon=glon.max(), urcrnrlat=glat.max(), figs=[os.path.normpath(os.path.join(sonpath,'class_map'+str(p)+'.png'))], colorbar=os.path.normpath(os.path.join(sonpath,'class_legend'+str(p)+'.png')), kmzfile=os.path.normpath(os.path.join(sonpath,'class_GroundOverlay'+str(p)+'.kmz')), name='Texture Lengthscale') except: print "error: map could not be created..."
def mosaic_texture(humfile, sonpath, cs2cs_args="epsg:26949", res=99, nn=5, weight=1): ''' Create mosaics of the spatially referenced sidescan echograms Syntax ---------- [] = PyHum.mosaic_texture(humfile, sonpath, cs2cs_args, res, nn, weight) Parameters ---------- humfile : str path to the .DAT file sonpath : str path where the *.SON files are cs2cs_args : int, *optional* [Default="epsg:26949"] arguments to create coordinates in a projected coordinate system this argument gets given to pyproj to turn wgs84 (lat/lon) coordinates into any projection supported by the proj.4 libraries res : float, *optional* [Default=0] grid resolution of output gridded texture map if res=99, res will be determined automatically from the spatial resolution of 1 pixel nn: int, *optional* [Default=5] number of nearest neighbours for gridding weight: int, *optional* [Default=1] specifies the type of pixel weighting in the gridding process weight = 1, based on grazing angle and inverse distance weighting weight = 2, based on grazing angle only weight = 3, inverse distance weighting only weight = 4, no weighting Returns ------- sonpath+'GroundOverlay.kml': kml file contains gridded (or point cloud) sidescan intensity map for importing into google earth of the pth chunk sonpath+'map.png' : image overlay associated with the kml file ''' # prompt user to supply file if no input file given if not humfile: print('An input file is required!!!!!!') Tk().withdraw( ) # we don't want a full GUI, so keep the root window from appearing humfile = askopenfilename(filetypes=[("DAT files", "*.DAT")]) # prompt user to supply directory if no input sonpath is given if not sonpath: print('A *.SON directory is required!!!!!!') Tk().withdraw( ) # we don't want a full GUI, so keep the root window from appearing sonpath = askdirectory() # print given arguments to screen and convert data type where necessary if humfile: print('Input file is %s' % (humfile)) if sonpath: print('Sonar file path is %s' % (sonpath)) if cs2cs_args: print('cs2cs arguments are %s' % (cs2cs_args)) if res: res = np.asarray(res, float) print('Gridding resolution: %s' % (str(res))) if nn: nn = int(nn) print('Number of nearest neighbours for gridding: %s' % (str(nn))) if weight: weight = int(weight) print('Weighting for gridding: %s' % (str(weight))) ##nn = 5 #number of nearest neighbours in gridding ##noisefloor=10 # noise threshold in dB W # start timer if os.name == 'posix': # true if linux/mac or cygwin on windows start = time.time() else: # windows start = time.clock() trans = pyproj.Proj(init=cs2cs_args) # if son path name supplied has no separator at end, put one on if sonpath[-1] != os.sep: sonpath = sonpath + os.sep base = humfile.split('.DAT') # get base of file name for output base = base[0].split(os.sep)[-1] # remove underscores, negatives and spaces from basename base = humutils.strip_base(base) meta = loadmat(os.path.normpath(os.path.join(sonpath, base + 'meta.mat'))) esi = np.squeeze(meta['e']) nsi = np.squeeze(meta['n']) theta = np.squeeze(meta['heading']) / (180 / np.pi) # load memory mapped scans shape_port = np.squeeze(meta['shape_port']) if shape_port != '': if os.path.isfile( os.path.normpath( os.path.join(sonpath, base + '_data_port_lar.dat'))): port_fp = io.get_mmap_data(sonpath, base, '_data_port_lar.dat', 'float32', tuple(shape_port)) else: port_fp = io.get_mmap_data(sonpath, base, '_data_port_la.dat', 'float32', tuple(shape_port)) shape_star = np.squeeze(meta['shape_star']) if shape_star != '': if os.path.isfile( os.path.normpath( os.path.join(sonpath, base + '_data_star_lar.dat'))): star_fp = io.get_mmap_data(sonpath, base, '_data_star_lar.dat', 'float32', tuple(shape_star)) else: star_fp = io.get_mmap_data(sonpath, base, '_data_star_la.dat', 'float32', tuple(shape_star)) # time varying gain tvg = ((8.5 * 10**-5) + (3 / 76923) + ((8.5 * 10**-5) / 4)) * meta['c'] # depth correction dist_tvg = np.squeeze(( (np.tan(np.radians(25))) * np.squeeze(meta['dep_m'])) - (tvg)) # read in range data R_fp = io.get_mmap_data(sonpath, base, '_data_range.dat', 'float32', tuple(shape_star)) dx = np.arcsin(meta['c'] / (1000 * meta['t'] * meta['f'])) pix_m = meta['pix_m'] c = meta['c'] if not os.path.isfile(os.path.normpath(os.path.join(sonpath, base + "S.p"))): #if 2 > 1: inputfiles = [] if len(shape_star) > 2: for p in range(len(star_fp)): e = esi[shape_port[-1] * p:shape_port[-1] * (p + 1)] n = nsi[shape_port[-1] * p:shape_port[-1] * (p + 1)] t = theta[shape_port[-1] * p:shape_port[-1] * (p + 1)] d = dist_tvg[shape_port[-1] * p:shape_port[-1] * (p + 1)] dat_port = port_fp[p] dat_star = star_fp[p] data_R = R_fp[p] print("writing chunk %s " % (str(p))) write_points(e, n, t, d, dat_port, dat_star, data_R, pix_m, res, cs2cs_args, sonpath, p, c, dx) inputfiles.append( os.path.normpath( os.path.join(sonpath, 'x_y_class' + str(p) + '.asc'))) else: p = 0 print("writing chunk %s " % (str(p))) write_points(esi, nsi, theta, dist_tvg, port_fp, star_fp, R_fp, meta['pix_m'], res, cs2cs_args, sonpath, 0, c, dx) inputfiles.append( os.path.normpath( os.path.join(sonpath, 'x_y_class' + str(p) + '.asc'))) #trans = pyproj.Proj(init=cs2cs_args) # D, R, h, t print("reading points from %s files" % (str(len(inputfiles)))) X, Y, S, D, R, h, t, i = getxys(inputfiles) print("%s points read from %s files" % (str(len(S)), str(len(inputfiles)))) # remove values where sidescan intensity is zero ind = np.where(np.logical_not(S == 0))[0] X = X[ind] Y = Y[ind] S = S[ind] D = D[ind] R = R[ind] h = h[ind] t = t[ind] i = i[ind] del ind # save to file for temporary storage pickle.dump( S, open(os.path.normpath(os.path.join(sonpath, base + "S.p")), "wb")) del S pickle.dump( D, open(os.path.normpath(os.path.join(sonpath, base + "D.p")), "wb")) del D pickle.dump( t, open(os.path.normpath(os.path.join(sonpath, base + "t.p")), "wb")) del t pickle.dump( i, open(os.path.normpath(os.path.join(sonpath, base + "i.p")), "wb")) del i pickle.dump( X, open(os.path.normpath(os.path.join(sonpath, base + "X.p")), "wb")) del X pickle.dump( Y, open(os.path.normpath(os.path.join(sonpath, base + "Y.p")), "wb")) del Y pickle.dump( R, open(os.path.normpath(os.path.join(sonpath, base + "R.p")), "wb")) pickle.dump( h, open(os.path.normpath(os.path.join(sonpath, base + "h.p")), "wb")) #grazing angle g = np.arctan(R.flatten(), h.flatten()) pickle.dump( g, open(os.path.normpath(os.path.join(sonpath, base + "g.p")), "wb")) del g, R, h print("creating grids ...") if res == 0: res = 99 if res == 99: #### prepare grids R = pickle.load( open(os.path.normpath(os.path.join(sonpath, base + "R.p")), "rb")) ## actual along-track resolution is this: dx times dy = Af tmp = R * dx * (c * 0.007 / 2) del R resg = np.min(tmp[tmp > 0]) del tmp else: resg = res X = pickle.load( open(os.path.normpath(os.path.join(sonpath, base + "X.p")), "rb")) Y = pickle.load( open(os.path.normpath(os.path.join(sonpath, base + "Y.p")), "rb")) humlon, humlat = trans(X, Y, inverse=True) grid_x, grid_y = np.meshgrid(np.arange(np.min(X), np.max(X), resg), np.arange(np.min(Y), np.max(Y), resg)) shape = np.shape(grid_x) tree = KDTree(zip(X.flatten(), Y.flatten())) del X, Y print("mosaicking ...") #k nearest neighbour try: dist, inds = tree.query(zip(grid_x.flatten(), grid_y.flatten()), k=nn, n_jobs=-1) except: #print ".... update your scipy installation to use faster kd-tree" dist, inds = tree.query(zip(grid_x.flatten(), grid_y.flatten()), k=nn) #del grid_x, grid_y if weight == 1: g = pickle.load( open(os.path.normpath(os.path.join(sonpath, base + "g.p")), "rb")) w = g[inds] + 1.0 / dist**2 del g elif weight == 2: g = pickle.load( open(os.path.normpath(os.path.join(sonpath, base + "g.p")), "rb")) w = g[inds] del g elif weight == 3: w = 1.0 / dist**2 elif weight == 4: w = 1.0 #g = pickle.load( open( os.path.normpath(os.path.join(sonpath,base+"g.p")), "rb" ) ) #w = g[inds] + 1.0 / dist**2 #del g if weight < 4: w[np.isinf(w)] = 1 w[np.isnan(w)] = 1 w[w > 10000] = 10000 w[w <= 0] = 1 # load in sidescan intensity S = pickle.load( open(os.path.normpath(os.path.join(sonpath, base + "S.p")), "rb")) # filter out noise pixels S[S < noisefloor] = np.nan if nn == 1: Sdat_g = (w * S.flatten()[inds]).reshape(shape) del w dist = dist.reshape(shape) else: if weight < 4: Sdat_g = (np.nansum(w * S.flatten()[inds], axis=1) / np.nansum(w, axis=1)).reshape(shape) else: Sdat_g = (np.nansum(S.flatten()[inds], axis=1)).reshape(shape) del w dist = np.nanmean(dist, axis=1).reshape(shape) del S Sdat_g[dist > 1] = np.nan Sdat_g[Sdat_g < noisefloor] = np.nan dat = Sdat_g.copy() dat[dist > 1] = 0 dat2 = replace_nans.RN(dat.astype('float64'), 1000, 0.01, 2, 'localmean').getdata() dat2[dat == 0] = np.nan del dat dat2[dat2 < noisefloor] = np.nan Sdat_g = dat2.copy() del dat2 Sdat_g[Sdat_g == 0] = np.nan Sdat_g[np.isinf(Sdat_g)] = np.nan Sdat_gm = np.ma.masked_invalid(Sdat_g) del Sdat_g glon, glat = trans(grid_x, grid_y, inverse=True) del grid_x, grid_y # ========================================================= print("creating kmz file ...") ## new way to create kml file pixels = 1024 * 10 fig, ax = humutils.gearth_fig(llcrnrlon=glon.min(), llcrnrlat=glat.min(), urcrnrlon=glon.max(), urcrnrlat=glat.max(), pixels=pixels) cs = ax.pcolormesh(glon, glat, Sdat_gm) ax.set_axis_off() fig.savefig(os.path.normpath(os.path.join(sonpath, 'class_overlay1.png')), transparent=True, format='png') fig = plt.figure(figsize=(1.0, 4.0), facecolor=None, frameon=False) ax = fig.add_axes([0.0, 0.05, 0.2, 0.9]) cb = fig.colorbar(cs, cax=ax) cb.set_label('Texture lengthscale [m]', rotation=-90, color='k', labelpad=20) fig.savefig(os.path.normpath(os.path.join(sonpath, 'class_legend.png')), transparent=False, format='png') humutils.make_kml( llcrnrlon=glon.min(), llcrnrlat=glat.min(), urcrnrlon=glon.max(), urcrnrlat=glat.max(), figs=[os.path.normpath(os.path.join(sonpath, 'class_overlay1.png'))], colorbar=os.path.normpath(os.path.join(sonpath, 'class_legend.png')), kmzfile=os.path.normpath( os.path.join(sonpath, 'class_GroundOverlay.kmz')), name='Sidescan Intensity') # ========================================================= print("drawing and printing map ...") fig = plt.figure(frameon=False) map = Basemap( projection='merc', epsg=cs2cs_args.split(':')[1], resolution='i', #h #f llcrnrlon=np.min(humlon) - 0.001, llcrnrlat=np.min(humlat) - 0.001, urcrnrlon=np.max(humlon) + 0.001, urcrnrlat=np.max(humlat) + 0.001) gx, gy = map.projtran(glon, glat) try: map.arcgisimage(server='http://server.arcgisonline.com/ArcGIS', service='ESRI_Imagery_World_2D', xpixels=1000, ypixels=None, dpi=300) except: map.arcgisimage(server='http://server.arcgisonline.com/ArcGIS', service='World_Imagery', xpixels=1000, ypixels=None, dpi=300) #finally: # print "error: map could not be created..." ax = plt.Axes( fig, [0., 0., 1., 1.], ) ax.set_axis_off() fig.add_axes(ax) if Sdat_gm.size > 25000000: print( "matrix size > 25,000,000 - decimating by factor of 5 for display") map.pcolormesh(gx[::5, ::5], gy[::5, ::5], Sdat_gm[::5, ::5], vmin=np.nanmin(Sdat_gm), vmax=np.nanmax(Sdat_gm)) else: map.pcolormesh(gx, gy, Sdat_gm, vmin=np.nanmin(Sdat_gm), vmax=np.nanmax(Sdat_gm)) custom_save2(sonpath, 'class_map_imagery') del fig if os.name == 'posix': # true if linux/mac elapsed = (time.time() - start) else: # windows elapsed = (time.clock() - start) print("Processing took " + str(elapsed) + "seconds to analyse") print("Done!")
def make_map(e, n, t, d, dat_port, dat_star, data_R, pix_m, res, cs2cs_args, sonpath, p, mode, nn, numstdevs, c, dx, use_uncorrected, scalemax): #dogrid, influence,dowrite, thres=5 trans = pyproj.Proj(init=cs2cs_args) mp = np.nanmean(dat_port) ms = np.nanmean(dat_star) if mp>ms: merge = np.vstack((dat_port,dat_star*(mp/ms))) else: merge = np.vstack((dat_port*(ms/mp),dat_star)) del dat_port, dat_star merge[np.isnan(merge)] = 0 merge = merge[:,:len(n)] ## actual along-track resolution is this: dx times dy = Af tmp = data_R * dx * (c*0.007 / 2) #dx = np.arcsin(c/(1000*meta['t']*meta['f'])) res_grid = np.sqrt(np.vstack((tmp, tmp))) del tmp res_grid = res_grid[:np.shape(merge)[0],:np.shape(merge)[1]] #if use_uncorrected != 1: # merge = merge - 10*np.log10(res_grid) res_grid = res_grid.astype('float32') merge[np.isnan(merge)] = 0 merge[merge<0] = 0 merge = merge.astype('float32') merge = denoise_tv_chambolle(merge.copy(), weight=.2, multichannel=False).astype('float32') R = np.vstack((np.flipud(data_R),data_R)) del data_R R = R[:np.shape(merge)[0],:np.shape(merge)[1]] # get number pixels in scan line extent = int(np.shape(merge)[0]/2) yvec = np.squeeze(np.linspace(np.squeeze(pix_m),extent*np.squeeze(pix_m),extent)) X, Y, D, h, t = getXY(e,n,yvec,np.squeeze(d),t,extent) X = X.astype('float32') Y = Y.astype('float32') D = D.astype('float32') h = h.astype('float32') t = t.astype('float32') X = X.astype('float32') D[np.isnan(D)] = 0 h[np.isnan(h)] = 0 t[np.isnan(t)] = 0 X = X[np.where(np.logical_not(np.isnan(Y)))] merge = merge.flatten()[np.where(np.logical_not(np.isnan(Y)))] res_grid = res_grid.flatten()[np.where(np.logical_not(np.isnan(Y)))] Y = Y[np.where(np.logical_not(np.isnan(Y)))] D = D[np.where(np.logical_not(np.isnan(Y)))] R = R.flatten()[np.where(np.logical_not(np.isnan(Y)))] h = h[np.where(np.logical_not(np.isnan(Y)))] t = t[np.where(np.logical_not(np.isnan(Y)))] Y = Y[np.where(np.logical_not(np.isnan(X)))] merge = merge.flatten()[np.where(np.logical_not(np.isnan(X)))] res_grid = res_grid.flatten()[np.where(np.logical_not(np.isnan(X)))] X = X[np.where(np.logical_not(np.isnan(X)))] D = D[np.where(np.logical_not(np.isnan(X)))] R = R.flatten()[np.where(np.logical_not(np.isnan(X)))] h = h[np.where(np.logical_not(np.isnan(X)))] t = t[np.where(np.logical_not(np.isnan(X)))] X = X[np.where(np.logical_not(np.isnan(merge)))] Y = Y[np.where(np.logical_not(np.isnan(merge)))] merge = merge[np.where(np.logical_not(np.isnan(merge)))] res_grid = res_grid.flatten()[np.where(np.logical_not(np.isnan(merge)))] D = D[np.where(np.logical_not(np.isnan(merge)))] R = R[np.where(np.logical_not(np.isnan(merge)))] h = h[np.where(np.logical_not(np.isnan(merge)))] t = t[np.where(np.logical_not(np.isnan(merge)))] X = X[np.where(np.logical_not(np.isinf(merge)))] Y = Y[np.where(np.logical_not(np.isinf(merge)))] merge = merge[np.where(np.logical_not(np.isinf(merge)))] res_grid = res_grid.flatten()[np.where(np.logical_not(np.isinf(merge)))] D = D[np.where(np.logical_not(np.isinf(merge)))] R = R[np.where(np.logical_not(np.isinf(merge)))] h = h[np.where(np.logical_not(np.isinf(merge)))] t = t[np.where(np.logical_not(np.isinf(merge)))] print("writing point cloud") #if dowrite==1: ## write raw bs to file outfile = os.path.normpath(os.path.join(sonpath,'x_y_ss_raw'+str(p)+'.asc')) ##write.txtwrite( outfile, np.hstack((humutils.ascol(X.flatten()),humutils.ascol(Y.flatten()), humutils.ascol(merge.flatten()), humutils.ascol(D.flatten()), humutils.ascol(R.flatten()), humutils.ascol(h.flatten()), humutils.ascol(t.flatten()) )) ) np.savetxt(outfile, np.hstack((humutils.ascol(X.flatten()),humutils.ascol(Y.flatten()), humutils.ascol(merge.flatten()), humutils.ascol(D.flatten()), humutils.ascol(R.flatten()), humutils.ascol(h.flatten()), humutils.ascol(t.flatten()) )) , fmt="%8.6f %8.6f %8.6f %8.6f %8.6f %8.6f %8.6f") del D, R, h, t sigmas = 0.1 #m eps = 2 print("gridding ...") #if dogrid==1: if 2>1: if res==99: resg = np.min(res_grid[res_grid>0])/2 print('Gridding at resolution of %s' % str(resg)) else: resg = res tree = KDTree(np.c_[X.flatten(),Y.flatten()]) complete=0 while complete==0: try: grid_x, grid_y, res = getmesh(np.min(X), np.max(X), np.min(Y), np.max(Y), resg) longrid, latgrid = trans(grid_x, grid_y, inverse=True) longrid = longrid.astype('float32') latgrid = latgrid.astype('float32') shape = np.shape(grid_x) ## create mask for where the data is not if pykdtree==1: dist, _ = tree.query(np.c_[grid_x.ravel(), grid_y.ravel()], k=1) else: try: dist, _ = tree.query(np.c_[grid_x.ravel(), grid_y.ravel()], k=1, n_jobs=cpu_count()) except: #print ".... update your scipy installation to use faster kd-tree queries" dist, _ = tree.query(np.c_[grid_x.ravel(), grid_y.ravel()], k=1) dist = dist.reshape(grid_x.shape) targ_def = pyresample.geometry.SwathDefinition(lons=longrid.flatten(), lats=latgrid.flatten()) del longrid, latgrid humlon, humlat = trans(X, Y, inverse=True) orig_def = pyresample.geometry.SwathDefinition(lons=humlon.flatten(), lats=humlat.flatten()) del humlon, humlat if 'orig_def' in locals(): complete=1 except: print("memory error: trying grid resolution of %s" % (str(resg*2))) resg = resg*2 if mode==1: complete=0 while complete==0: try: try: dat = pyresample.kd_tree.resample_nearest(orig_def, merge.flatten(), targ_def, radius_of_influence=res*20, fill_value=None, nprocs = cpu_count(), reduce_data=1) except: dat = pyresample.kd_tree.resample_nearest(orig_def, merge.flatten(), targ_def, radius_of_influence=res*20, fill_value=None, nprocs = 1, reduce_data=1) try: r_dat = pyresample.kd_tree.resample_nearest(orig_def, res_grid.flatten(), targ_def, radius_of_influence=res*20, fill_value=None, nprocs = cpu_count(), reduce_data=1) except: r_dat = pyresample.kd_tree.resample_nearest(orig_def, res_grid.flatten(), targ_def, radius_of_influence=res*20, fill_value=None, nprocs = 1, reduce_data=1) stdev = None counts = None if 'dat' in locals(): complete=1 except: del grid_x, grid_y, targ_def, orig_def wf = None humlon, humlat = trans(X, Y, inverse=True) dat, stdev, counts, resg, complete, shape = getgrid_lm(humlon, humlat, merge, res*10, min(X), max(X), min(Y), max(Y), resg*2, mode, trans, nn, wf, sigmas, eps) r_dat, stdev, counts, resg, complete, shape = getgrid_lm(humlon, humlat, res_grid, res*10, min(X), max(X), min(Y), max(Y), resg*2, mode, trans, nn, wf, sigmas, eps) del humlon, humlat elif mode==2: # custom inverse distance wf = lambda r: 1/r**2 complete=0 while complete==0: try: try: dat, stdev, counts = pyresample.kd_tree.resample_custom(orig_def, merge.flatten(),targ_def, radius_of_influence=res*20, neighbours=nn, weight_funcs=wf, fill_value=None, with_uncert = True, nprocs = cpu_count(), reduce_data=1) except: dat, stdev, counts = pyresample.kd_tree.resample_custom(orig_def, merge.flatten(),targ_def, radius_of_influence=res*20, neighbours=nn, weight_funcs=wf, fill_value=None, with_uncert = True, nprocs = 1, reduce_data=1) try: r_dat = pyresample.kd_tree.resample_custom(orig_def, res_grid.flatten(), targ_def, radius_of_influence=res*20, neighbours=nn, weight_funcs=wf, fill_value=None, with_uncert = False, nprocs = cpu_count(), reduce_data=1) except: r_dat = pyresample.kd_tree.resample_custom(orig_def, res_grid.flatten(), targ_def, radius_of_influence=res*20, neighbours=nn, weight_funcs=wf, fill_value=None, with_uncert = False, nprocs = 1, reduce_data=1) if 'dat' in locals(): complete=1 except: del grid_x, grid_y, targ_def, orig_def humlon, humlat = trans(X, Y, inverse=True) dat, stdev, counts, resg, complete, shape = getgrid_lm(humlon, humlat, merge, res*2, min(X), max(X), min(Y), max(Y), resg*2, mode, trans, nn, wf, sigmas, eps) r_dat, stdev, counts, resg, complete, shape = getgrid_lm(humlon, humlat, res_grid, res*2, min(X), max(X), min(Y), max(Y), resg*2, mode, trans, nn, wf, sigmas, eps) del humlat, humlon del stdev_null, counts_null elif mode==3: wf = None complete=0 while complete==0: try: try: dat, stdev, counts = pyresample.kd_tree.resample_gauss(orig_def, merge.flatten(), targ_def, radius_of_influence=res*20, neighbours=nn, sigmas=sigmas, fill_value=None, with_uncert = True, nprocs = cpu_count(), epsilon = eps, reduce_data=1) except: dat, stdev, counts = pyresample.kd_tree.resample_gauss(orig_def, merge.flatten(), targ_def, radius_of_influence=res*20, neighbours=nn, sigmas=sigmas, fill_value=None, with_uncert = True, nprocs = 1, epsilon = eps, reduce_data=1) try: r_dat = pyresample.kd_tree.resample_gauss(orig_def, res_grid.flatten(), targ_def, radius_of_influence=res*20, neighbours=nn, sigmas=sigmas, fill_value=None, with_uncert = False, nprocs = cpu_count(), epsilon = eps, reduce_data=1) except: r_dat = pyresample.kd_tree.resample_gauss(orig_def, res_grid.flatten(), targ_def, radius_of_influence=res*20, neighbours=nn, sigmas=sigmas, fill_value=None, with_uncert = False, nprocs = 1, epsilon = eps, reduce_data=1) if 'dat' in locals(): complete=1 except: del grid_x, grid_y, targ_def, orig_def humlon, humlat = trans(X, Y, inverse=True) dat, stdev, counts, resg, complete, shape = getgrid_lm(humlon, humlat, merge, res*10, min(X), max(X), min(Y), max(Y), resg*2, mode, trans, nn, wf, sigmas, eps) r_dat, stdev_null, counts_null, resg, complete, shape = getgrid_lm(humlon, humlat, res_grid, res*10, min(X), max(X), min(Y), max(Y), resg*2, mode, trans, nn, wf, sigmas, eps) del humlat, humlon del stdev_null, counts_null humlon, humlat = trans(X, Y, inverse=True) del X, Y, res_grid, merge dat = dat.reshape(shape) dat[dist>res*30] = np.nan del dist r_dat = r_dat.reshape(shape) r_dat[r_dat<1] = 1 r_dat[r_dat > 2*np.pi] = 1 r_dat[np.isnan(dat)] = np.nan dat = dat + r_dat #np.sqrt(np.cos(np.deg2rad(r_dat))) #dat*np.sqrt(r_dat) + dat del r_dat if mode>1: stdev = stdev.reshape(shape) counts = counts.reshape(shape) mask = dat.mask.copy() dat[mask==1] = np.nan #dat[mask==1] = 0 if mode>1: dat[(stdev>numstdevs) & (mask!=0)] = np.nan dat[(counts<nn) & (counts>0)] = np.nan #if dogrid==1: dat[dat==0] = np.nan dat[np.isinf(dat)] = np.nan dat[dat<thres] = np.nan datm = np.ma.masked_invalid(dat) glon, glat = trans(grid_x, grid_y, inverse=True) #del grid_x, grid_y try: from osgeo import gdal,ogr,osr proj = osr.SpatialReference() proj.ImportFromEPSG(int(cs2cs_args.split(':')[-1])) #26949) datout = np.squeeze(np.ma.filled(dat))#.astype('int16') datout[np.isnan(datout)] = -99 driver = gdal.GetDriverByName('GTiff') #rows,cols = np.shape(datout) cols,rows = np.shape(datout) outFile = os.path.normpath(os.path.join(sonpath,'geotiff_map'+str(p)+'.tif')) ds = driver.Create( outFile, rows, cols, 1, gdal.GDT_Float32, [ 'COMPRESS=LZW' ] ) if proj is not None: ds.SetProjection(proj.ExportToWkt()) xmin, ymin, xmax, ymax = [grid_x.min(), grid_y.min(), grid_x.max(), grid_y.max()] xres = (xmax - xmin) / float(rows) yres = (ymax - ymin) / float(cols) geotransform = (xmin, xres, 0, ymax, 0, -yres) ds.SetGeoTransform(geotransform) ss_band = ds.GetRasterBand(1) ss_band.WriteArray(np.flipud(datout)) #datout) ss_band.SetNoDataValue(-99) ss_band.FlushCache() ss_band.ComputeStatistics(False) del ds except: print("error: geotiff could not be created... check your gdal/ogr install") try: # ========================================================= print("creating kmz file ...") ## new way to create kml file pixels = 1024 * 10 fig, ax = humutils.gearth_fig(llcrnrlon=glon.min(), llcrnrlat=glat.min(), urcrnrlon=glon.max(), urcrnrlat=glat.max(), pixels=pixels) cs = ax.pcolormesh(glon, glat, datm, vmax=scalemax, cmap='gray') ax.set_axis_off() fig.savefig(os.path.normpath(os.path.join(sonpath,'map'+str(p)+'.png')), transparent=True, format='png') del fig, ax # ========================================================= fig = plt.figure(figsize=(1.0, 4.0), facecolor=None, frameon=False) ax = fig.add_axes([0.0, 0.05, 0.2, 0.9]) cb = fig.colorbar(cs, cax=ax) cb.set_label('Intensity [dB W]', rotation=-90, color='k', labelpad=20) fig.savefig(os.path.normpath(os.path.join(sonpath,'legend'+str(p)+'.png')), transparent=False, format='png') del fig, ax, cs, cb # ========================================================= humutils.make_kml(llcrnrlon=glon.min(), llcrnrlat=glat.min(), urcrnrlon=glon.max(), urcrnrlat=glat.max(), figs=[os.path.normpath(os.path.join(sonpath,'map'+str(p)+'.png'))], colorbar=os.path.normpath(os.path.join(sonpath,'legend'+str(p)+'.png')), kmzfile=os.path.normpath(os.path.join(sonpath,'GroundOverlay'+str(p)+'.kmz')), name='Sidescan Intensity') except: print("error: map could not be created...") #y1 = np.min(glat)-0.001 #x1 = np.min(glon)-0.001 #y2 = np.max(glat)+0.001 #x2 = np.max(glon)+0.001 print("drawing and printing map ...") fig = plt.figure(frameon=False) map = Basemap(projection='merc', epsg=cs2cs_args.split(':')[1], resolution = 'i', #h #f llcrnrlon=np.min(humlon)-0.001, llcrnrlat=np.min(glat)-0.001, urcrnrlon=np.max(humlon)+0.001, urcrnrlat=np.max(glat)+0.001) try: map.arcgisimage(server='http://server.arcgisonline.com/ArcGIS', service='World_Imagery', xpixels=1000, ypixels=None, dpi=300) except: map.arcgisimage(server='http://server.arcgisonline.com/ArcGIS', service='ESRI_Imagery_World_2D', xpixels=1000, ypixels=None, dpi=300) #finally: # print "error: map could not be created..." #if dogrid==1: gx,gy = map.projtran(glon, glat) ax = plt.Axes(fig, [0., 0., 1., 1.], ) ax.set_axis_off() fig.add_axes(ax) #if dogrid==1: if 2>1: if datm.size > 25000000: print("matrix size > 25,000,000 - decimating by factor of 5 for display") map.pcolormesh(gx[::5,::5], gy[::5,::5], datm[::5,::5], cmap='gray', vmin=np.nanmin(datm), vmax=scalemax) #vmax=np.nanmax(datm) else: map.pcolormesh(gx, gy, datm, cmap='gray', vmin=np.nanmin(datm), vmax=scalemax) #vmax=np.nanmax(datm) del datm, dat else: ## draw point cloud x,y = map.projtran(humlon, humlat) map.scatter(x.flatten(), y.flatten(), 0.5, merge.flatten(), cmap='gray', linewidth = '0') #map.drawmapscale(x1+0.001, y1+0.001, x1, y1, 200., units='m', barstyle='fancy', labelstyle='simple', fontcolor='k') #'#F8F8FF') #map.drawparallels(np.arange(y1-0.001, y2+0.001, 0.005),labels=[1,0,0,1], linewidth=0.0, rotation=30, fontsize=8) #map.drawmeridians(np.arange(x1, x2, 0.002),labels=[1,0,0,1], linewidth=0.0, rotation=30, fontsize=8) custom_save2(sonpath,'map_imagery'+str(p)) del fig del humlat, humlon return res #return the new resolution
def mosaic_texture(humfile, sonpath, cs2cs_args = "epsg:26949", res = 99, nn = 5, weight = 1): ''' Create mosaics of the spatially referenced sidescan echograms Syntax ---------- [] = PyHum.mosaic_texture(humfile, sonpath, cs2cs_args, res, nn, weight) Parameters ---------- humfile : str path to the .DAT file sonpath : str path where the *.SON files are cs2cs_args : int, *optional* [Default="epsg:26949"] arguments to create coordinates in a projected coordinate system this argument gets given to pyproj to turn wgs84 (lat/lon) coordinates into any projection supported by the proj.4 libraries res : float, *optional* [Default=0] grid resolution of output gridded texture map if res=99, res will be determined automatically from the spatial resolution of 1 pixel nn: int, *optional* [Default=5] number of nearest neighbours for gridding weight: int, *optional* [Default=1] specifies the type of pixel weighting in the gridding process weight = 1, based on grazing angle and inverse distance weighting weight = 2, based on grazing angle only weight = 3, inverse distance weighting only weight = 4, no weighting Returns ------- sonpath+'GroundOverlay.kml': kml file contains gridded (or point cloud) sidescan intensity map for importing into google earth of the pth chunk sonpath+'map.png' : image overlay associated with the kml file ''' # prompt user to supply file if no input file given if not humfile: print('An input file is required!!!!!!') Tk().withdraw() # we don't want a full GUI, so keep the root window from appearing humfile = askopenfilename(filetypes=[("DAT files","*.DAT")]) # prompt user to supply directory if no input sonpath is given if not sonpath: print('A *.SON directory is required!!!!!!') Tk().withdraw() # we don't want a full GUI, so keep the root window from appearing sonpath = askdirectory() # print given arguments to screen and convert data type where necessary if humfile: print('Input file is %s' % (humfile)) if sonpath: print('Sonar file path is %s' % (sonpath)) if cs2cs_args: print('cs2cs arguments are %s' % (cs2cs_args)) if res: res = np.asarray(res,float) print('Gridding resolution: %s' % (str(res))) if nn: nn = int(nn) print('Number of nearest neighbours for gridding: %s' % (str(nn))) if weight: weight = int(weight) print('Weighting for gridding: %s' % (str(weight))) ##nn = 5 #number of nearest neighbours in gridding ##noisefloor=10 # noise threshold in dB W # start timer if os.name=='posix': # true if linux/mac or cygwin on windows start = time.time() else: # windows start = time.clock() trans = pyproj.Proj(init=cs2cs_args) # if son path name supplied has no separator at end, put one on if sonpath[-1]!=os.sep: sonpath = sonpath + os.sep base = humfile.split('.DAT') # get base of file name for output base = base[0].split(os.sep)[-1] # remove underscores, negatives and spaces from basename base = humutils.strip_base(base) meta = loadmat(os.path.normpath(os.path.join(sonpath,base+'meta.mat'))) esi = np.squeeze(meta['e']) nsi = np.squeeze(meta['n']) theta = np.squeeze(meta['heading'])/(180/np.pi) # load memory mapped scans shape_port = np.squeeze(meta['shape_port']) if shape_port!='': if os.path.isfile(os.path.normpath(os.path.join(sonpath,base+'_data_port_lar.dat'))): port_fp = io.get_mmap_data(sonpath, base, '_data_port_lar.dat', 'float32', tuple(shape_port)) else: port_fp = io.get_mmap_data(sonpath, base, '_data_port_la.dat', 'float32', tuple(shape_port)) shape_star = np.squeeze(meta['shape_star']) if shape_star!='': if os.path.isfile(os.path.normpath(os.path.join(sonpath,base+'_data_star_lar.dat'))): star_fp = io.get_mmap_data(sonpath, base, '_data_star_lar.dat', 'float32', tuple(shape_star)) else: star_fp = io.get_mmap_data(sonpath, base, '_data_star_la.dat', 'float32', tuple(shape_star)) # time varying gain tvg = ((8.5*10**-5)+(3/76923)+((8.5*10**-5)/4))*meta['c'] # depth correction dist_tvg = np.squeeze(((np.tan(np.radians(25)))*np.squeeze(meta['dep_m']))-(tvg)) # read in range data R_fp = io.get_mmap_data(sonpath, base, '_data_range.dat', 'float32', tuple(shape_star)) dx = np.arcsin(meta['c']/(1000*meta['t']*meta['f'])) pix_m = meta['pix_m'] c = meta['c'] if not os.path.isfile( os.path.normpath(os.path.join(sonpath,base+"S.p")) ): #if 2 > 1: inputfiles = [] if len(shape_star)>2: for p in range(len(star_fp)): e = esi[shape_port[-1]*p:shape_port[-1]*(p+1)] n = nsi[shape_port[-1]*p:shape_port[-1]*(p+1)] t = theta[shape_port[-1]*p:shape_port[-1]*(p+1)] d = dist_tvg[shape_port[-1]*p:shape_port[-1]*(p+1)] dat_port = port_fp[p] dat_star = star_fp[p] data_R = R_fp[p] print("writing chunk %s " % (str(p))) write_points(e, n, t, d, dat_port, dat_star, data_R, pix_m, res, cs2cs_args, sonpath, p, c, dx) inputfiles.append(os.path.normpath(os.path.join(sonpath,'x_y_class'+str(p)+'.asc'))) else: p=0 print("writing chunk %s " % (str(p))) write_points(esi, nsi, theta, dist_tvg, port_fp, star_fp, R_fp, meta['pix_m'], res, cs2cs_args, sonpath, 0, c, dx) inputfiles.append(os.path.normpath(os.path.join(sonpath,'x_y_class'+str(p)+'.asc'))) #trans = pyproj.Proj(init=cs2cs_args) # D, R, h, t print("reading points from %s files" % (str(len(inputfiles)))) X,Y,S,D,R,h,t,i = getxys(inputfiles) print("%s points read from %s files" % (str(len(S)), str(len(inputfiles)))) # remove values where sidescan intensity is zero ind = np.where(np.logical_not(S==0))[0] X = X[ind]; Y = Y[ind] S = S[ind]; D = D[ind] R = R[ind]; h = h[ind] t = t[ind]; i = i[ind] del ind # save to file for temporary storage pickle.dump( S, open( os.path.normpath(os.path.join(sonpath,base+"S.p")), "wb" ) ); del S pickle.dump( D, open( os.path.normpath(os.path.join(sonpath,base+"D.p")), "wb" ) ); del D pickle.dump( t, open( os.path.normpath(os.path.join(sonpath,base+"t.p")), "wb" ) ); del t pickle.dump( i, open( os.path.normpath(os.path.join(sonpath,base+"i.p")), "wb" ) ); del i pickle.dump( X, open( os.path.normpath(os.path.join(sonpath,base+"X.p")), "wb" ) ); del X pickle.dump( Y, open( os.path.normpath(os.path.join(sonpath,base+"Y.p")), "wb" ) ); del Y pickle.dump( R, open( os.path.normpath(os.path.join(sonpath,base+"R.p")), "wb" ) ); pickle.dump( h, open( os.path.normpath(os.path.join(sonpath,base+"h.p")), "wb" ) ); #grazing angle g = np.arctan(R.flatten(),h.flatten()) pickle.dump( g, open( os.path.normpath(os.path.join(sonpath,base+"g.p")), "wb" ) ); del g, R, h print("creating grids ...") if res==0: res=99 if res==99: #### prepare grids R = pickle.load( open( os.path.normpath(os.path.join(sonpath,base+"R.p")), "rb" ) ) ## actual along-track resolution is this: dx times dy = Af tmp = R * dx * (c*0.007 / 2) del R resg = np.min(tmp[tmp>0]) del tmp else: resg = res X = pickle.load( open( os.path.normpath(os.path.join(sonpath,base+"X.p")), "rb" ) ) Y = pickle.load( open( os.path.normpath(os.path.join(sonpath,base+"Y.p")), "rb" ) ) humlon, humlat = trans(X, Y, inverse=True) grid_x, grid_y = np.meshgrid( np.arange(np.min(X), np.max(X), resg), np.arange(np.min(Y), np.max(Y), resg) ) shape = np.shape(grid_x) tree = KDTree(zip(X.flatten(), Y.flatten())) del X, Y print("mosaicking ...") #k nearest neighbour try: dist, inds = tree.query(zip(grid_x.flatten(), grid_y.flatten()), k = nn, n_jobs=-1) except: #print ".... update your scipy installation to use faster kd-tree" dist, inds = tree.query(zip(grid_x.flatten(), grid_y.flatten()), k = nn) #del grid_x, grid_y if weight==1: g = pickle.load( open( os.path.normpath(os.path.join(sonpath,base+"g.p")), "rb" ) ) w = g[inds] + 1.0 / dist**2 del g elif weight==2: g = pickle.load( open( os.path.normpath(os.path.join(sonpath,base+"g.p")), "rb" ) ) w = g[inds] del g elif weight==3: w = 1.0 / dist**2 elif weight==4: w = 1.0 #g = pickle.load( open( os.path.normpath(os.path.join(sonpath,base+"g.p")), "rb" ) ) #w = g[inds] + 1.0 / dist**2 #del g if weight < 4: w[np.isinf(w)]=1 w[np.isnan(w)]=1 w[w>10000]=10000 w[w<=0]=1 # load in sidescan intensity S = pickle.load( open( os.path.normpath(os.path.join(sonpath,base+"S.p")), "rb" ) ) # filter out noise pixels S[S<noisefloor] = np.nan if nn==1: Sdat_g = (w * S.flatten()[inds]).reshape(shape) del w dist = dist.reshape(shape) else: if weight < 4: Sdat_g = (np.nansum(w * S.flatten()[inds], axis=1) / np.nansum(w, axis=1)).reshape(shape) else: Sdat_g = (np.nansum(S.flatten()[inds], axis=1)).reshape(shape) del w dist = np.nanmean(dist,axis=1).reshape(shape) del S Sdat_g[dist>1] = np.nan Sdat_g[Sdat_g<noisefloor] = np.nan dat = Sdat_g.copy() dat[dist>1] = 0 dat2 = replace_nans.RN(dat.astype('float64'),1000,0.01,2,'localmean').getdata() dat2[dat==0] = np.nan del dat dat2[dat2<noisefloor] = np.nan Sdat_g = dat2.copy() del dat2 Sdat_g[Sdat_g==0] = np.nan Sdat_g[np.isinf(Sdat_g)] = np.nan Sdat_gm = np.ma.masked_invalid(Sdat_g) del Sdat_g glon, glat = trans(grid_x, grid_y, inverse=True) del grid_x, grid_y # ========================================================= print("creating kmz file ...") ## new way to create kml file pixels = 1024 * 10 fig, ax = humutils.gearth_fig(llcrnrlon=glon.min(), llcrnrlat=glat.min(), urcrnrlon=glon.max(), urcrnrlat=glat.max(), pixels=pixels) cs = ax.pcolormesh(glon, glat, Sdat_gm) ax.set_axis_off() fig.savefig(os.path.normpath(os.path.join(sonpath,'class_overlay1.png')), transparent=True, format='png') fig = plt.figure(figsize=(1.0, 4.0), facecolor=None, frameon=False) ax = fig.add_axes([0.0, 0.05, 0.2, 0.9]) cb = fig.colorbar(cs, cax=ax) cb.set_label('Texture lengthscale [m]', rotation=-90, color='k', labelpad=20) fig.savefig(os.path.normpath(os.path.join(sonpath,'class_legend.png')), transparent=False, format='png') humutils.make_kml(llcrnrlon=glon.min(), llcrnrlat=glat.min(), urcrnrlon=glon.max(), urcrnrlat=glat.max(), figs=[os.path.normpath(os.path.join(sonpath,'class_overlay1.png'))], colorbar=os.path.normpath(os.path.join(sonpath,'class_legend.png')), kmzfile=os.path.normpath(os.path.join(sonpath,'class_GroundOverlay.kmz')), name='Sidescan Intensity') # ========================================================= print("drawing and printing map ...") fig = plt.figure(frameon=False) map = Basemap(projection='merc', epsg=cs2cs_args.split(':')[1], resolution = 'i', #h #f llcrnrlon=np.min(humlon)-0.001, llcrnrlat=np.min(humlat)-0.001, urcrnrlon=np.max(humlon)+0.001, urcrnrlat=np.max(humlat)+0.001) gx,gy = map.projtran(glon, glat) try: map.arcgisimage(server='http://server.arcgisonline.com/ArcGIS', service='ESRI_Imagery_World_2D', xpixels=1000, ypixels=None, dpi=300) except: map.arcgisimage(server='http://server.arcgisonline.com/ArcGIS', service='World_Imagery', xpixels=1000, ypixels=None, dpi=300) #finally: # print "error: map could not be created..." ax = plt.Axes(fig, [0., 0., 1., 1.], ) ax.set_axis_off() fig.add_axes(ax) if Sdat_gm.size > 25000000: print("matrix size > 25,000,000 - decimating by factor of 5 for display") map.pcolormesh(gx[::5,::5], gy[::5,::5], Sdat_gm[::5,::5], vmin=np.nanmin(Sdat_gm), vmax=np.nanmax(Sdat_gm)) else: map.pcolormesh(gx, gy, Sdat_gm, vmin=np.nanmin(Sdat_gm), vmax=np.nanmax(Sdat_gm)) custom_save2(sonpath,'class_map_imagery') del fig if os.name=='posix': # true if linux/mac elapsed = (time.time() - start) else: # windows elapsed = (time.clock() - start) print("Processing took "+str(elapsed)+"seconds to analyse") print("Done!")