def map(humfile, sonpath, cs2cs_args, res, mode, nn, numstdevs, use_uncorrected, scalemax): #dogrid = 1, influence = 1, dowrite = 0, ''' Create plots of the spatially referenced sidescan echograms Syntax ---------- [] = PyHum.map(humfile, sonpath, cs2cs_args, res, mode, nn, numstdevs) Parameters ---------- humfile : str path to the .DAT file sonpath : str path where the *.SON files are cs2cs_args : int, *optional* [Default="epsg:26949"] arguments to create coordinates in a projected coordinate system this argument gets given to pyproj to turn wgs84 (lat/lon) coordinates into any projection supported by the proj.4 libraries res : float, *optional* [Default=0] grid resolution of output gridded texture map if res=0, res will be determined automatically from the spatial resolution of 1 pixel mode: int, *optional* [Default=3] gridding mode. 1 = nearest neighbour 2 = inverse weighted nearest neighbour 3 = Gaussian weighted nearest neighbour nn: int, *optional* [Default=64] number of nearest neighbours for gridding (used if mode > 1) numstdevs: int, *optional* [Default = 4] Threshold number of standard deviations in sidescan intensity per grid cell up to which to accept Returns ------- sonpath+'x_y_ss_raw'+str(p)+'.asc' : text file contains the point cloud of easting, northing, and sidescan intensity of the pth chunk sonpath+'GroundOverlay'+str(p)+'.kml': kml file contains gridded (or point cloud) sidescan intensity map for importing into google earth of the pth chunk sonpath+'map'+str(p)+'.png' : image overlay associated with the kml file ''' # prompt user to supply file if no input file given if not humfile: print('An input file is required!!!!!!') Tk().withdraw() # we don't want a full GUI, so keep the root window from appearing humfile = askopenfilename(filetypes=[("DAT files","*.DAT")]) # prompt user to supply directory if no input sonpath is given if not sonpath: print('A *.SON directory is required!!!!!!') Tk().withdraw() # we don't want a full GUI, so keep the root window from appearing sonpath = askdirectory() # print given arguments to screen and convert data type where necessary if humfile: print('Input file is %s' % (humfile)) if sonpath: print('Sonar file path is %s' % (sonpath)) if cs2cs_args: print('cs2cs arguments are %s' % (cs2cs_args)) if res: res = np.asarray(res,float) print('Gridding resolution: %s' % (str(res))) if mode: mode = int(mode) print('Mode for gridding: %s' % (str(mode))) if nn: nn = int(nn) print('Number of nearest neighbours for gridding: %s' % (str(nn))) if numstdevs: numstdevs = int(numstdevs) print('Threshold number of standard deviations in sidescan intensity per grid cell up to which to accept: %s' % (str(numstdevs))) if use_uncorrected: use_uncorrected = int(use_uncorrected) if use_uncorrected==1: print("Radiometrically uncorrected scans will be used") # start timer if os.name=='posix': # true if linux/mac or cygwin on windows start = time.time() else: # windows start = time.clock() #trans = pyproj.Proj(init=cs2cs_args) # if son path name supplied has no separator at end, put one on if sonpath[-1]!=os.sep: sonpath = sonpath + os.sep base = humfile.split('.DAT') # get base of file name for output base = base[0].split(os.sep)[-1] # remove underscores, negatives and spaces from basename base = humutils.strip_base(base) meta = loadmat(os.path.normpath(os.path.join(sonpath,base+'meta.mat'))) esi = np.squeeze(meta['e']) nsi = np.squeeze(meta['n']) theta = np.squeeze(meta['heading'])/(180/np.pi) # load memory mapped scans shape_port = np.squeeze(meta['shape_port']) shape_star = np.squeeze(meta['shape_star']) if use_uncorrected == 1: print("using uncorrected scans") if shape_port!='': port_fp = io.get_mmap_data(sonpath, base, '_data_port_l.dat', 'float32', tuple(shape_port)) if shape_port!='': star_fp = io.get_mmap_data(sonpath, base, '_data_star_l.dat', 'float32', tuple(shape_star)) else: if shape_port!='': if os.path.isfile(os.path.normpath(os.path.join(sonpath,base+'_data_port_lar.dat'))): port_fp = io.get_mmap_data(sonpath, base, '_data_port_lar.dat', 'float32', tuple(shape_port)) else: port_fp = io.get_mmap_data(sonpath, base, '_data_port_la.dat', 'float32', tuple(shape_port)) if shape_star!='': if os.path.isfile(os.path.normpath(os.path.join(sonpath,base+'_data_star_lar.dat'))): star_fp = io.get_mmap_data(sonpath, base, '_data_star_lar.dat', 'float32', tuple(shape_star)) else: star_fp = io.get_mmap_data(sonpath, base, '_data_star_la.dat', 'float32', tuple(shape_star)) # time varying gain tvg = ((8.5*10**-5)+(3/76923)+((8.5*10**-5)/4))*meta['c'] # depth correction dist_tvg = np.squeeze(((np.tan(np.radians(25)))*np.squeeze(meta['dep_m']))-(tvg)) # read in range data R_fp = io.get_mmap_data(sonpath, base, '_data_range.dat', 'float32', tuple(shape_star)) # for debugging/testing # p=0 # e = esi[shape_port[-1]*p:shape_port[-1]*(p+1)] # n = nsi[shape_port[-1]*p:shape_port[-1]*(p+1)] # t = theta[shape_port[-1]*p:shape_port[-1]*(p+1)] # d = dist_tvg[shape_port[-1]*p:shape_port[-1]*(p+1)] # dat_port = port_fp[p] # dat_star = star_fp[p] # data_R = R_fp[p] # dx=np.arcsin(meta['c']/(1000*meta['t']*meta['f'])) # e = esi;# del esi # n = nsi; #del nsi # t = theta;# del theta # d = dist_tvg;# del dist_tvg # dat_port = port_fp;# del port_fp # dat_star = star_fp; #del star_fp # data_R = R_fp; #del R_fp dx = np.arcsin(meta['c']/(1000*meta['t']*meta['f'])) pix_m = meta['pix_m']*1.1 c = meta['c'] if res==0: res=99 print("Number of chunks for mapping: %s" % (len(star_fp))) if len(shape_star)>2: for p in range(len(star_fp)): try: print("progress: " + str(p) + " / " + str(len(star_fp))) res = make_map(esi[shape_port[-1]*p:shape_port[-1]*(p+1)], nsi[shape_port[-1]*p:shape_port[-1]*(p+1)], theta[shape_port[-1]*p:shape_port[-1]*(p+1)], dist_tvg[shape_port[-1]*p:shape_port[-1]*(p+1)], port_fp[p], star_fp[p], R_fp[p], meta['pix_m'], res, cs2cs_args, sonpath, p, mode, nn, numstdevs, meta['c'], np.arcsin(meta['c']/(1000*meta['t']*meta['f'])), use_uncorrected, scalemax) #dogrid, influence, dowrite, print("grid resolution is %s" % (str(res))) except: print("error on chunk "+str(p)) else: res = make_map(esi, nsi, theta, dist_tvg, port_fp, star_fp, R_fp, meta['pix_m'], res, cs2cs_args, sonpath, 0, mode, nn, numstdevs, meta['c'], np.arcsin(meta['c']/(1000*meta['t']*meta['f'])), use_uncorrected, scalemax) #dogrid, influence,dowrite, if os.name=='posix': # true if linux/mac elapsed = (time.time() - start) else: # windows elapsed = (time.clock() - start) print("Processing took "+str(elapsed)+"seconds to analyse") print("Done!") print("===================================================")
def mosaic_texture(humfile, sonpath, cs2cs_args="epsg:26949", res=99, nn=5, weight=1): ''' Create mosaics of the spatially referenced sidescan echograms Syntax ---------- [] = PyHum.mosaic_texture(humfile, sonpath, cs2cs_args, res, nn, weight) Parameters ---------- humfile : str path to the .DAT file sonpath : str path where the *.SON files are cs2cs_args : int, *optional* [Default="epsg:26949"] arguments to create coordinates in a projected coordinate system this argument gets given to pyproj to turn wgs84 (lat/lon) coordinates into any projection supported by the proj.4 libraries res : float, *optional* [Default=0] grid resolution of output gridded texture map if res=99, res will be determined automatically from the spatial resolution of 1 pixel nn: int, *optional* [Default=5] number of nearest neighbours for gridding weight: int, *optional* [Default=1] specifies the type of pixel weighting in the gridding process weight = 1, based on grazing angle and inverse distance weighting weight = 2, based on grazing angle only weight = 3, inverse distance weighting only weight = 4, no weighting Returns ------- sonpath+'GroundOverlay.kml': kml file contains gridded (or point cloud) sidescan intensity map for importing into google earth of the pth chunk sonpath+'map.png' : image overlay associated with the kml file ''' # prompt user to supply file if no input file given if not humfile: print('An input file is required!!!!!!') Tk().withdraw( ) # we don't want a full GUI, so keep the root window from appearing humfile = askopenfilename(filetypes=[("DAT files", "*.DAT")]) # prompt user to supply directory if no input sonpath is given if not sonpath: print('A *.SON directory is required!!!!!!') Tk().withdraw( ) # we don't want a full GUI, so keep the root window from appearing sonpath = askdirectory() # print given arguments to screen and convert data type where necessary if humfile: print('Input file is %s' % (humfile)) if sonpath: print('Sonar file path is %s' % (sonpath)) if cs2cs_args: print('cs2cs arguments are %s' % (cs2cs_args)) if res: res = np.asarray(res, float) print('Gridding resolution: %s' % (str(res))) if nn: nn = int(nn) print('Number of nearest neighbours for gridding: %s' % (str(nn))) if weight: weight = int(weight) print('Weighting for gridding: %s' % (str(weight))) ##nn = 5 #number of nearest neighbours in gridding ##noisefloor=10 # noise threshold in dB W # start timer if os.name == 'posix': # true if linux/mac or cygwin on windows start = time.time() else: # windows start = time.clock() trans = pyproj.Proj(init=cs2cs_args) # if son path name supplied has no separator at end, put one on if sonpath[-1] != os.sep: sonpath = sonpath + os.sep base = humfile.split('.DAT') # get base of file name for output base = base[0].split(os.sep)[-1] # remove underscores, negatives and spaces from basename base = humutils.strip_base(base) meta = loadmat(os.path.normpath(os.path.join(sonpath, base + 'meta.mat'))) esi = np.squeeze(meta['e']) nsi = np.squeeze(meta['n']) theta = np.squeeze(meta['heading']) / (180 / np.pi) # load memory mapped scans shape_port = np.squeeze(meta['shape_port']) if shape_port != '': if os.path.isfile( os.path.normpath( os.path.join(sonpath, base + '_data_port_lar.dat'))): port_fp = io.get_mmap_data(sonpath, base, '_data_port_lar.dat', 'float32', tuple(shape_port)) else: port_fp = io.get_mmap_data(sonpath, base, '_data_port_la.dat', 'float32', tuple(shape_port)) shape_star = np.squeeze(meta['shape_star']) if shape_star != '': if os.path.isfile( os.path.normpath( os.path.join(sonpath, base + '_data_star_lar.dat'))): star_fp = io.get_mmap_data(sonpath, base, '_data_star_lar.dat', 'float32', tuple(shape_star)) else: star_fp = io.get_mmap_data(sonpath, base, '_data_star_la.dat', 'float32', tuple(shape_star)) # time varying gain tvg = ((8.5 * 10**-5) + (3 / 76923) + ((8.5 * 10**-5) / 4)) * meta['c'] # depth correction dist_tvg = np.squeeze(( (np.tan(np.radians(25))) * np.squeeze(meta['dep_m'])) - (tvg)) # read in range data R_fp = io.get_mmap_data(sonpath, base, '_data_range.dat', 'float32', tuple(shape_star)) dx = np.arcsin(meta['c'] / (1000 * meta['t'] * meta['f'])) pix_m = meta['pix_m'] c = meta['c'] if not os.path.isfile(os.path.normpath(os.path.join(sonpath, base + "S.p"))): #if 2 > 1: inputfiles = [] if len(shape_star) > 2: for p in range(len(star_fp)): e = esi[shape_port[-1] * p:shape_port[-1] * (p + 1)] n = nsi[shape_port[-1] * p:shape_port[-1] * (p + 1)] t = theta[shape_port[-1] * p:shape_port[-1] * (p + 1)] d = dist_tvg[shape_port[-1] * p:shape_port[-1] * (p + 1)] dat_port = port_fp[p] dat_star = star_fp[p] data_R = R_fp[p] print("writing chunk %s " % (str(p))) write_points(e, n, t, d, dat_port, dat_star, data_R, pix_m, res, cs2cs_args, sonpath, p, c, dx) inputfiles.append( os.path.normpath( os.path.join(sonpath, 'x_y_class' + str(p) + '.asc'))) else: p = 0 print("writing chunk %s " % (str(p))) write_points(esi, nsi, theta, dist_tvg, port_fp, star_fp, R_fp, meta['pix_m'], res, cs2cs_args, sonpath, 0, c, dx) inputfiles.append( os.path.normpath( os.path.join(sonpath, 'x_y_class' + str(p) + '.asc'))) #trans = pyproj.Proj(init=cs2cs_args) # D, R, h, t print("reading points from %s files" % (str(len(inputfiles)))) X, Y, S, D, R, h, t, i = getxys(inputfiles) print("%s points read from %s files" % (str(len(S)), str(len(inputfiles)))) # remove values where sidescan intensity is zero ind = np.where(np.logical_not(S == 0))[0] X = X[ind] Y = Y[ind] S = S[ind] D = D[ind] R = R[ind] h = h[ind] t = t[ind] i = i[ind] del ind # save to file for temporary storage pickle.dump( S, open(os.path.normpath(os.path.join(sonpath, base + "S.p")), "wb")) del S pickle.dump( D, open(os.path.normpath(os.path.join(sonpath, base + "D.p")), "wb")) del D pickle.dump( t, open(os.path.normpath(os.path.join(sonpath, base + "t.p")), "wb")) del t pickle.dump( i, open(os.path.normpath(os.path.join(sonpath, base + "i.p")), "wb")) del i pickle.dump( X, open(os.path.normpath(os.path.join(sonpath, base + "X.p")), "wb")) del X pickle.dump( Y, open(os.path.normpath(os.path.join(sonpath, base + "Y.p")), "wb")) del Y pickle.dump( R, open(os.path.normpath(os.path.join(sonpath, base + "R.p")), "wb")) pickle.dump( h, open(os.path.normpath(os.path.join(sonpath, base + "h.p")), "wb")) #grazing angle g = np.arctan(R.flatten(), h.flatten()) pickle.dump( g, open(os.path.normpath(os.path.join(sonpath, base + "g.p")), "wb")) del g, R, h print("creating grids ...") if res == 0: res = 99 if res == 99: #### prepare grids R = pickle.load( open(os.path.normpath(os.path.join(sonpath, base + "R.p")), "rb")) ## actual along-track resolution is this: dx times dy = Af tmp = R * dx * (c * 0.007 / 2) del R resg = np.min(tmp[tmp > 0]) del tmp else: resg = res X = pickle.load( open(os.path.normpath(os.path.join(sonpath, base + "X.p")), "rb")) Y = pickle.load( open(os.path.normpath(os.path.join(sonpath, base + "Y.p")), "rb")) humlon, humlat = trans(X, Y, inverse=True) grid_x, grid_y = np.meshgrid(np.arange(np.min(X), np.max(X), resg), np.arange(np.min(Y), np.max(Y), resg)) shape = np.shape(grid_x) tree = KDTree(zip(X.flatten(), Y.flatten())) del X, Y print("mosaicking ...") #k nearest neighbour try: dist, inds = tree.query(zip(grid_x.flatten(), grid_y.flatten()), k=nn, n_jobs=-1) except: #print ".... update your scipy installation to use faster kd-tree" dist, inds = tree.query(zip(grid_x.flatten(), grid_y.flatten()), k=nn) #del grid_x, grid_y if weight == 1: g = pickle.load( open(os.path.normpath(os.path.join(sonpath, base + "g.p")), "rb")) w = g[inds] + 1.0 / dist**2 del g elif weight == 2: g = pickle.load( open(os.path.normpath(os.path.join(sonpath, base + "g.p")), "rb")) w = g[inds] del g elif weight == 3: w = 1.0 / dist**2 elif weight == 4: w = 1.0 #g = pickle.load( open( os.path.normpath(os.path.join(sonpath,base+"g.p")), "rb" ) ) #w = g[inds] + 1.0 / dist**2 #del g if weight < 4: w[np.isinf(w)] = 1 w[np.isnan(w)] = 1 w[w > 10000] = 10000 w[w <= 0] = 1 # load in sidescan intensity S = pickle.load( open(os.path.normpath(os.path.join(sonpath, base + "S.p")), "rb")) # filter out noise pixels S[S < noisefloor] = np.nan if nn == 1: Sdat_g = (w * S.flatten()[inds]).reshape(shape) del w dist = dist.reshape(shape) else: if weight < 4: Sdat_g = (np.nansum(w * S.flatten()[inds], axis=1) / np.nansum(w, axis=1)).reshape(shape) else: Sdat_g = (np.nansum(S.flatten()[inds], axis=1)).reshape(shape) del w dist = np.nanmean(dist, axis=1).reshape(shape) del S Sdat_g[dist > 1] = np.nan Sdat_g[Sdat_g < noisefloor] = np.nan dat = Sdat_g.copy() dat[dist > 1] = 0 dat2 = replace_nans.RN(dat.astype('float64'), 1000, 0.01, 2, 'localmean').getdata() dat2[dat == 0] = np.nan del dat dat2[dat2 < noisefloor] = np.nan Sdat_g = dat2.copy() del dat2 Sdat_g[Sdat_g == 0] = np.nan Sdat_g[np.isinf(Sdat_g)] = np.nan Sdat_gm = np.ma.masked_invalid(Sdat_g) del Sdat_g glon, glat = trans(grid_x, grid_y, inverse=True) del grid_x, grid_y # ========================================================= print("creating kmz file ...") ## new way to create kml file pixels = 1024 * 10 fig, ax = humutils.gearth_fig(llcrnrlon=glon.min(), llcrnrlat=glat.min(), urcrnrlon=glon.max(), urcrnrlat=glat.max(), pixels=pixels) cs = ax.pcolormesh(glon, glat, Sdat_gm) ax.set_axis_off() fig.savefig(os.path.normpath(os.path.join(sonpath, 'class_overlay1.png')), transparent=True, format='png') fig = plt.figure(figsize=(1.0, 4.0), facecolor=None, frameon=False) ax = fig.add_axes([0.0, 0.05, 0.2, 0.9]) cb = fig.colorbar(cs, cax=ax) cb.set_label('Texture lengthscale [m]', rotation=-90, color='k', labelpad=20) fig.savefig(os.path.normpath(os.path.join(sonpath, 'class_legend.png')), transparent=False, format='png') humutils.make_kml( llcrnrlon=glon.min(), llcrnrlat=glat.min(), urcrnrlon=glon.max(), urcrnrlat=glat.max(), figs=[os.path.normpath(os.path.join(sonpath, 'class_overlay1.png'))], colorbar=os.path.normpath(os.path.join(sonpath, 'class_legend.png')), kmzfile=os.path.normpath( os.path.join(sonpath, 'class_GroundOverlay.kmz')), name='Sidescan Intensity') # ========================================================= print("drawing and printing map ...") fig = plt.figure(frameon=False) map = Basemap( projection='merc', epsg=cs2cs_args.split(':')[1], resolution='i', #h #f llcrnrlon=np.min(humlon) - 0.001, llcrnrlat=np.min(humlat) - 0.001, urcrnrlon=np.max(humlon) + 0.001, urcrnrlat=np.max(humlat) + 0.001) gx, gy = map.projtran(glon, glat) try: map.arcgisimage(server='http://server.arcgisonline.com/ArcGIS', service='ESRI_Imagery_World_2D', xpixels=1000, ypixels=None, dpi=300) except: map.arcgisimage(server='http://server.arcgisonline.com/ArcGIS', service='World_Imagery', xpixels=1000, ypixels=None, dpi=300) #finally: # print "error: map could not be created..." ax = plt.Axes( fig, [0., 0., 1., 1.], ) ax.set_axis_off() fig.add_axes(ax) if Sdat_gm.size > 25000000: print( "matrix size > 25,000,000 - decimating by factor of 5 for display") map.pcolormesh(gx[::5, ::5], gy[::5, ::5], Sdat_gm[::5, ::5], vmin=np.nanmin(Sdat_gm), vmax=np.nanmax(Sdat_gm)) else: map.pcolormesh(gx, gy, Sdat_gm, vmin=np.nanmin(Sdat_gm), vmax=np.nanmax(Sdat_gm)) custom_save2(sonpath, 'class_map_imagery') del fig if os.name == 'posix': # true if linux/mac elapsed = (time.time() - start) else: # windows elapsed = (time.clock() - start) print("Processing took " + str(elapsed) + "seconds to analyse") print("Done!")
def texture(humfile, sonpath, win=100, shift=10, doplot=1, density=50, numclasses=4, maxscale=20, notes=4): ''' Create a texture lengthscale map using the algorithm detailed by Buscombe et al. (2015) This textural lengthscale is not a direct measure of grain size. Rather, it is a statistical representation that integrates over many attributes of bed texture, of which grain size is the most important. The technique is a physically based means to identify regions of texture within a sidescan echogram, and could provide a basis for objective, automated riverbed sediment classification. Syntax ---------- [] = PyHum.texture(humfile, sonpath, win, shift, doplot, density, numclasses, maxscale, notes) Parameters ---------- humfile : str path to the .DAT file sonpath : str path where the *.SON files are win : int, *optional* [Default=100] pixel in pixels of the moving window shift : int, *optional* [Default=10] shift in pixels for moving window operation doplot : int, *optional* [Default=1] if 1, make plots, otherwise do not make plots density : int, *optional* [Default=win/2] echogram will be sampled every 'density' pixels numclasses : int, *optional* [Default=4] number of 'k means' that the texture lengthscale will be segmented into maxscale : int, *optional* [Default=20] Max scale as inverse fraction of data length for wavelet analysis notes : int, *optional* [Default=100] notes per octave for wavelet analysis Returns ------- sonpath+base+'_data_class.dat': memory-mapped file contains the texture lengthscale map sonpath+base+'_data_kclass.dat': memory-mapped file contains the k-means segmented texture lengthscale map References ---------- .. [1] Buscombe, D., Grams, P.E., and Smith, S.M.C., 2015, Automated riverbed sediment classification using low-cost sidescan sonar. Journal of Hydraulic Engineering 10.1061/(ASCE)HY.1943-7900.0001079, 06015019. ''' # prompt user to supply file if no input file given if not humfile: print 'An input file is required!!!!!!' Tk().withdraw( ) # we don't want a full GUI, so keep the root window from appearing humfile = askopenfilename(filetypes=[("DAT files", "*.DAT")]) # prompt user to supply directory if no input sonpath is given if not sonpath: print 'A *.SON directory is required!!!!!!' Tk().withdraw( ) # we don't want a full GUI, so keep the root window from appearing sonpath = askdirectory() # print given arguments to screen and convert data type where necessary if humfile: print 'Input file is %s' % (humfile) if sonpath: print 'Sonar file path is %s' % (sonpath) if win: win = np.asarray(win, int) print 'Window is %s square pixels' % (str(win)) if shift: shift = np.asarray(shift, int) print 'Min shift is %s pixels' % (str(shift)) if density: density = np.asarray(density, int) print 'Image will be sampled every %s pixels' % (str(density)) if numclasses: numclasses = np.asarray(numclasses, int) print 'Number of sediment classes: %s' % (str(numclasses)) if maxscale: maxscale = np.asarray(maxscale, int) print 'Max scale as inverse fraction of data length: %s' % ( str(maxscale)) if notes: notes = np.asarray(notes, int) print 'Notes per octave: %s' % (str(notes)) if doplot: doplot = int(doplot) if doplot == 0: print "Plots will not be made" print '[Default] Number of processors is %s' % (str(cpu_count())) ######################################################## ######################################################## # start timer if os.name == 'posix': # true if linux/mac or cygwin on windows start = time.time() else: # windows start = time.clock() # if son path name supplied has no separator at end, put one on if sonpath[-1] != os.sep: sonpath = sonpath + os.sep base = humfile.split('.DAT') # get base of file name for output base = base[0].split(os.sep)[-1] # remove underscores, negatives and spaces from basename base = humutils.strip_base(base) meta = loadmat(os.path.normpath(os.path.join(sonpath, base + 'meta.mat'))) ft = 1 / loadmat(sonpath + base + 'meta.mat')['pix_m'] #pix_m = np.squeeze(meta['pix_m']) #dep_m = np.squeeze(meta['dep_m']) dist_m = np.squeeze(meta['dist_m']) ### port print "processing port side ..." # load memory mapped scan ... port shape_port = np.squeeze(meta['shape_port']) if shape_port != '': if os.path.isfile( os.path.normpath( os.path.join(sonpath, base + '_data_port_lar.dat'))): port_fp = io.get_mmap_data(sonpath, base, '_data_port_lar.dat', 'float32', tuple(shape_port)) else: port_fp = io.get_mmap_data(sonpath, base, '_data_port_la.dat', 'float32', tuple(shape_port)) port_fp2 = io.get_mmap_data(sonpath, base, '_data_port_l.dat', 'float32', tuple(shape_port)) ### star print "processing starboard side ..." # load memory mapped scan ... port shape_star = np.squeeze(loadmat(sonpath + base + 'meta.mat')['shape_star']) if shape_star != '': if os.path.isfile( os.path.normpath( os.path.join(sonpath, base + '_data_star_lar.dat'))): star_fp = io.get_mmap_data(sonpath, base, '_data_star_lar.dat', 'float32', tuple(shape_star)) else: star_fp = io.get_mmap_data(sonpath, base, '_data_star_la.dat', 'float32', tuple(shape_star)) star_fp2 = io.get_mmap_data(sonpath, base, '_data_star_l.dat', 'float32', tuple(shape_star)) if len(shape_star) > 2: shape = shape_port.copy() shape[1] = shape_port[1] + shape_star[1] else: shape = [] shape.append(1) shape.append(shape_port[0]) shape.append(shape_port[1]) shape[1] = shape_port[0] + shape_star[0] # create memory mapped file for Sp with open( os.path.normpath(os.path.join(sonpath, base + '_data_class.dat')), 'w+') as ff: fp = np.memmap(ff, dtype='float32', mode='w+', shape=tuple(shape)) if len(shape_star) > 2: #SRT = [] for p in xrange(len(port_fp)): Z, ind = humutils.sliding_window_sliced( np.vstack((np.flipud(port_fp[p]), star_fp[p])), density, (win, win), (shift, shift)) Snn = get_srt(Z, ind, maxscale, notes, win) #, density) # replace nans using infilling algorithm rn = replace_nans.RN(Snn.astype('float64'), 1000, 0.01, 2, 'localmean') Snn = rn.getdata() del rn Ny, Nx = np.shape(np.vstack((np.flipud(port_fp[p]), star_fp[p]))) Snn = median_filter(Snn, (int(Nx / 100), int(Ny / 100))) Sp = humutils.im_resize(Snn, Nx, Ny) del Snn Sp[np.isnan(np.vstack( (np.flipud(port_fp[p]), star_fp[p])))] = np.nan Sp[np.isnan(np.vstack( (np.flipud(port_fp2[p]), star_fp2[p])))] = np.nan #extent = shape_port[1] #Zdist = dist_m[shape_port[-1]*p:shape_port[-1]*(p+1)] #yvec = np.linspace(pix_m,extent*pix_m,extent) #d = dep_m[shape_port[-1]*p:shape_port[-1]*(p+1)] R_fp = io.get_mmap_data(sonpath, base, '_data_range.dat', 'float32', tuple(shape_star)) R = np.vstack((np.flipud(R_fp[0]), R_fp[0])) R = R / np.max(R) #R[R>0.8] = np.nan rn = replace_nans.RN(R.astype('float64'), 1000, 0.01, 2, 'localmean') R = rn.getdata() del rn Sp = (Sp**2) * np.cos(R) / shift**2 fp[p] = Sp.astype('float32') del Sp del fp # flush data to file class_fp = io.get_mmap_data(sonpath, base, '_data_class.dat', 'float32', tuple(shape)) else: Z, ind = humutils.sliding_window_sliced( np.vstack((np.flipud(port_fp), star_fp)), density, (win, win), (shift, shift)) Snn = get_srt(Z, ind, maxscale, notes, win) #, density) # replace nans using infilling algorithm rn = replace_nans.RN(Snn.astype('float64'), 1000, 0.01, 2, 'localmean') Snn = rn.getdata() del rn Ny, Nx = np.shape(np.vstack((np.flipud(port_fp), star_fp))) Snn = median_filter(Snn, (int(Nx / 100), int(Ny / 100))) Sp = humutils.im_resize(Snn, Nx, Ny) del Snn Sp[np.isnan(np.vstack((np.flipud(port_fp), star_fp)))] = np.nan Sp[np.isnan(np.vstack((np.flipud(port_fp2), star_fp2)))] = np.nan #extent = shape_port[0] #Zdist = dist_m #yvec = np.linspace(pix_m,extent*pix_m,extent) #d = dep_m R_fp = io.get_mmap_data(sonpath, base, '_data_range.dat', 'float32', tuple(shape_star)) R = np.vstack((np.flipud(R_fp), R_fp)) R = R / np.max(R) #R[R>0.8] = np.nan rn = replace_nans.RN(R.astype('float64'), 1000, 0.01, 2, 'localmean') R = rn.getdata() del rn Sp = (Sp**2) * np.cos(R) / shift**2 with open( os.path.normpath( os.path.join(sonpath, base + '_data_class.dat')), 'w+') as ff: np.save(ff, np.squeeze(Sp).astype('float32')) with open( os.path.normpath( os.path.join(sonpath, base + '_data_class.dat')), 'r') as ff: class_fp = np.load(ff) del Sp dist_m = np.squeeze(loadmat(sonpath + base + 'meta.mat')['dist_m']) ######################################################## if doplot == 1: if len(shape_star) > 2: for p in xrange(len(star_fp)): plot_class(dist_m, shape_port, port_fp[p], star_fp[p], class_fp[p], ft, humfile, sonpath, base, p) else: plot_class(dist_m, shape_port, port_fp, star_fp, class_fp, ft, humfile, sonpath, base, 0) if len(shape_star) > 2: for p in xrange(len(star_fp)): plot_contours(dist_m, shape_port, class_fp[p], ft, humfile, sonpath, base, numclasses, p) else: plot_contours(dist_m, shape_port, class_fp, ft, humfile, sonpath, base, numclasses, 0) ####################################################### # k-means if len(shape_star) > 2: with open( os.path.normpath( os.path.join(sonpath, base + '_data_kclass.dat')), 'w+') as ff: fp = np.memmap(ff, dtype='float32', mode='w+', shape=tuple(shape)) for p in xrange(len(port_fp)): wc = get_kclass(class_fp[p].copy(), numclasses) fp[p] = wc.astype('float32') del wc del fp kclass_fp = io.get_mmap_data(sonpath, base, '_data_kclass.dat', 'float32', tuple(shape)) else: wc = get_kclass(class_fp.copy(), numclasses) with open( os.path.normpath( os.path.join(sonpath, base + '_data_kclass.dat')), 'w+') as ff: np.save(ff, np.squeeze(wc).astype('float32')) del wc with open( os.path.normpath( os.path.join(sonpath, base + '_data_kclass.dat')), 'r') as ff: kclass_fp = np.load(ff) ######################################################## if doplot == 1: if len(shape_star) > 2: for p in xrange(len(star_fp)): plot_kmeans(dist_m, shape_port, port_fp[p], star_fp[p], kclass_fp[p], ft, humfile, sonpath, base, p) else: plot_kmeans(dist_m, shape_port, port_fp, star_fp, kclass_fp, ft, humfile, sonpath, base, 0) if os.name == 'posix': # true if linux/mac elapsed = (time.time() - start) else: # windows elapsed = (time.clock() - start) print "Processing took ", elapsed, "seconds to analyse" print "Done!"
def mosaic_texture(humfile, sonpath, cs2cs_args = "epsg:26949", res = 99, nn = 5, weight = 1): ''' Create mosaics of the spatially referenced sidescan echograms Syntax ---------- [] = PyHum.mosaic_texture(humfile, sonpath, cs2cs_args, res, nn, weight) Parameters ---------- humfile : str path to the .DAT file sonpath : str path where the *.SON files are cs2cs_args : int, *optional* [Default="epsg:26949"] arguments to create coordinates in a projected coordinate system this argument gets given to pyproj to turn wgs84 (lat/lon) coordinates into any projection supported by the proj.4 libraries res : float, *optional* [Default=0] grid resolution of output gridded texture map if res=99, res will be determined automatically from the spatial resolution of 1 pixel nn: int, *optional* [Default=5] number of nearest neighbours for gridding weight: int, *optional* [Default=1] specifies the type of pixel weighting in the gridding process weight = 1, based on grazing angle and inverse distance weighting weight = 2, based on grazing angle only weight = 3, inverse distance weighting only weight = 4, no weighting Returns ------- sonpath+'GroundOverlay.kml': kml file contains gridded (or point cloud) sidescan intensity map for importing into google earth of the pth chunk sonpath+'map.png' : image overlay associated with the kml file ''' # prompt user to supply file if no input file given if not humfile: print('An input file is required!!!!!!') Tk().withdraw() # we don't want a full GUI, so keep the root window from appearing humfile = askopenfilename(filetypes=[("DAT files","*.DAT")]) # prompt user to supply directory if no input sonpath is given if not sonpath: print('A *.SON directory is required!!!!!!') Tk().withdraw() # we don't want a full GUI, so keep the root window from appearing sonpath = askdirectory() # print given arguments to screen and convert data type where necessary if humfile: print('Input file is %s' % (humfile)) if sonpath: print('Sonar file path is %s' % (sonpath)) if cs2cs_args: print('cs2cs arguments are %s' % (cs2cs_args)) if res: res = np.asarray(res,float) print('Gridding resolution: %s' % (str(res))) if nn: nn = int(nn) print('Number of nearest neighbours for gridding: %s' % (str(nn))) if weight: weight = int(weight) print('Weighting for gridding: %s' % (str(weight))) ##nn = 5 #number of nearest neighbours in gridding ##noisefloor=10 # noise threshold in dB W # start timer if os.name=='posix': # true if linux/mac or cygwin on windows start = time.time() else: # windows start = time.clock() trans = pyproj.Proj(init=cs2cs_args) # if son path name supplied has no separator at end, put one on if sonpath[-1]!=os.sep: sonpath = sonpath + os.sep base = humfile.split('.DAT') # get base of file name for output base = base[0].split(os.sep)[-1] # remove underscores, negatives and spaces from basename base = humutils.strip_base(base) meta = loadmat(os.path.normpath(os.path.join(sonpath,base+'meta.mat'))) esi = np.squeeze(meta['e']) nsi = np.squeeze(meta['n']) theta = np.squeeze(meta['heading'])/(180/np.pi) # load memory mapped scans shape_port = np.squeeze(meta['shape_port']) if shape_port!='': if os.path.isfile(os.path.normpath(os.path.join(sonpath,base+'_data_port_lar.dat'))): port_fp = io.get_mmap_data(sonpath, base, '_data_port_lar.dat', 'float32', tuple(shape_port)) else: port_fp = io.get_mmap_data(sonpath, base, '_data_port_la.dat', 'float32', tuple(shape_port)) shape_star = np.squeeze(meta['shape_star']) if shape_star!='': if os.path.isfile(os.path.normpath(os.path.join(sonpath,base+'_data_star_lar.dat'))): star_fp = io.get_mmap_data(sonpath, base, '_data_star_lar.dat', 'float32', tuple(shape_star)) else: star_fp = io.get_mmap_data(sonpath, base, '_data_star_la.dat', 'float32', tuple(shape_star)) # time varying gain tvg = ((8.5*10**-5)+(3/76923)+((8.5*10**-5)/4))*meta['c'] # depth correction dist_tvg = np.squeeze(((np.tan(np.radians(25)))*np.squeeze(meta['dep_m']))-(tvg)) # read in range data R_fp = io.get_mmap_data(sonpath, base, '_data_range.dat', 'float32', tuple(shape_star)) dx = np.arcsin(meta['c']/(1000*meta['t']*meta['f'])) pix_m = meta['pix_m'] c = meta['c'] if not os.path.isfile( os.path.normpath(os.path.join(sonpath,base+"S.p")) ): #if 2 > 1: inputfiles = [] if len(shape_star)>2: for p in range(len(star_fp)): e = esi[shape_port[-1]*p:shape_port[-1]*(p+1)] n = nsi[shape_port[-1]*p:shape_port[-1]*(p+1)] t = theta[shape_port[-1]*p:shape_port[-1]*(p+1)] d = dist_tvg[shape_port[-1]*p:shape_port[-1]*(p+1)] dat_port = port_fp[p] dat_star = star_fp[p] data_R = R_fp[p] print("writing chunk %s " % (str(p))) write_points(e, n, t, d, dat_port, dat_star, data_R, pix_m, res, cs2cs_args, sonpath, p, c, dx) inputfiles.append(os.path.normpath(os.path.join(sonpath,'x_y_class'+str(p)+'.asc'))) else: p=0 print("writing chunk %s " % (str(p))) write_points(esi, nsi, theta, dist_tvg, port_fp, star_fp, R_fp, meta['pix_m'], res, cs2cs_args, sonpath, 0, c, dx) inputfiles.append(os.path.normpath(os.path.join(sonpath,'x_y_class'+str(p)+'.asc'))) #trans = pyproj.Proj(init=cs2cs_args) # D, R, h, t print("reading points from %s files" % (str(len(inputfiles)))) X,Y,S,D,R,h,t,i = getxys(inputfiles) print("%s points read from %s files" % (str(len(S)), str(len(inputfiles)))) # remove values where sidescan intensity is zero ind = np.where(np.logical_not(S==0))[0] X = X[ind]; Y = Y[ind] S = S[ind]; D = D[ind] R = R[ind]; h = h[ind] t = t[ind]; i = i[ind] del ind # save to file for temporary storage pickle.dump( S, open( os.path.normpath(os.path.join(sonpath,base+"S.p")), "wb" ) ); del S pickle.dump( D, open( os.path.normpath(os.path.join(sonpath,base+"D.p")), "wb" ) ); del D pickle.dump( t, open( os.path.normpath(os.path.join(sonpath,base+"t.p")), "wb" ) ); del t pickle.dump( i, open( os.path.normpath(os.path.join(sonpath,base+"i.p")), "wb" ) ); del i pickle.dump( X, open( os.path.normpath(os.path.join(sonpath,base+"X.p")), "wb" ) ); del X pickle.dump( Y, open( os.path.normpath(os.path.join(sonpath,base+"Y.p")), "wb" ) ); del Y pickle.dump( R, open( os.path.normpath(os.path.join(sonpath,base+"R.p")), "wb" ) ); pickle.dump( h, open( os.path.normpath(os.path.join(sonpath,base+"h.p")), "wb" ) ); #grazing angle g = np.arctan(R.flatten(),h.flatten()) pickle.dump( g, open( os.path.normpath(os.path.join(sonpath,base+"g.p")), "wb" ) ); del g, R, h print("creating grids ...") if res==0: res=99 if res==99: #### prepare grids R = pickle.load( open( os.path.normpath(os.path.join(sonpath,base+"R.p")), "rb" ) ) ## actual along-track resolution is this: dx times dy = Af tmp = R * dx * (c*0.007 / 2) del R resg = np.min(tmp[tmp>0]) del tmp else: resg = res X = pickle.load( open( os.path.normpath(os.path.join(sonpath,base+"X.p")), "rb" ) ) Y = pickle.load( open( os.path.normpath(os.path.join(sonpath,base+"Y.p")), "rb" ) ) humlon, humlat = trans(X, Y, inverse=True) grid_x, grid_y = np.meshgrid( np.arange(np.min(X), np.max(X), resg), np.arange(np.min(Y), np.max(Y), resg) ) shape = np.shape(grid_x) tree = KDTree(zip(X.flatten(), Y.flatten())) del X, Y print("mosaicking ...") #k nearest neighbour try: dist, inds = tree.query(zip(grid_x.flatten(), grid_y.flatten()), k = nn, n_jobs=-1) except: #print ".... update your scipy installation to use faster kd-tree" dist, inds = tree.query(zip(grid_x.flatten(), grid_y.flatten()), k = nn) #del grid_x, grid_y if weight==1: g = pickle.load( open( os.path.normpath(os.path.join(sonpath,base+"g.p")), "rb" ) ) w = g[inds] + 1.0 / dist**2 del g elif weight==2: g = pickle.load( open( os.path.normpath(os.path.join(sonpath,base+"g.p")), "rb" ) ) w = g[inds] del g elif weight==3: w = 1.0 / dist**2 elif weight==4: w = 1.0 #g = pickle.load( open( os.path.normpath(os.path.join(sonpath,base+"g.p")), "rb" ) ) #w = g[inds] + 1.0 / dist**2 #del g if weight < 4: w[np.isinf(w)]=1 w[np.isnan(w)]=1 w[w>10000]=10000 w[w<=0]=1 # load in sidescan intensity S = pickle.load( open( os.path.normpath(os.path.join(sonpath,base+"S.p")), "rb" ) ) # filter out noise pixels S[S<noisefloor] = np.nan if nn==1: Sdat_g = (w * S.flatten()[inds]).reshape(shape) del w dist = dist.reshape(shape) else: if weight < 4: Sdat_g = (np.nansum(w * S.flatten()[inds], axis=1) / np.nansum(w, axis=1)).reshape(shape) else: Sdat_g = (np.nansum(S.flatten()[inds], axis=1)).reshape(shape) del w dist = np.nanmean(dist,axis=1).reshape(shape) del S Sdat_g[dist>1] = np.nan Sdat_g[Sdat_g<noisefloor] = np.nan dat = Sdat_g.copy() dat[dist>1] = 0 dat2 = replace_nans.RN(dat.astype('float64'),1000,0.01,2,'localmean').getdata() dat2[dat==0] = np.nan del dat dat2[dat2<noisefloor] = np.nan Sdat_g = dat2.copy() del dat2 Sdat_g[Sdat_g==0] = np.nan Sdat_g[np.isinf(Sdat_g)] = np.nan Sdat_gm = np.ma.masked_invalid(Sdat_g) del Sdat_g glon, glat = trans(grid_x, grid_y, inverse=True) del grid_x, grid_y # ========================================================= print("creating kmz file ...") ## new way to create kml file pixels = 1024 * 10 fig, ax = humutils.gearth_fig(llcrnrlon=glon.min(), llcrnrlat=glat.min(), urcrnrlon=glon.max(), urcrnrlat=glat.max(), pixels=pixels) cs = ax.pcolormesh(glon, glat, Sdat_gm) ax.set_axis_off() fig.savefig(os.path.normpath(os.path.join(sonpath,'class_overlay1.png')), transparent=True, format='png') fig = plt.figure(figsize=(1.0, 4.0), facecolor=None, frameon=False) ax = fig.add_axes([0.0, 0.05, 0.2, 0.9]) cb = fig.colorbar(cs, cax=ax) cb.set_label('Texture lengthscale [m]', rotation=-90, color='k', labelpad=20) fig.savefig(os.path.normpath(os.path.join(sonpath,'class_legend.png')), transparent=False, format='png') humutils.make_kml(llcrnrlon=glon.min(), llcrnrlat=glat.min(), urcrnrlon=glon.max(), urcrnrlat=glat.max(), figs=[os.path.normpath(os.path.join(sonpath,'class_overlay1.png'))], colorbar=os.path.normpath(os.path.join(sonpath,'class_legend.png')), kmzfile=os.path.normpath(os.path.join(sonpath,'class_GroundOverlay.kmz')), name='Sidescan Intensity') # ========================================================= print("drawing and printing map ...") fig = plt.figure(frameon=False) map = Basemap(projection='merc', epsg=cs2cs_args.split(':')[1], resolution = 'i', #h #f llcrnrlon=np.min(humlon)-0.001, llcrnrlat=np.min(humlat)-0.001, urcrnrlon=np.max(humlon)+0.001, urcrnrlat=np.max(humlat)+0.001) gx,gy = map.projtran(glon, glat) try: map.arcgisimage(server='http://server.arcgisonline.com/ArcGIS', service='ESRI_Imagery_World_2D', xpixels=1000, ypixels=None, dpi=300) except: map.arcgisimage(server='http://server.arcgisonline.com/ArcGIS', service='World_Imagery', xpixels=1000, ypixels=None, dpi=300) #finally: # print "error: map could not be created..." ax = plt.Axes(fig, [0., 0., 1., 1.], ) ax.set_axis_off() fig.add_axes(ax) if Sdat_gm.size > 25000000: print("matrix size > 25,000,000 - decimating by factor of 5 for display") map.pcolormesh(gx[::5,::5], gy[::5,::5], Sdat_gm[::5,::5], vmin=np.nanmin(Sdat_gm), vmax=np.nanmax(Sdat_gm)) else: map.pcolormesh(gx, gy, Sdat_gm, vmin=np.nanmin(Sdat_gm), vmax=np.nanmax(Sdat_gm)) custom_save2(sonpath,'class_map_imagery') del fig if os.name=='posix': # true if linux/mac elapsed = (time.time() - start) else: # windows elapsed = (time.clock() - start) print("Processing took "+str(elapsed)+"seconds to analyse") print("Done!")
def sliding_window(a,ws,ss = None,flatten = True): ''' Return a sliding window over a in any number of dimensions ''' if None is ss: # ss was not provided. the windows will not overlap in any direction. ss = ws ws = norm_shape(ws) ss = norm_shape(ss) # convert ws, ss, and a.shape to numpy arrays ws = np.array(ws) ss = np.array(ss) #import PyHum.io as io shape_tmp = io.set_mmap_data('', '', 'tmp.dat', 'float32', a) del a a = io.get_mmap_data('', '', 'tmp.dat', 'float32', shape_tmp) shap = np.array(a.shape) try: os.remove('tmp.dat') except: pass # ensure that ws, ss, and a.shape all have the same number of dimensions ls = [len(shap),len(ws),len(ss)] if 1 != len(set(ls)): raise ValueError(\ 'a.shape, ws and ss must all have the same length. They were %s' % str(ls)) # ensure that ws is smaller than a in every dimension if np.any(ws > shap): raise ValueError(\ 'ws cannot be larger than a in any dimension.\ a.shape was %s and ws was %s' % (str(a.shape),str(ws))) # how many slices will there be in each dimension? newshape = norm_shape(((shap - ws) // ss) + 1) # the shape of the strided array will be the number of slices in each dimension # plus the shape of the window (tuple addition) newshape += norm_shape(ws) # the strides tuple will be the array's strides multiplied by step size, plus try: # the array's strides (tuple addition) newstrides = norm_shape(np.array(a.strides) * ss) + a.strides a = ast(a,shape = newshape,strides = newstrides) if not flatten: return a # Collapse strided so that it has one more dimension than the window. I.e., # the new array is a flat list of slices. meat = len(ws) if ws.shape else 0 firstdim = (int(np.product(newshape[:-meat])),) if ws.shape else () dim = firstdim + (newshape[-meat:]) # remove any dimensions with size 1 dim = filter(lambda i : i != 1,dim) return a.reshape(dim), newshape except: from itertools import product print "memory error, windowing using slower method" # For each dimension, create a list of all valid slices slices = [[] for i in range(len(ws))] for i in xrange(len(ws)): nslices = ((shap[i] - ws[i]) // ss[i]) + 1 for j in xrange(0,nslices): start = j * ss[i] stop = start + ws[i] slices[i].append(slice(start,stop)) # Get an iterator over all valid n-dimensional slices of the input allslices = product(*slices) # Allocate memory to hold all valid n-dimensional slices nslices = np.product([len(s) for s in slices]) #out = np.ndarray((nslices,) + tuple(ws),dtype = a.dtype) out=[] for i,s in enumerate(allslices): #out[i] = a[s] out.append(a[s]) del a import dask.bag as db tmp = db.from_sequence(out, npartitions=1000) del out return tmp.compute(), newshape
def read(humfile, sonpath, cs2cs_args="epsg:26949", c=1450.0, draft=0.3, doplot=1, t=0.108, bedpick=1, flip_lr=0, model=998, calc_bearing = 0, filt_bearing = 0, chunk='d100'): #cog = 1, ''' Read a .DAT and associated set of .SON files recorded by a Humminbird(R) instrument. Parse the data into a set of memory mapped files that will subsequently be used by the other functions of the PyHum module. Export time-series data and metadata in other formats. Create a kml file for visualising boat track Syntax ---------- [] = PyHum.read(humfile, sonpath, cs2cs_args, c, draft, doplot, t, bedpick, flip_lr, chunksize, model, calc_bearing, filt_bearing, chunk) Parameters ------------ humfile : str path to the .DAT file sonpath : str path where the *.SON files are cs2cs_args : int, *optional* [Default="epsg:26949"] arguments to create coordinates in a projected coordinate system this argument gets given to pyproj to turn wgs84 (lat/lon) coordinates into any projection supported by the proj.4 libraries c : float, *optional* [Default=1450.0] speed of sound in water (m/s). Defaults to a value of freshwater draft : float, *optional* [Default=0.3] draft from water surface to transducer face (m) doplot : float, *optional* [Default=1] if 1, plots will be made t : float, *optional* [Default=0.108] length of transducer array (m). Default value is that of the 998 series Humminbird(R) bedpick : int, *optional* [Default=1] if 1, bedpicking with be carried out automatically if 0, user will be prompted to pick the bed location on screen flip_lr : int, *optional* [Default=0] if 1, port and starboard scans will be flipped (for situations where the transducer is flipped 180 degrees) model: int, *optional* [Default=998] A 3 or 4 number code indicating the model number Examples: 998, 997, 1198, 1199 calc_bearing : float, *optional* [Default=0] if 1, bearing will be calculated from coordinates filt_bearing : float, *optional* [Default=0] if 1, bearing will be filtered chunk : str, *optional* [Default='d100' (distance, 100 m)] letter, followed by a number. There are the following letter options: 'd' - parse chunks based on distance, then number which is distance in m 'p' - parse chunks based on number of pings, then number which is number of pings 'h' - parse chunks based on change in heading, then number which is the change in heading in degrees '1' - process just 1 chunk Returns --------- sonpath+base+'_data_port.dat': memory-mapped file contains the raw echogram from the port side sidescan sonar (where present) sonpath+base+'_data_port.dat': memory-mapped file contains the raw echogram from the starboard side sidescan sonar (where present) sonpath+base+'_data_dwnhi.dat': memory-mapped file contains the raw echogram from the high-frequency echosounder (where present) sonpath+base+'_data_dwnlow.dat': memory-mapped file contains the raw echogram from the low-frequency echosounder (where present) sonpath+base+"trackline.kml": google-earth kml file contains the trackline of the vessel during data acquisition sonpath+base+'rawdat.csv': comma separated value file contains time-series data. columns corresponding to longitude latitude easting (m) northing (m) depth to bed (m) alongtrack cumulative distance (m) vessel heading (deg.) sonpath+base+'meta.mat': .mat file matlab format file containing a dictionary object holding metadata information. Fields are: e : ndarray, easting (m) n : ndarray, northing (m) es : ndarray, low-pass filtered easting (m) ns : ndarray, low-pass filtered northing (m) lat : ndarray, latitude lon : ndarray, longitude shape_port : tuple, shape of port scans in memory mapped file shape_star : tuple, shape of starboard scans in memory mapped file shape_hi : tuple, shape of high-freq. scans in memory mapped file shape_low : tuple, shape of low-freq. scans in memory mapped file dep_m : ndarray, depth to bed (m) dist_m : ndarray, distance along track (m) heading : ndarray, heading of vessel (deg. N) pix_m: float, size of 1 pixel in across-track dimension (m) bed : ndarray, depth to bed (m) c : float, speed of sound in water (m/s) t : length of sidescan transducer array (m) spd : ndarray, vessel speed (m/s) time_s : ndarray, time elapsed (s) caltime : ndarray, unix epoch time (s) ''' # prompt user to supply file if no input file given if not humfile: print 'An input file is required!!!!!!' Tk().withdraw() # we don't want a full GUI, so keep the root window from appearing humfile = askopenfilename(filetypes=[("DAT files","*.DAT")]) # prompt user to supply directory if no input sonpath is given if not sonpath: print 'A *.SON directory is required!!!!!!' Tk().withdraw() # we don't want a full GUI, so keep the root window from appearing sonpath = askdirectory() # print given arguments to screen and convert data type where necessary if humfile: print 'Input file is %s' % (humfile) if sonpath: print 'Son files are in %s' % (sonpath) if cs2cs_args: print 'cs2cs arguments are %s' % (cs2cs_args) if draft: draft = float(draft) print 'Draft: %s' % (str(draft)) if c: c = float(c) print 'Celerity of sound: %s m/s' % (str(c)) if doplot: doplot = int(doplot) if doplot==0: print "Plots will not be made" if flip_lr: flip_lr = int(flip_lr) if flip_lr==1: print "Port and starboard will be flipped" if t: t = np.asarray(t,float) print 'Transducer length is %s m' % (str(t)) if bedpick: bedpick = np.asarray(bedpick,int) if bedpick==1: print 'Bed picking is auto' elif bedpick==0: print 'Bed picking is manual' else: print 'User will be prompted per chunk about bed picking method' if chunk: chunk = str(chunk) if chunk[0]=='d': chunkmode=1 chunkval = int(chunk[1:]) print 'Chunks based on distance of %s m' % (str(chunkval)) elif chunk[0]=='p': chunkmode=2 chunkval = int(chunk[1:]) print 'Chunks based on %s pings' % (str(chunkval)) elif chunk[0]=='h': chunkmode=3 chunkval = int(chunk[1:]) print 'Chunks based on heading devation of %s degrees' % (str(chunkval)) elif chunk[0]=='1': chunkmode=4 chunkval = 1 print 'Only 1 chunk will be produced' else: print "Chunk mode not understood - should be 'd', 'p', or 'h' - using defaults" chunkmode=1 chunkval = 100 print 'Chunks based on distance of %s m' % (str(chunkval)) if model: model = int(model) print "Data is from the %s series" % (str(model)) # if cog: # cog = int(cog) # if cog==1: # print "Heading based on course-over-ground" if calc_bearing: calc_bearing = int(calc_bearing) if calc_bearing==1: print "Bearing will be calculated from coordinates" if filt_bearing: filt_bearing = int(filt_bearing) if filt_bearing==1: print "Bearing will be filtered" ## for debugging #humfile = r"test.DAT"; sonpath = "test_data" #cs2cs_args = "epsg:26949"; doplot = 1; draft = 0 #c=1450; bedpick=1; fliplr=1; chunk = 'd100' #model=998; cog=1; calc_bearing=0; filt_bearing=0 f = 455 try: print "Checking the epsg code you have chosen for compatibility with Basemap ... " from mpl_toolkits.basemap import Basemap m = Basemap(projection='merc', epsg=cs2cs_args.split(':')[1], resolution = 'i', llcrnrlon=10, llcrnrlat=10, urcrnrlon=30, urcrnrlat=30) del m print "... epsg code compatible" except: print "Error: the epsg code you have chosen is not compatible with Basemap" print "please choose a different epsg code (http://spatialreference.org/)" print "program will now close" sys.exit() # start timer if os.name=='posix': # true if linux/mac or cygwin on windows start = time.time() else: # windows start = time.clock() # if son path name supplied has no separator at end, put one on if sonpath[-1]!=os.sep: sonpath = sonpath + os.sep # get the SON files from this directory sonfiles = glob.glob(sonpath+'*.SON') if not sonfiles: sonfiles = glob.glob(os.getcwd()+os.sep+sonpath+'*.SON') base = humfile.split('.DAT') # get base of file name for output base = base[0].split(os.sep)[-1] # remove underscores, negatives and spaces from basename base = humutils.strip_base(base) print "WARNING: Because files have to be read in byte by byte," print "this could take a very long time ..." #reading each sonfile in parallel should be faster ... try: o = Parallel(n_jobs = np.min([len(sonfiles), cpu_count()]), verbose=0)(delayed(getscans)(sonfiles[k], humfile, c, model, cs2cs_args) for k in xrange(len(sonfiles))) X, Y, A, B = zip(*o) for k in xrange(len(Y)): if Y[k] == 'sidescan_port': dat = A[k] #data.gethumdat() metadat = B[k] #data.getmetadata() if flip_lr==0: data_port = X[k].astype('int16') else: data_star = X[k].astype('int16') elif Y[k] == 'sidescan_starboard': if flip_lr==0: data_star = X[k].astype('int16') else: data_port = X[k].astype('int16') elif Y[k] == 'down_lowfreq': data_dwnlow = X[k].astype('int16') elif Y[k] == 'down_highfreq': data_dwnhi = X[k].astype('int16') del X, Y, A, B, o old_pyread = 0 if 'data_port' not in locals(): data_port = '' print "portside scan not available" if 'data_star' not in locals(): data_star = '' print "starboardside scan not available" if 'data_dwnhi' not in locals(): data_dwnlow = '' print "high-frq. downward scan not available" if 'data_dwnlow' not in locals(): data_dwnlow = '' print "low-frq. downward scan not available" except: # revert back to older version if paralleleised version fails print "something went wrong with the parallelised version of pyread ..." import pyread data = pyread.pyread(sonfiles, humfile, c, model, cs2cs_args) dat = data.gethumdat() metadat = data.getmetadata() old_pyread = 1 nrec = len(metadat['n']) metadat['instr_heading'] = metadat['heading'][:nrec] #metadat['heading'] = humutils.get_bearing(calc_bearing, filt_bearing, cog, metadat['lat'], metadat['lon'], metadat['instr_heading']) try: es = humutils.runningMeanFast(metadat['e'][:nrec],len(metadat['e'][:nrec])/100) ns = humutils.runningMeanFast(metadat['n'][:nrec],len(metadat['n'][:nrec])/100) except: es = metadat['e'][:nrec] ns = metadat['n'][:nrec] metadat['es'] = es metadat['ns'] = ns try: trans = pyproj.Proj(init=cs2cs_args) except: trans = pyproj.Proj(cs2cs_args.lstrip(), inverse=True) lon, lat = trans(es, ns, inverse=True) metadat['lon'] = lon metadat['lat'] = lat metadat['heading'] = humutils.get_bearing(calc_bearing, filt_bearing, metadat['lat'], metadat['lon'], metadat['instr_heading']) #cog dist_m = humutils.get_dist(lat, lon) metadat['dist_m'] = dist_m if calc_bearing==1: # recalculate speed, m/s ds=np.gradient(np.squeeze(metadat['time_s'])) dx=np.gradient(np.squeeze(metadat['dist_m'])) metadat['spd'] = dx[:nrec]/ds[:nrec] # theta at 3dB in the horizontal theta3dB = np.arcsin(c/(t*(f*1000))) #resolution of 1 sidescan pixel to nadir ft = (np.pi/2)*(1/theta3dB) #/ (f/455) dep_m = humutils.get_depth(metadat['dep_m'][:nrec]) if old_pyread == 1: #older pyread version # port scan try: if flip_lr==0: data_port = data.getportscans().astype('int16') else: data_port = data.getstarscans().astype('int16') except: data_port = '' print "portside scan not available" if data_port!='': Zt, ind_port = makechunks_scan(chunkmode, chunkval, metadat, data_port, 0) del data_port ## create memory mapped file for Z shape_port = io.set_mmap_data(sonpath, base, '_data_port.dat', 'int16', Zt) ##we are only going to access the portion of memory required port_fp = io.get_mmap_data(sonpath, base, '_data_port.dat', 'int16', shape_port) if old_pyread == 1: #older pyread version # starboard scan try: if flip_lr==0: data_star = data.getstarscans().astype('int16') else: data_star = data.getportscans().astype('int16') except: data_star = '' print "starboardside scan not available" if data_star!='': Zt, ind_star = makechunks_scan(chunkmode, chunkval, metadat, data_star, 1) del data_star # create memory mapped file for Z shape_star = io.set_mmap_data(sonpath, base, '_data_star.dat', 'int16', Zt) star_fp = io.get_mmap_data(sonpath, base, '_data_star.dat', 'int16', shape_star) if 'star_fp' in locals() and 'port_fp' in locals(): # check that port and starboard are same size # and trim if not if np.shape(star_fp)!=np.shape(port_fp): print "port and starboard scans are different sizes ... rectifying" if np.shape(port_fp[0])[1] > np.shape(star_fp[0])[1]: tmp = port_fp.copy() tmp2 = np.empty_like(star_fp) for k in xrange(len(tmp)): tmp2[k] = tmp[k][:,:np.shape(star_fp[k])[1]] del tmp # create memory mapped file for Z shape_port = io.set_mmap_data(sonpath, base, '_data_port2.dat', 'int16', tmp2) #shape_star = shape_port.copy() shape_star = tuple(np.asarray(shape_port).copy()) ##we are only going to access the portion of memory required port_fp = io.get_mmap_data(sonpath, base, '_data_port2.dat', 'int16', shape_port) ind_port = list(ind_port) ind_port[-1] = np.shape(star_fp[0])[1] ind_port = tuple(ind_port) elif np.shape(port_fp[0])[1] < np.shape(star_fp[0])[1]: tmp = star_fp.copy() tmp2 = np.empty_like(port_fp) for k in xrange(len(tmp)): tmp2[k] = tmp[k][:,:np.shape(port_fp[k])[1]] del tmp # create memory mapped file for Z shape_port = io.set_mmap_data(sonpath, base, '_data_star2.dat', 'int16', tmp2) #shape_star = shape_port.copy() shape_star = tuple(np.asarray(shape_port).copy()) #we are only going to access the portion of memory required star_fp = io.get_mmap_data(sonpath, base, '_data_star2.dat', 'int16', shape_star) ind_star = list(ind_star) ind_star[-1] = np.shape(port_fp[0])[1] ind_star = tuple(ind_star) if old_pyread == 1: #older pyread version # low-freq. sonar try: data_dwnlow = data.getlowscans().astype('int16') except: data_dwnlow = '' print "low-freq. scan not available" if data_dwnlow!='': Zt, ind_low = makechunks_scan(chunkmode, chunkval, metadat, data_dwnlow, 2) del data_dwnlow # create memory mapped file for Z shape_low = io.set_mmap_data(sonpath, base, '_data_dwnlow.dat', 'int16', Zt) ##we are only going to access the portion of memory required dwnlow_fp = io.get_mmap_data(sonpath, base, '_data_dwnlow.dat', 'int16', shape_low) if old_pyread == 1: #older pyread version # hi-freq. sonar try: data_dwnhi = data.gethiscans().astype('int16') except: data_dwnhi = '' print "high-freq. scan not available" if data_dwnhi!='': Zt, ind_hi = makechunks_scan(chunkmode, chunkval, metadat, data_dwnhi, 3) del data_dwnhi # create memory mapped file for Z shape_hi = io.set_mmap_data(sonpath, base, '_data_dwnhi.dat', 'int16', Zt) dwnhi_fp = io.get_mmap_data(sonpath, base, '_data_dwnhi.dat', 'int16', shape_hi) if 'dwnhi_fp' in locals() and 'dwnlow_fp' in locals(): # check that low and high are same size # and trim if not if (np.shape(dwnhi_fp)!=np.shape(dwnlow_fp)) and (chunkmode!=4): print "dwnhi and dwnlow are different sizes ... rectifying" if np.shape(dwnhi_fp[0])[1] > np.shape(dwnlow_fp[0])[1]: tmp = dwnhi_fp.copy() tmp2 = np.empty_like(dwnlow_fp) for k in xrange(len(tmp)): tmp2[k] = tmp[k][:,:np.shape(dwnlow_fp[k])[1]] del tmp # create memory mapped file for Z shape_low = io.set_mmap_data(sonpath, base, '_data_dwnhi2.dat', 'int16', tmp2) #shape_hi = shape_low.copy() shape_hi = tuple(np.asarray(shape_low).copy()) ##we are only going to access the portion of memory required dwnhi_fp = io.get_mmap_data(sonpath, base, '_data_dwnhi2.dat', 'int16', shape_hi) ind_hi = list(ind_hi) ind_hi[-1] = np.shape(dwnlow_fp[0])[1] ind_hi = tuple(ind_hi) elif np.shape(dwnhi_fp[0])[1] < np.shape(dwnlow_fp[0])[1]: tmp = dwnlow_fp.copy() tmp2 = np.empty_like(dwnhi_fp) for k in xrange(len(tmp)): tmp2[k] = tmp[k][:,:np.shape(dwnhi_fp[k])[1]] del tmp # create memory mapped file for Z shape_low = io.set_mmap_data(sonpath, base, '_data_dwnlow2.dat', 'int16', tmp2) #shape_hi = shape_low.copy() shape_hi = tuple(np.asarray(shape_low).copy()) ##we are only going to access the portion of memory required dwnlow_fp = io.get_mmap_data(sonpath, base, '_data_dwnlow2.dat', 'int16', shape_low) ind_low = list(ind_low) ind_low[-1] = np.shape(dwnhi_fp[0])[1] ind_low = tuple(ind_low) if old_pyread == 1: #older pyread version del data if ('shape_port' in locals()) and (chunkmode!=4): metadat['shape_port'] = shape_port nrec = metadat['shape_port'][0] * metadat['shape_port'][2] elif ('shape_port' in locals()) and (chunkmode==4): metadat['shape_port'] = shape_port nrec = metadat['shape_port'][1] else: metadat['shape_port'] = '' if ('shape_star' in locals()) and (chunkmode!=4): metadat['shape_star'] = shape_star nrec = metadat['shape_star'][0] * metadat['shape_star'][2] elif ('shape_star' in locals()) and (chunkmode==4): metadat['shape_star'] = shape_star nrec = metadat['shape_star'][1] else: metadat['shape_star'] = '' if ('shape_hi' in locals()) and (chunkmode!=4): metadat['shape_hi'] = shape_hi #nrec = metadat['shape_hi'][0] * metadat['shape_hi'][2] * 2 elif ('shape_hi' in locals()) and (chunkmode==4): metadat['shape_hi'] = shape_hi else: metadat['shape_hi'] = '' if ('shape_low' in locals()) and (chunkmode!=4): metadat['shape_low'] = shape_low #nrec = metadat['shape_low'][0] * metadat['shape_low'][2] * 2 elif ('shape_low' in locals()) and (chunkmode==4): metadat['shape_low'] = shape_low else: metadat['shape_low'] = '' #make kml boat trackline humutils.make_trackline(lon,lat, sonpath, base) if 'port_fp' in locals() and 'star_fp' in locals(): #if not os.path.isfile(os.path.normpath(os.path.join(sonpath,base+'meta.mat'))): if 2>1: if bedpick == 1: # auto x, bed = humutils.auto_bedpick(ft, dep_m, chunkmode, port_fp, c) if len(dist_m)<len(bed): dist_m = np.append(dist_m,dist_m[-1]*np.ones(len(bed)-len(dist_m))) if doplot==1: if chunkmode!=4: for k in xrange(len(star_fp)): plot_2bedpicks(port_fp[k], star_fp[k], bed[ind_port[-1]*k:ind_port[-1]*(k+1)], dist_m[ind_port[-1]*k:ind_port[-1]*(k+1)], x[ind_port[-1]*k:ind_port[-1]*(k+1)], ft, shape_port, sonpath, k, chunkmode) else: plot_2bedpicks(port_fp, star_fp, bed, dist_m, x, ft, shape_port, sonpath, 0, chunkmode) # 'real' bed is estimated to be the minimum of the two bed = np.min(np.vstack((bed[:nrec],np.squeeze(x[:nrec]))),axis=0) bed = humutils.runningMeanFast(bed, 3) elif bedpick>1: # user prompt x, bed = humutils.auto_bedpick(ft, dep_m, chunkmode, port_fp, c) if len(dist_m)<len(bed): dist_m = np.append(dist_m,dist_m[-1]*np.ones(len(bed)-len(dist_m))) # 'real' bed is estimated to be the minimum of the two bed = np.min(np.vstack((bed[:nrec],np.squeeze(x[:nrec]))),axis=0) bed = humutils.runningMeanFast(bed, 3) # manually intervene fig = plt.figure() ax = plt.gca() if chunkmode !=4: im = ax.imshow(np.hstack(port_fp), cmap = 'gray', origin = 'upper') else: im = ax.imshow(port_fp, cmap = 'gray', origin = 'upper') plt.plot(bed,'r') plt.axis('normal'); plt.axis('tight') pts1 = plt.ginput(n=300, timeout=30) # it will wait for 200 clicks or 60 seconds x1=map(lambda x: x[0],pts1) # map applies the function passed as y1=map(lambda x: x[1],pts1) # first parameter to each element of pts plt.close() del fig if x1 != []: # if x1 is not empty tree = KDTree(zip(np.arange(1,len(bed)), bed)) try: dist, inds = tree.query(zip(x1, y1), k = 100, eps=5, n_jobs=-1) except: dist, inds = tree.query(zip(x1, y1), k = 100, eps=5) b = np.interp(inds,x1,y1) bed2 = bed.copy() bed2[inds] = b bed = bed2 if doplot==1: if chunkmode!=4: for k in xrange(len(star_fp)): plot_2bedpicks(port_fp[k], star_fp[k], bed[ind_port[-1]*k:ind_port[-1]*(k+1)], dist_m[ind_port[-1]*k:ind_port[-1]*(k+1)], x[ind_port[-1]*k:ind_port[-1]*(k+1)], ft, shape_port, sonpath, k, chunkmode) else: plot_2bedpicks(port_fp, star_fp, bed, dist_m, x, ft, shape_port, sonpath, 0, chunkmode) else: #manual beds=[] if chunkmode!=4: for k in xrange(len(port_fp)): raw_input("Bed picking "+str(k+1)+" of "+str(len(port_fp))+", are you ready? 30 seconds. Press Enter to continue...") bed={} fig = plt.figure() ax = plt.gca() im = ax.imshow(port_fp[k], cmap = 'gray', origin = 'upper') pts1 = plt.ginput(n=300, timeout=30) # it will wait for 200 clicks or 60 seconds x1=map(lambda x: x[0],pts1) # map applies the function passed as y1=map(lambda x: x[1],pts1) # first parameter to each element of pts bed = np.interp(np.r_[:ind_port[-1]],x1,y1) plt.close() del fig beds.append(bed) extent = np.shape(port_fp[k])[0] bed = np.asarray(np.hstack(beds),'float') else: raw_input("Bed picking - are you ready? 30 seconds. Press Enter to continue...") bed={} fig = plt.figure() ax = plt.gca() im = ax.imshow(port_fp, cmap = 'gray', origin = 'upper') pts1 = plt.ginput(n=300, timeout=30) # it will wait for 200 clicks or 60 seconds x1=map(lambda x: x[0],pts1) # map applies the function passed as y1=map(lambda x: x[1],pts1) # first parameter to each element of pts bed = np.interp(np.r_[:ind_port[-1]],x1,y1) plt.close() del fig beds.append(bed) extent = np.shape(port_fp)[1] bed = np.asarray(np.hstack(beds),'float') # now revise the depth in metres dep_m = (1/ft)*bed if doplot==1: if chunkmode!=4: for k in xrange(len(star_fp)): plot_bedpick(port_fp[k], star_fp[k], (1/ft)*bed[ind_port[-1]*k:ind_port[-1]*(k+1)], dist_m[ind_port[-1]*k:ind_port[-1]*(k+1)], ft, shape_port, sonpath, k, chunkmode) else: plot_bedpick(port_fp, star_fp, (1/ft)*bed, dist_m, ft, shape_port, sonpath, 0, chunkmode) metadat['bed'] = bed[:nrec] else: metadat['bed'] = dep_m[:nrec]*ft metadat['heading'] = metadat['heading'][:nrec] metadat['lon'] = lon[:nrec] metadat['lat'] = lat[:nrec] metadat['dist_m'] = dist_m[:nrec] metadat['dep_m'] = dep_m[:nrec] metadat['pix_m'] = 1/ft metadat['bed'] = metadat['bed'][:nrec] metadat['c'] = c metadat['t'] = t metadat['f'] = f metadat['spd'] = metadat['spd'][:nrec] metadat['time_s'] = metadat['time_s'][:nrec] metadat['e'] = metadat['e'][:nrec] metadat['n'] = metadat['n'][:nrec] metadat['es'] = metadat['es'][:nrec] metadat['ns'] = metadat['ns'][:nrec] metadat['caltime'] = metadat['caltime'][:nrec] savemat(os.path.normpath(os.path.join(sonpath,base+'meta.mat')), metadat ,oned_as='row') f = open(os.path.normpath(os.path.join(sonpath,base+'rawdat.csv')), 'wt') writer = csv.writer(f) writer.writerow( ('longitude', 'latitude', 'easting', 'northing', 'depth (m)', 'distance (m)', 'instr. heading (deg)', 'heading (deg.)' ) ) for i in range(0, nrec): writer.writerow(( float(lon[i]),float(lat[i]),float(es[i]),float(ns[i]),float(dep_m[i]),float(dist_m[i]), float(metadat['instr_heading'][i]), float(metadat['heading'][i]) )) f.close() del lat, lon, dep_m #, dist_m if doplot==1: plot_pos(sonpath, metadat, es, ns) if 'dwnlow_fp' in locals(): plot_dwnlow(dwnlow_fp, chunkmode, sonpath) if 'dwnhi_fp' in locals(): plot_dwnhi(dwnhi_fp, chunkmode, sonpath) if os.name=='posix': # true if linux/mac elapsed = (time.time() - start) else: # windows elapsed = (time.clock() - start) print "Processing took ", elapsed , "seconds to analyse" print "Done!"
def correct(humfile, sonpath, maxW=1000, doplot=1, dofilt=0, correct_withwater=0, ph=7, temp=10, salinity=0, dconcfile=None): ''' Remove water column and carry out some rudimentary radiometric corrections, accounting for directivity and attenuation with range Syntax ---------- [] = PyHum.correct(humfile, sonpath, maxW, doplot, correct_withwater, ph, temp, salinity, dconcfile) Parameters ---------- humfile : str path to the .DAT file sonpath : str path where the *.SON files are maxW : int, *optional* [Default=1000] maximum transducer power doplot : int, *optional* [Default=1] 1 = make plots, otherwise do not dofilt : int, *optional* [Default=0] 1 = apply a phase preserving filter to the scans correct_withwater : int, *optional* [Default=0] 1 = apply radiometric correction but don't remove water column from scans ph : float, *optional* [Default=7.0] water acidity in pH temp : float, *optional* [Default=10.0] water temperature in degrees Celsius salinity : float, *optional* [Default=0.0] salinity of water in parts per thousand dconcfile : str, *optional* [Default=None] file path of a text file containing sediment concentration data this file must contain the following fields separated by spaces: size (microns) conc (mg/L) dens (kg/m3) with one row per grain size, for example: 30 1700 2200 100 15 2650 Returns ------- sonpath+base+'_data_star_l.dat': memory-mapped file contains the starboard scan with water column removed sonpath+base+'_data_port_l.dat': memory-mapped file contains the portside scan with water column removed sonpath+base+'_data_star_la.dat': memory-mapped file contains the starboard scan with water column removed and radiometrically corrected sonpath+base+'_data_port_la.dat': memory-mapped file contains the portside scan with water column removed and radiometrically corrected sonpath+base+'_data_range.dat': memory-mapped file contains the cosine of the range which is used to correct for attenuation with range sonpath+base+'_data_dwnlow_l.dat': memory-mapped file contains the low freq. downward scan with water column removed sonpath+base+'_data_dwnhi_l.dat': memory-mapped file contains the high freq. downward scan with water column removed sonpath+base+'_data_dwnlow_la.dat': memory-mapped file contains the low freq. downward scan with water column removed and radiometrically corrected sonpath+base+'_data_dwnhi_la.dat': memory-mapped file contains the high freq. downward scan with water column removed and radiometrically corrected if correct_withwater == 1: sonpath+base+'_data_star_lw.dat': memory-mapped file contains the starboard scan with water column retained and radiometrically corrected sonpath+base+'_data_port_lw.dat': memory-mapped file contains the portside scan with water column retained and radiometrically corrected ''' # prompt user to supply file if no input file given if not humfile: print 'An input file is required!!!!!!' Tk().withdraw( ) # we don't want a full GUI, so keep the root window from appearing inputfile = askopenfilename(filetypes=[("DAT files", "*.DAT")]) # prompt user to supply directory if no input sonpath is given if not sonpath: print 'A *.SON directory is required!!!!!!' Tk().withdraw( ) # we don't want a full GUI, so keep the root window from appearing sonpath = askdirectory() # print given arguments to screen and convert data type where necessary if humfile: print 'Input file is %s' % (humfile) if sonpath: print 'Sonar file path is %s' % (sonpath) if maxW: maxW = np.asarray(maxW, float) print 'Max. transducer power is %s W' % (str(maxW)) if doplot: doplot = int(doplot) if doplot == 0: print "Plots will not be made" if dofilt: dofilt = int(dofilt) if dofilt == 0: print "Phase preserving filter will not be applied" else: print "Phase preserving filter will be applied" if correct_withwater: correct_withwater = int(correct_withwater) if correct_withwater == 1: print "Correction will be applied without removing water column" if salinity: salinity = np.asarray(salinity, float) print 'Salinity is %s ppt' % (str(salinity)) if ph: ph = np.asarray(ph, float) print 'pH is %s' % (str(ph)) if temp: temp = np.asarray(temp, float) print 'Temperature is %s' % (str(temp)) if dconcfile is not None: try: print 'Suspended sediment size/conc. file is %s' % (dconcfile) dconc = np.genfromtxt(dconcfile).T conc = dconc[1] dens = dconc[2] d = dconc[0] except: pass #================================ # start timer if os.name == 'posix': # true if linux/mac or cygwin on windows start = time.time() else: # windows start = time.clock() # if son path name supplied has no separator at end, put one on if sonpath[-1] != os.sep: sonpath = sonpath + os.sep base = humfile.split('.DAT') # get base of file name for output base = base[0].split(os.sep)[-1] # remove underscores, negatives and spaces from basename base = humutils.strip_base(base) # add wattage to metadata dict meta = loadmat(os.path.normpath(os.path.join(sonpath, base + 'meta.mat'))) dep_m = meta['dep_m'][0] pix_m = meta['pix_m'][0] meta['maxW'] = maxW savemat(os.path.normpath(os.path.join(sonpath, base + 'meta.mat')), meta, oned_as='row') bed = np.squeeze(meta['bed']) ft = 1 / (meta['pix_m']) dist_m = np.squeeze(meta['dist_m']) try: if dconcfile is not None: # sediment attenuation alpha = sed_atten(meta['f'], conc, dens, d, meta['c']) else: alpha = 0 except: alpha = 0 # load memory mapped scans shape_port = np.squeeze(meta['shape_port']) if shape_port != '': if os.path.isfile( os.path.normpath( os.path.join(sonpath, base + '_data_port2.dat'))): port_fp = io.get_mmap_data(sonpath, base, '_data_port2.dat', 'int16', tuple(shape_port)) else: port_fp = io.get_mmap_data(sonpath, base, '_data_port.dat', 'int16', tuple(shape_port)) shape_star = np.squeeze(meta['shape_star']) if shape_star != '': if os.path.isfile( os.path.normpath( os.path.join(sonpath, base + '_data_star2.dat'))): star_fp = io.get_mmap_data(sonpath, base, '_data_star2.dat', 'int16', tuple(shape_star)) else: star_fp = io.get_mmap_data(sonpath, base, '_data_star.dat', 'int16', tuple(shape_star)) if len(shape_star) == 2: extent = shape_star[0] else: extent = shape_star[1] #np.shape(data_port)[0] bed = np.asarray(bed, 'int') + int(0.25 * ft) # calculate in dB ######### star Zt, R, A = remove_water(star_fp, bed, shape_star, dep_m, pix_m, 1, maxW) Zt = np.squeeze(Zt) # create memory mapped file for Z) shape_star = io.set_mmap_data(sonpath, base, '_data_star_l.dat', 'float32', Zt) del Zt A = np.squeeze(A) # create memory mapped file for A shape_A = io.set_mmap_data(sonpath, base, '_data_incidentangle.dat', 'float32', A) del A R = np.squeeze(R) R[np.isnan(R)] = 0 try: alpha_w = water_atten(R, meta['f'], meta['c'], ph, temp, salinity) except: alpha_w = 1e-5 # compute transmission losses TL = (40 * np.log10(R) + alpha_w + (2 * alpha) * R / 1000) / 255 del alpha_w # create memory mapped file for R shape_R = io.set_mmap_data(sonpath, base, '_data_range.dat', 'float32', R) del R TL[np.isnan(TL)] = 0 TL[TL < 0] = 0 shape_TL = io.set_mmap_data(sonpath, base, '_data_TL.dat', 'float32', TL) del TL A_fp = io.get_mmap_data(sonpath, base, '_data_incidentangle.dat', 'float32', shape_star) TL_fp = io.get_mmap_data(sonpath, base, '_data_TL.dat', 'float32', shape_star) R_fp = io.get_mmap_data(sonpath, base, '_data_range.dat', 'float32', shape_star) if correct_withwater == 1: Zt = correct_scans(star_fp, A_fp, TL_fp, dofilt) # create memory mapped file for Z) shape_star = io.set_mmap_data(sonpath, base, '_data_star_lw.dat', 'float32', Zt) #we are only going to access the portion of memory required star_fp = io.get_mmap_data(sonpath, base, '_data_star_l.dat', 'float32', shape_star) ##Zt = correct_scans(star_fp, A_fp, TL_fp, dofilt) #phi=1.69 alpha = 59 # vertical beam width at 3db theta = 35 #opening angle theta # lambertian correction Zt = correct_scans_lambertian(star_fp, A_fp, TL_fp, R_fp, meta['c'], meta['f'], theta, alpha) Zt = np.squeeze(Zt) avg = np.nanmedian(Zt, axis=1) Zt2 = np.empty(np.shape(Zt)) for kk in xrange(np.shape(Zt)[1]): Zt2[:, kk] = (Zt[:, kk] - avg) + np.nanmean(avg) Zt2[Zt <= 0] = np.nan Zt2[Zt2 <= 0] = np.nan del Zt # create memory mapped file for Z shape_star = io.set_mmap_data(sonpath, base, '_data_star_la.dat', 'float32', Zt2) del Zt2 #we are only going to access the portion of memory required star_fp = io.get_mmap_data(sonpath, base, '_data_star_la.dat', 'float32', shape_star) ######### port if correct_withwater == 1: Zt = correct_scans(port_fp, A_fp, TL, dofilt) # create memory mapped file for Z) shape_port = io.set_mmap_data(sonpath, base, '_data_port_lw.dat', 'float32', Zt) Zt = remove_water(port_fp, bed, shape_port, dep_m, pix_m, 0, maxW) Zt = np.squeeze(Zt) # create memory mapped file for Z shape_port = io.set_mmap_data(sonpath, base, '_data_port_l.dat', 'float32', Zt) #we are only going to access the portion of memory required port_fp = io.get_mmap_data(sonpath, base, '_data_port_l.dat', 'float32', shape_port) ##Zt = correct_scans(port_fp, A_fp, TL_fp, dofilt) # lambertian correction Zt = correct_scans_lambertian(port_fp, A_fp, TL_fp, R_fp, meta['c'], meta['f'], theta, alpha) Zt = np.squeeze(Zt) Zt2 = np.empty(np.shape(Zt)) for kk in xrange(np.shape(Zt)[1]): Zt2[:, kk] = (Zt[:, kk] - avg) + np.nanmean(avg) Zt2[Zt <= 0] = np.nan Zt2[Zt2 <= 0] = np.nan del Zt # create memory mapped file for Z shape_port = io.set_mmap_data(sonpath, base, '_data_port_la.dat', 'float32', Zt2) del Zt2 #we are only going to access the portion of memory required port_fp = io.get_mmap_data(sonpath, base, '_data_port_la.dat', 'float32', shape_port) ## do plots of merged scans if doplot == 1: if correct_withwater == 1: port_fpw = io.get_mmap_data(sonpath, base, '_data_port_lw.dat', 'float32', shape_port) star_fpw = io.get_mmap_data(sonpath, base, '_data_star_lw.dat', 'float32', shape_star) if len(np.shape(star_fpw)) > 2: for p in xrange(len(star_fpw)): plot_merged_scans(port_fpw[p], star_fpw[p], dist_m, shape_port, ft, sonpath, p) else: plot_merged_scans(port_fpw, star_fpw, dist_m, shape_port, ft, sonpath, 0) else: if len(np.shape(star_fp)) > 2: for p in xrange(len(star_fp)): plot_merged_scans(port_fp[p], star_fp[p], dist_m, shape_port, ft, sonpath, p) else: plot_merged_scans(port_fp, star_fp, dist_m, shape_port, ft, sonpath, 0) # load memory mapped scans shape_low = np.squeeze(meta['shape_low']) shape_hi = np.squeeze(meta['shape_hi']) if shape_low != '': if os.path.isfile( os.path.normpath( os.path.join(sonpath, base + '_data_dwnlow2.dat'))): try: low_fp = io.get_mmap_data(sonpath, base, '_data_dwnlow2.dat', 'int16', tuple(shape_low)) except: low_fp = io.get_mmap_data(sonpath, base, '_data_dwnlow.dat', 'int16', tuple(shape_low)) finally: low_fp = io.get_mmap_data(sonpath, base, '_data_dwnlow.dat', 'int16', tuple(shape_hi)) #if 'shape_hi' in locals(): # low_fp = io.get_mmap_data(sonpath, base, '_data_dwnlow2.dat', 'int16', tuple(shape_hi)) else: try: low_fp = io.get_mmap_data(sonpath, base, '_data_dwnlow.dat', 'int16', tuple(shape_low)) except: if 'shape_hi' in locals(): low_fp = io.get_mmap_data(sonpath, base, '_data_dwnlow.dat', 'int16', tuple(shape_hi)) shape_hi = np.squeeze(meta['shape_hi']) if shape_hi != '': if os.path.isfile( os.path.normpath( os.path.join(sonpath, base + '_data_dwnhi2.dat'))): try: hi_fp = io.get_mmap_data(sonpath, base, '_data_dwnhi2.dat', 'int16', tuple(shape_hi)) except: hi_fp = io.get_mmap_data(sonpath, base, '_data_dwnhi.dat', 'int16', tuple(shape_hi)) finally: hi_fp = io.get_mmap_data(sonpath, base, '_data_dwnhi.dat', 'int16', tuple(shape_low)) #if 'shape_low' in locals(): # hi_fp = io.get_mmap_data(sonpath, base, '_data_dwnhi2.dat', 'int16', tuple(shape_low)) else: try: hi_fp = io.get_mmap_data(sonpath, base, '_data_dwnhi.dat', 'int16', tuple(shape_hi)) except: if 'shape_low' in locals(): hi_fp = io.get_mmap_data(sonpath, base, '_data_dwnhi.dat', 'int16', tuple(shape_low)) if 'low_fp' in locals(): ######### low Zt = remove_water(low_fp, bed, shape_low, dep_m, pix_m, 0, maxW) Zt = np.squeeze(Zt) # create memory mapped file for Z shape_low = io.set_mmap_data(sonpath, base, '_data_dwnlow_l.dat', 'float32', Zt) del Zt #we are only going to access the portion of memory required low_fp = io.get_mmap_data(sonpath, base, '_data_dwnlow_l.dat', 'float32', shape_low) Zt = correct_scans2(low_fp, TL_fp) # create memory mapped file for Z shape_low = io.set_mmap_data(sonpath, base, '_data_dwnlow_la.dat', 'float32', Zt) del Zt #we are only going to access the lowion of memory required low_fp = io.get_mmap_data(sonpath, base, '_data_dwnlow_la.dat', 'float32', shape_low) if doplot == 1: if len(np.shape(low_fp)) > 2: for p in xrange(len(low_fp)): plot_dwnlow_scans(low_fp[p], dist_m, shape_low, ft, sonpath, p) else: plot_dwnlow_scans(low_fp, dist_m, shape_low, ft, sonpath, 0) if 'hi_fp' in locals(): ######### hi Zt = remove_water(hi_fp, bed, shape_hi, dep_m, pix_m, 0, maxW) Zt = np.squeeze(Zt) # create memory mapped file for Z shape_hi = io.set_mmap_data(sonpath, base, '_data_dwnhi_l.dat', 'float32', Zt) del Zt #we are only going to access the portion of memory required hi_fp = io.get_mmap_data(sonpath, base, '_data_dwnhi_l.dat', 'float32', shape_hi) Zt = correct_scans2(hi_fp, TL_fp) # create memory mapped file for Z shape_hi = io.set_mmap_data(sonpath, base, '_data_dwnhi_la.dat', 'float32', Zt) del Zt #we are only going to access the hiion of memory required hi_fp = io.get_mmap_data(sonpath, base, '_data_dwnhi_la.dat', 'float32', shape_hi) if doplot == 1: if len(np.shape(hi_fp)) > 2: for p in xrange(len(hi_fp)): plot_dwnhi_scans(hi_fp[p], dist_m, shape_hi, ft, sonpath, p) else: plot_dwnhi_scans(hi_fp, dist_m, shape_hi, ft, sonpath, 0) if os.name == 'posix': # true if linux/mac elapsed = (time.time() - start) else: # windows elapsed = (time.clock() - start) print "Processing took ", elapsed, "seconds to analyse" print "Done!"
def map_texture(humfile, sonpath, cs2cs_args = "epsg:26949", res = 0.5, mode=3, nn = 64, numstdevs=5): #influence = 10, ''' Create plots of the texture lengthscale maps made in PyHum.texture module using the algorithm detailed by Buscombe et al. (2015) This textural lengthscale is not a direct measure of grain size. Rather, it is a statistical representation that integrates over many attributes of bed texture, of which grain size is the most important. The technique is a physically based means to identify regions of texture within a sidescan echogram, and could provide a basis for objective, automated riverbed sediment classification. Syntax ---------- [] = PyHum.map_texture(humfile, sonpath, cs2cs_args, res, mode, nn, numstdevs) Parameters ---------- humfile : str path to the .DAT file sonpath : str path where the *.SON files are cs2cs_args : int, *optional* [Default="epsg:26949"] arguments to create coordinates in a projected coordinate system this argument gets given to pyproj to turn wgs84 (lat/lon) coordinates into any projection supported by the proj.4 libraries res : float, *optional* [Default=0.5] grid resolution of output gridded texture map mode: int, *optional* [Default=3] gridding mode. 1 = nearest neighbour 2 = inverse weighted nearest neighbour 3 = Gaussian weighted nearest neighbour nn: int, *optional* [Default=64] number of nearest neighbours for gridding (used if mode > 1) numstdevs: int, *optional* [Default = 4] Threshold number of standard deviations in texture lengthscale per grid cell up to which to accept Returns ------- sonpath+'x_y_class'+str(p)+'.asc' : text file contains the point cloud of easting, northing, and texture lengthscales of the pth chunk sonpath+'class_GroundOverlay'+str(p)+'.kml': kml file contains gridded (or point cloud) texture lengthscale map for importing into google earth of the pth chunk sonpath+'class_map'+str(p)+'.png' : image overlay associated with the kml file sonpath+'class_map_imagery'+str(p)+'.png' : png image file gridded (or point cloud) texture lengthscale map overlain onto an image pulled from esri image server References ---------- .. [1] Buscombe, D., Grams, P.E., and Smith, S.M.C., 2015, Automated riverbed sediment classification using low-cost sidescan sonar. Journal of Hydraulic Engineering 10.1061/(ASCE)HY.1943-7900.0001079, 06015019. ''' # prompt user to supply file if no input file given if not humfile: print 'An input file is required!!!!!!' Tk().withdraw() # we don't want a full GUI, so keep the root window from appearing humfile = askopenfilename(filetypes=[("DAT files","*.DAT")]) # prompt user to supply directory if no input sonpath is given if not sonpath: print 'A *.SON directory is required!!!!!!' Tk().withdraw() # we don't want a full GUI, so keep the root window from appearing sonpath = askdirectory() # print given arguments to screen and convert data type where necessary if humfile: print 'Input file is %s' % (humfile) if sonpath: print 'Sonar file path is %s' % (sonpath) if cs2cs_args: print 'cs2cs arguments are %s' % (cs2cs_args) if res: res = np.asarray(res,float) print 'Gridding resolution: %s' % (str(res)) if mode: mode = int(mode) print 'Mode for gridding: %s' % (str(mode)) if nn: nn = int(nn) print 'Number of nearest neighbours for gridding: %s' % (str(nn)) #if influence: # influence = int(influence) # print 'Radius of influence for gridding: %s (m)' % (str(influence)) if numstdevs: numstdevs = int(numstdevs) print 'Threshold number of standard deviations in texture lengthscale per grid cell up to which to accept: %s' % (str(numstdevs)) # start timer if os.name=='posix': # true if linux/mac or cygwin on windows start = time.time() else: # windows start = time.clock() trans = pyproj.Proj(init=cs2cs_args) # if son path name supplied has no separator at end, put one on if sonpath[-1]!=os.sep: sonpath = sonpath + os.sep base = humfile.split('.DAT') # get base of file name for output base = base[0].split(os.sep)[-1] # remove underscores, negatives and spaces from basename base = humutils.strip_base(base) meta = loadmat(os.path.normpath(os.path.join(sonpath,base+'meta.mat'))) esi = np.squeeze(meta['e']) nsi = np.squeeze(meta['n']) pix_m = np.squeeze(meta['pix_m']) dep_m = np.squeeze(meta['dep_m']) c = np.squeeze(meta['c']) #dist_m = np.squeeze(meta['dist_m']) theta = np.squeeze(meta['heading'])/(180/np.pi) # load memory mapped scans shape_port = np.squeeze(meta['shape_port']) if shape_port!='': if os.path.isfile(os.path.normpath(os.path.join(sonpath,base+'_data_port_lar.dat'))): port_fp = io.get_mmap_data(sonpath, base, '_data_port_lar.dat', 'float32', tuple(shape_port)) else: port_fp = io.get_mmap_data(sonpath, base, '_data_port_la.dat', 'float32', tuple(shape_port)) shape_star = np.squeeze(meta['shape_star']) if shape_star!='': if os.path.isfile(os.path.normpath(os.path.join(sonpath,base+'_data_star_lar.dat'))): star_fp = io.get_mmap_data(sonpath, base, '_data_star_lar.dat', 'float32', tuple(shape_star)) else: star_fp = io.get_mmap_data(sonpath, base, '_data_star_la.dat', 'float32', tuple(shape_star)) if len(shape_star)>2: shape = shape_port.copy() shape[1] = shape_port[1] + shape_star[1] class_fp = io.get_mmap_data(sonpath, base, '_data_class.dat', 'float32', tuple(shape)) #with open(os.path.normpath(os.path.join(sonpath,base+'_data_class.dat')), 'r') as ff: # class_fp = np.memmap(ff, dtype='float32', mode='r', shape=tuple(shape)) else: with open(os.path.normpath(os.path.join(sonpath,base+'_data_class.dat')), 'r') as ff: class_fp = np.load(ff) tvg = ((8.5*10**-5)+(3/76923)+((8.5*10**-5)/4))*c dist_tvg = ((np.tan(np.radians(25)))*dep_m)-(tvg) if len(shape_star)>2: for p in xrange(len(class_fp)): e = esi[shape_port[-1]*p:shape_port[-1]*(p+1)] n = nsi[shape_port[-1]*p:shape_port[-1]*(p+1)] t = theta[shape_port[-1]*p:shape_port[-1]*(p+1)] d = dist_tvg[shape_port[-1]*p:shape_port[-1]*(p+1)] len_n = len(n) merge = class_fp[p].copy() merge[np.isnan(merge)] = 0 merge[np.isnan(np.vstack((np.flipud(port_fp[p]),star_fp[p])))] = 0 extent = shape_port[1] R1 = merge[extent:,:] R2 = np.flipud(merge[:extent,:]) merge = np.vstack((R2,R1)) del R1, R2 # get number pixels in scan line extent = int(np.shape(merge)[0]/2) yvec = np.linspace(pix_m,extent*pix_m,extent) X, Y = getXY(e,n,yvec,d,t,extent) merge[merge==0] = np.nan if len(merge.flatten()) != len(X): merge = merge[:,:len_n] merge = merge.T.flatten() index = np.where(np.logical_not(np.isnan(merge)))[0] X, Y, merge = trim_xys(X, Y, merge, index) X = X.astype('float32') Y = Y.astype('float32') merge = merge.astype('float32') # write raw bs to file outfile = os.path.normpath(os.path.join(sonpath,'x_y_class'+str(p)+'.asc')) with open(outfile, 'w') as f: np.savetxt(f, np.hstack((humutils.ascol(X),humutils.ascol(Y), humutils.ascol(merge))), delimiter=' ', fmt="%8.6f %8.6f %8.6f") humlon, humlat = trans(X, Y, inverse=True) #if dogrid==1: orig_def, targ_def, grid_x, grid_y, res, shape = get_griddefs(np.min(X), np.max(X), np.min(Y), np.max(Y), res, humlon, humlat, trans) grid_x = grid_x.astype('float32') grid_y = grid_y.astype('float32') sigmas = 1 #m eps = 2 dat, res = get_grid(mode, orig_def, targ_def, merge, res*10, np.min(X), np.max(X), np.min(Y), np.max(Y), res, nn, sigmas, eps, shape, numstdevs, trans, humlon, humlat) del merge dat[dat==0] = np.nan dat[np.isinf(dat)] = np.nan datm = np.ma.masked_invalid(dat) del dat glon, glat = trans(grid_x, grid_y, inverse=True) del grid_x, grid_y vmin=np.nanmin(datm)+0.1 vmax=np.nanmax(datm)-0.1 if vmin > vmax: vmin=np.nanmin(datm) vmax=np.nanmax(datm) print_map(cs2cs_args, glon, glat, datm, sonpath, p, vmin=vmin, vmax=vmax) else: #just 1 chunk e = esi n = nsi t = theta d = dist_tvg len_n = len(n) merge = class_fp.copy() merge[np.isnan(merge)] = 0 merge[np.isnan(np.vstack((np.flipud(port_fp),star_fp)))] = 0 extent = shape_port[0] R1 = merge[extent:,:] R2 = np.flipud(merge[:extent,:]) merge = np.vstack((R2,R1)) del R1, R2 # get number pixels in scan line extent = int(np.shape(merge)[0]/2) yvec = np.linspace(pix_m,extent*pix_m,extent) X, Y = getXY(e,n,yvec,d,t,extent) merge[merge==0] = np.nan if len(merge.flatten()) != len(X): merge = merge[:,:len_n] merge = merge.T.flatten() index = np.where(np.logical_not(np.isnan(merge)))[0] X, Y, merge = trim_xys(X, Y, merge, index) # write raw bs to file outfile = os.path.normpath(os.path.join(sonpath,'x_y_class'+str(0)+'.asc')) with open(outfile, 'w') as f: np.savetxt(f, np.hstack((humutils.ascol(X),humutils.ascol(Y), humutils.ascol(merge))), delimiter=' ', fmt="%8.6f %8.6f %8.6f") humlon, humlat = trans(X, Y, inverse=True) #if dogrid==1: if 2>1: orig_def, targ_def, grid_x, grid_y, res, shape = get_griddefs(np.min(X), np.max(X), np.min(Y), np.max(Y), res, humlon, humlat, trans) ## create mask for where the data is not tree = KDTree(np.c_[X.flatten(),Y.flatten()]) if pykdtree==1: dist, _ = tree.query(np.c_[grid_x.ravel(), grid_y.ravel()], k=1) else: try: dist, _ = tree.query(np.c_[grid_x.ravel(), grid_y.ravel()], k=1, n_jobs=cpu_count()) except: #print ".... update your scipy installation to use faster kd-tree queries" dist, _ = tree.query(np.c_[grid_x.ravel(), grid_y.ravel()], k=1) dist = dist.reshape(grid_x.shape) sigmas = 1 #m eps = 2 dat, res = get_grid(mode, orig_def, targ_def, merge, res*10, np.min(X), np.max(X), np.min(Y), np.max(Y), res, nn, sigmas, eps, shape, numstdevs, trans, humlon, humlat) del merge #if dogrid==1: if 2>1: dat[dat==0] = np.nan dat[np.isinf(dat)] = np.nan dat[dist>res*2] = np.nan del dist datm = np.ma.masked_invalid(dat) glon, glat = trans(grid_x, grid_y, inverse=True) del grid_x, grid_y vmin=np.nanmin(datm)+0.1 vmax=np.nanmax(datm)-0.1 if vmin > vmax: vmin=np.nanmin(datm) vmax=np.nanmax(datm) Parallel(n_jobs = 2, verbose=0)(delayed(doplots)(k, humlon, humlat, cs2cs_args, glon, glat, datm, sonpath, 0, vmin=vmin, vmax=vmax) for k in xrange(2)) #print_map(cs2cs_args, glon, glat, datm, sonpath, 0, vmin=vmin, vmax=vmax) #print_contour_map(cs2cs_args, humlon, humlat, glon, glat, datm, sonpath, 0, vmin=vmin, vmax=vmax) if os.name=='posix': # true if linux/mac elapsed = (time.time() - start) else: # windows elapsed = (time.clock() - start) print "Processing took ", elapsed , "seconds to analyse" print "Done!"
def sliding_window(a,ws,ss = None,flatten = True): ''' Return a sliding window over a in any number of dimensions ''' if None is ss: # ss was not provided. the windows will not overlap in any direction. ss = ws ws = norm_shape(ws) ss = norm_shape(ss) # convert ws, ss, and a.shape to numpy arrays ws = np.array(ws) ss = np.array(ss) import PyHum.io as io shape_tmp = io.set_mmap_data('', '', 'tmp.dat', 'float32', a) del a a = io.get_mmap_data('', '', 'tmp.dat', 'float32', shape_tmp) shap = np.array(a.shape) try: os.remove('tmp.dat') except: pass # ensure that ws, ss, and a.shape all have the same number of dimensions ls = [len(shap),len(ws),len(ss)] if 1 != len(set(ls)): raise ValueError(\ 'a.shape, ws and ss must all have the same length. They were %s' % str(ls)) # ensure that ws is smaller than a in every dimension if np.any(ws > shap): raise ValueError(\ 'ws cannot be larger than a in any dimension.\ a.shape was %s and ws was %s' % (str(a.shape),str(ws))) # how many slices will there be in each dimension? newshape = norm_shape(((shap - ws) // ss) + 1) # the shape of the strided array will be the number of slices in each dimension # plus the shape of the window (tuple addition) newshape += norm_shape(ws) # the strides tuple will be the array's strides multiplied by step size, plus try: # the array's strides (tuple addition) newstrides = norm_shape(np.array(a.strides) * ss) + a.strides a = ast(a,shape = newshape,strides = newstrides) if not flatten: return a # Collapse strided so that it has one more dimension than the window. I.e., # the new array is a flat list of slices. meat = len(ws) if ws.shape else 0 firstdim = (int(np.product(newshape[:-meat])),) if ws.shape else () dim = firstdim + (newshape[-meat:]) # remove any dimensions with size 1 dim = filter(lambda i : i != 1,dim) return a.reshape(dim), newshape except: from itertools import product print("memory error, windowing using slower method") # For each dimension, create a list of all valid slices slices = [[] for i in range(len(ws))] for i in range(len(ws)): nslices = ((shap[i] - ws[i]) // ss[i]) + 1 for j in range(0,nslices): start = j * ss[i] stop = start + ws[i] slices[i].append(slice(start,stop)) # Get an iterator over all valid n-dimensional slices of the input allslices = product(*slices) # Allocate memory to hold all valid n-dimensional slices nslices = np.product([len(s) for s in slices]) #out = np.ndarray((nslices,) + tuple(ws),dtype = a.dtype) out=[] for i,s in enumerate(allslices): #out[i] = a[s] out.append(a[s]) del a import dask.bag as db tmp = db.from_sequence(out, npartitions=1000) del out return tmp.compute(), newshape
def map_texture(humfile, sonpath, cs2cs_args, res, mode, nn, numstdevs): #influence = 10, ''' Create plots of the texture lengthscale maps made in PyHum.texture module using the algorithm detailed by Buscombe et al. (2015) This textural lengthscale is not a direct measure of grain size. Rather, it is a statistical representation that integrates over many attributes of bed texture, of which grain size is the most important. The technique is a physically based means to identify regions of texture within a sidescan echogram, and could provide a basis for objective, automated riverbed sediment classification. Syntax ---------- [] = PyHum.map_texture(humfile, sonpath, cs2cs_args, res, mode, nn, numstdevs) Parameters ---------- humfile : str path to the .DAT file sonpath : str path where the *.SON files are cs2cs_args : int, *optional* [Default="epsg:26949"] arguments to create coordinates in a projected coordinate system this argument gets given to pyproj to turn wgs84 (lat/lon) coordinates into any projection supported by the proj.4 libraries res : float, *optional* [Default=0.5] grid resolution of output gridded texture map mode: int, *optional* [Default=3] gridding mode. 1 = nearest neighbour 2 = inverse weighted nearest neighbour 3 = Gaussian weighted nearest neighbour nn: int, *optional* [Default=64] number of nearest neighbours for gridding (used if mode > 1) numstdevs: int, *optional* [Default = 4] Threshold number of standard deviations in texture lengthscale per grid cell up to which to accept Returns ------- sonpath+'x_y_class'+str(p)+'.asc' : text file contains the point cloud of easting, northing, and texture lengthscales of the pth chunk sonpath+'class_GroundOverlay'+str(p)+'.kml': kml file contains gridded (or point cloud) texture lengthscale map for importing into google earth of the pth chunk sonpath+'class_map'+str(p)+'.png' : image overlay associated with the kml file sonpath+'class_map_imagery'+str(p)+'.png' : png image file gridded (or point cloud) texture lengthscale map overlain onto an image pulled from esri image server References ---------- .. [1] Buscombe, D., Grams, P.E., and Smith, S.M.C., 2015, Automated riverbed sediment classification using low-cost sidescan sonar. Journal of Hydraulic Engineering 10.1061/(ASCE)HY.1943-7900.0001079, 06015019. ''' # prompt user to supply file if no input file given if not humfile: print('An input file is required!!!!!!') Tk().withdraw() # we don't want a full GUI, so keep the root window from appearing humfile = askopenfilename(filetypes=[("DAT files","*.DAT")]) # prompt user to supply directory if no input sonpath is given if not sonpath: print('A *.SON directory is required!!!!!!') Tk().withdraw() # we don't want a full GUI, so keep the root window from appearing sonpath = askdirectory() # print given arguments to screen and convert data type where necessary if humfile: print('Input file is %s' % (humfile)) if sonpath: print('Sonar file path is %s' % (sonpath)) if cs2cs_args: print('cs2cs arguments are %s' % (cs2cs_args)) if res: res = np.asarray(res,float) print('Gridding resolution: %s' % (str(res))) if mode: mode = int(mode) print('Mode for gridding: %s' % (str(mode))) if nn: nn = int(nn) print('Number of nearest neighbours for gridding: %s' % (str(nn))) #if influence: # influence = int(influence) # print 'Radius of influence for gridding: %s (m)' % (str(influence)) if numstdevs: numstdevs = int(numstdevs) print('Threshold number of standard deviations in texture lengthscale per grid cell up to which to accept: %s' % (str(numstdevs))) # start timer if os.name=='posix': # true if linux/mac or cygwin on windows start = time.time() else: # windows start = time.clock() trans = pyproj.Proj(init=cs2cs_args) # if son path name supplied has no separator at end, put one on if sonpath[-1]!=os.sep: sonpath = sonpath + os.sep base = humfile.split('.DAT') # get base of file name for output base = base[0].split(os.sep)[-1] # remove underscores, negatives and spaces from basename base = humutils.strip_base(base) meta = loadmat(os.path.normpath(os.path.join(sonpath,base+'meta.mat'))) esi = np.squeeze(meta['e']) nsi = np.squeeze(meta['n']) pix_m = np.squeeze(meta['pix_m'])*1.1 dep_m = np.squeeze(meta['dep_m']) c = np.squeeze(meta['c']) #dist_m = np.squeeze(meta['dist_m']) theta = np.squeeze(meta['heading'])/(180/np.pi) # load memory mapped scans shape_port = np.squeeze(meta['shape_port']) if shape_port!='': if os.path.isfile(os.path.normpath(os.path.join(sonpath,base+'_data_port_lar.dat'))): port_fp = io.get_mmap_data(sonpath, base, '_data_port_lar.dat', 'float32', tuple(shape_port)) else: port_fp = io.get_mmap_data(sonpath, base, '_data_port_la.dat', 'float32', tuple(shape_port)) shape_star = np.squeeze(meta['shape_star']) if shape_star!='': if os.path.isfile(os.path.normpath(os.path.join(sonpath,base+'_data_star_lar.dat'))): star_fp = io.get_mmap_data(sonpath, base, '_data_star_lar.dat', 'float32', tuple(shape_star)) else: star_fp = io.get_mmap_data(sonpath, base, '_data_star_la.dat', 'float32', tuple(shape_star)) if len(shape_star)>2: shape = shape_port.copy() shape[1] = shape_port[1] + shape_star[1] class_fp = io.get_mmap_data(sonpath, base, '_data_class.dat', 'float32', tuple(shape)) #with open(os.path.normpath(os.path.join(sonpath,base+'_data_class.dat')), 'r') as ff: # class_fp = np.memmap(ff, dtype='float32', mode='r', shape=tuple(shape)) else: with open(os.path.normpath(os.path.join(sonpath,base+'_data_class.dat')), 'r') as ff: class_fp = np.load(ff) tvg = ((8.5*10**-5)+(3/76923)+((8.5*10**-5)/4))*c dist_tvg = ((np.tan(np.radians(25)))*dep_m)-(tvg) if len(shape_star)>2: for p in range(len(class_fp)): e = esi[shape_port[-1]*p:shape_port[-1]*(p+1)] n = nsi[shape_port[-1]*p:shape_port[-1]*(p+1)] t = theta[shape_port[-1]*p:shape_port[-1]*(p+1)] d = dist_tvg[shape_port[-1]*p:shape_port[-1]*(p+1)] len_n = len(n) merge = class_fp[p].copy() merge[np.isnan(merge)] = 0 merge[np.isnan(np.vstack((np.flipud(port_fp[p]),star_fp[p])))] = 0 extent = shape_port[1] R1 = merge[extent:,:] R2 = np.flipud(merge[:extent,:]) merge = np.vstack((R2,R1)) del R1, R2 # get number pixels in scan line extent = int(np.shape(merge)[0]/2) yvec = np.linspace(pix_m,extent*pix_m,extent) X, Y = getXY(e,n,yvec,d,t,extent) merge[merge==0] = np.nan if len(merge.flatten()) != len(X): merge = merge[:,:len_n] merge = merge.T.flatten() index = np.where(np.logical_not(np.isnan(merge)))[0] X, Y, merge = trim_xys(X, Y, merge, index) X = X.astype('float32') Y = Y.astype('float32') merge = merge.astype('float32') # write raw bs to file outfile = os.path.normpath(os.path.join(sonpath,'x_y_class'+str(p)+'.asc')) with open(outfile, 'w') as f: np.savetxt(f, np.hstack((humutils.ascol(X),humutils.ascol(Y), humutils.ascol(merge))), delimiter=' ', fmt="%8.6f %8.6f %8.6f") humlon, humlat = trans(X, Y, inverse=True) #if dogrid==1: orig_def, targ_def, grid_x, grid_y, res, shape = get_griddefs(np.min(X), np.max(X), np.min(Y), np.max(Y), res, humlon, humlat, trans) grid_x = grid_x.astype('float32') grid_y = grid_y.astype('float32') sigmas = 1 #m eps = 2 dat, res = get_grid(mode, orig_def, targ_def, merge, res*10, np.min(X), np.max(X), np.min(Y), np.max(Y), res, nn, sigmas, eps, shape, numstdevs, trans, humlon, humlat) del merge dat[dat==0] = np.nan dat[np.isinf(dat)] = np.nan datm = np.ma.masked_invalid(dat) del dat glon, glat = trans(grid_x, grid_y, inverse=True) del grid_x, grid_y vmin=np.nanmin(datm)+0.1 vmax=np.nanmax(datm)-0.1 if vmin > vmax: vmin=np.nanmin(datm) vmax=np.nanmax(datm) print_map(cs2cs_args, glon, glat, datm, sonpath, p, vmin=vmin, vmax=vmax) else: #just 1 chunk e = esi n = nsi t = theta d = dist_tvg len_n = len(n) merge = class_fp.copy() merge[np.isnan(merge)] = 0 merge[np.isnan(np.vstack((np.flipud(port_fp),star_fp)))] = 0 extent = shape_port[0] R1 = merge[extent:,:] R2 = np.flipud(merge[:extent,:]) merge = np.vstack((R2,R1)) del R1, R2 # get number pixels in scan line extent = int(np.shape(merge)[0]/2) yvec = np.linspace(pix_m,extent*pix_m,extent) X, Y = getXY(e,n,yvec,d,t,extent) merge[merge==0] = np.nan if len(merge.flatten()) != len(X): merge = merge[:,:len_n] merge = merge.T.flatten() index = np.where(np.logical_not(np.isnan(merge)))[0] X, Y, merge = trim_xys(X, Y, merge, index) # write raw bs to file outfile = os.path.normpath(os.path.join(sonpath,'x_y_class'+str(0)+'.asc')) with open(outfile, 'w') as f: np.savetxt(f, np.hstack((humutils.ascol(X),humutils.ascol(Y), humutils.ascol(merge))), delimiter=' ', fmt="%8.6f %8.6f %8.6f") humlon, humlat = trans(X, Y, inverse=True) #if dogrid==1: if 2>1: orig_def, targ_def, grid_x, grid_y, res, shape = get_griddefs(np.min(X), np.max(X), np.min(Y), np.max(Y), res, humlon, humlat, trans) ## create mask for where the data is not tree = KDTree(np.c_[X.flatten(),Y.flatten()]) if pykdtree==1: dist, _ = tree.query(np.c_[grid_x.ravel(), grid_y.ravel()], k=1) else: try: dist, _ = tree.query(np.c_[grid_x.ravel(), grid_y.ravel()], k=1, n_jobs=cpu_count()) except: #print ".... update your scipy installation to use faster kd-tree queries" dist, _ = tree.query(np.c_[grid_x.ravel(), grid_y.ravel()], k=1) dist = dist.reshape(grid_x.shape) sigmas = 1 #m eps = 2 dat, res = get_grid(mode, orig_def, targ_def, merge, res*10, np.min(X), np.max(X), np.min(Y), np.max(Y), res, nn, sigmas, eps, shape, numstdevs, trans, humlon, humlat) del merge #if dogrid==1: if 2>1: dat[dat==0] = np.nan dat[np.isinf(dat)] = np.nan dat[dist>res*2] = np.nan del dist datm = np.ma.masked_invalid(dat) glon, glat = trans(grid_x, grid_y, inverse=True) del grid_x, grid_y vmin=np.nanmin(datm)+0.1 vmax=np.nanmax(datm)-0.1 if vmin > vmax: vmin=np.nanmin(datm) vmax=np.nanmax(datm) Parallel(n_jobs = 2, verbose=0)(delayed(doplots)(k, humlon, humlat, cs2cs_args, glon, glat, datm, sonpath, 0, vmin=vmin, vmax=vmax) for k in range(2)) #print_map(cs2cs_args, glon, glat, datm, sonpath, 0, vmin=vmin, vmax=vmax) #print_contour_map(cs2cs_args, humlon, humlat, glon, glat, datm, sonpath, 0, vmin=vmin, vmax=vmax) if os.name=='posix': # true if linux/mac elapsed = (time.time() - start) else: # windows elapsed = (time.clock() - start) print("Processing took "+str(elapsed)+"seconds to analyse") print("Done!") print("===================================================")
def correct(humfile, sonpath, maxW, doplot, dofilt, correct_withwater, ph, temp, salinity, dconcfile): ''' Remove water column and carry out some rudimentary radiometric corrections, accounting for directivity and attenuation with range Syntax ---------- [] = PyHum.correct(humfile, sonpath, maxW, doplot, correct_withwater, ph, temp, salinity, dconcfile) Parameters ---------- humfile : str path to the .DAT file sonpath : str path where the *.SON files are maxW : int, *optional* [Default=1000] maximum transducer power doplot : int, *optional* [Default=1] 1 = make plots, otherwise do not dofilt : int, *optional* [Default=0] 1 = apply a phase preserving filter to the scans correct_withwater : int, *optional* [Default=0] 1 = apply radiometric correction but don't remove water column from scans ph : float, *optional* [Default=7.0] water acidity in pH temp : float, *optional* [Default=10.0] water temperature in degrees Celsius salinity : float, *optional* [Default=0.0] salinity of water in parts per thousand dconcfile : str, *optional* [Default=None] file path of a text file containing sediment concentration data this file must contain the following fields separated by spaces: size (microns) conc (mg/L) dens (kg/m3) with one row per grain size, for example: 30 1700 2200 100 15 2650 Returns ------- sonpath+base+'_data_star_l.dat': memory-mapped file contains the starboard scan with water column removed sonpath+base+'_data_port_l.dat': memory-mapped file contains the portside scan with water column removed sonpath+base+'_data_star_la.dat': memory-mapped file contains the starboard scan with water column removed and radiometrically corrected sonpath+base+'_data_port_la.dat': memory-mapped file contains the portside scan with water column removed and radiometrically corrected sonpath+base+'_data_range.dat': memory-mapped file contains the cosine of the range which is used to correct for attenuation with range sonpath+base+'_data_dwnlow_l.dat': memory-mapped file contains the low freq. downward scan with water column removed sonpath+base+'_data_dwnhi_l.dat': memory-mapped file contains the high freq. downward scan with water column removed sonpath+base+'_data_dwnlow_la.dat': memory-mapped file contains the low freq. downward scan with water column removed and radiometrically corrected sonpath+base+'_data_dwnhi_la.dat': memory-mapped file contains the high freq. downward scan with water column removed and radiometrically corrected if correct_withwater == 1: sonpath+base+'_data_star_lw.dat': memory-mapped file contains the starboard scan with water column retained and radiometrically corrected sonpath+base+'_data_port_lw.dat': memory-mapped file contains the portside scan with water column retained and radiometrically corrected ''' # prompt user to supply file if no input file given if not humfile: print('An input file is required!!!!!!') Tk().withdraw() # we don't want a full GUI, so keep the root window from appearing inputfile = askopenfilename(filetypes=[("DAT files","*.DAT")]) # prompt user to supply directory if no input sonpath is given if not sonpath: print('A *.SON directory is required!!!!!!') Tk().withdraw() # we don't want a full GUI, so keep the root window from appearing sonpath = askdirectory() # print given arguments to screen and convert data type where necessary if humfile: print('Input file is %s' % (humfile)) if sonpath: print('Sonar file path is %s' % (sonpath)) if maxW: maxW = np.asarray(maxW,float) print('Max. transducer power is %s W' % (str(maxW))) if doplot: doplot = int(doplot) if doplot==0: print("Plots will not be made") if dofilt: dofilt = int(dofilt) if dofilt==0: print("Phase preserving filter will not be applied") else: print("Phase preserving filter will be applied") if correct_withwater: correct_withwater = int(correct_withwater) if correct_withwater==1: print("Correction will be applied without removing water column") if salinity: salinity = np.asarray(salinity,float) print('Salinity is %s ppt' % (str(salinity))) if ph: ph = np.asarray(ph,float) print('pH is %s' % (str(ph))) if temp: temp = np.asarray(temp,float) print('Temperature is %s' % (str(temp))) if dconcfile is not None: try: print('Suspended sediment size/conc. file is %s' % (dconcfile)) dconc = np.genfromtxt(dconcfile).T conc = dconc[1] dens = dconc[2] d = dconc[0] except: pass #================================ # start timer if os.name=='posix': # true if linux/mac or cygwin on windows start = time.time() else: # windows start = time.clock() # if son path name supplied has no separator at end, put one on if sonpath[-1]!=os.sep: sonpath = sonpath + os.sep base = humfile.split('.DAT') # get base of file name for output base = base[0].split(os.sep)[-1] # remove underscores, negatives and spaces from basename base = humutils.strip_base(base) # add wattage to metadata dict meta = loadmat(os.path.normpath(os.path.join(sonpath,base+'meta.mat'))) dep_m = meta['dep_m'][0] pix_m = meta['pix_m'][0] meta['maxW'] = maxW savemat(os.path.normpath(os.path.join(sonpath,base+'meta.mat')), meta ,oned_as='row') bed = np.squeeze(meta['bed']) ft = 1/(meta['pix_m']) dist_m = np.squeeze(meta['dist_m']) try: if dconcfile is not None: # sediment attenuation alpha = sed_atten(meta['f'],conc,dens,d,meta['c']) else: alpha = 0 except: alpha = 0 # load memory mapped scans shape_port = np.squeeze(meta['shape_port']) if shape_port!='': if os.path.isfile(os.path.normpath(os.path.join(sonpath,base+'_data_port2.dat'))): port_fp = io.get_mmap_data(sonpath, base, '_data_port2.dat', 'int16', tuple(shape_port)) else: port_fp = io.get_mmap_data(sonpath, base, '_data_port.dat', 'int16', tuple(shape_port)) shape_star = np.squeeze(meta['shape_star']) if shape_star!='': if os.path.isfile(os.path.normpath(os.path.join(sonpath,base+'_data_star2.dat'))): star_fp = io.get_mmap_data(sonpath, base, '_data_star2.dat', 'int16', tuple(shape_star)) else: star_fp = io.get_mmap_data(sonpath, base, '_data_star.dat', 'int16', tuple(shape_star)) if len(shape_star)==2: extent = shape_star[0] else: extent = shape_star[1] #np.shape(data_port)[0] bed = np.asarray(bed,'int')+int(0.25*ft) # calculate in dB ######### star Zt, R, A = remove_water(star_fp, bed, shape_star, dep_m, pix_m, 1, maxW) Zt = np.squeeze(Zt) # create memory mapped file for Z) shape_star = io.set_mmap_data(sonpath, base, '_data_star_l.dat', 'float32', Zt) del Zt A = np.squeeze(A) # create memory mapped file for A shape_A = io.set_mmap_data(sonpath, base, '_data_incidentangle.dat', 'float32', A) del A R = np.squeeze(R) R[np.isnan(R)] = 0 try: alpha_w = water_atten(R,meta['f'],meta['c'], ph, temp, salinity) except: alpha_w = 1e-5 # compute transmission losses TL = (40 * np.log10(R) + alpha_w + (2*alpha)*R/1000)/255 del alpha_w # create memory mapped file for R shape_R = io.set_mmap_data(sonpath, base, '_data_range.dat', 'float32', R) del R TL[np.isnan(TL)] = 0 TL[TL<0] = 0 shape_TL = io.set_mmap_data(sonpath, base, '_data_TL.dat', 'float32', TL) del TL A_fp = io.get_mmap_data(sonpath, base, '_data_incidentangle.dat', 'float32', shape_star) TL_fp = io.get_mmap_data(sonpath, base, '_data_TL.dat', 'float32', shape_star) R_fp = io.get_mmap_data(sonpath, base, '_data_range.dat', 'float32', shape_star) if correct_withwater == 1: Zt = correct_scans(star_fp, A_fp, TL_fp, dofilt) # create memory mapped file for Z) shape_star = io.set_mmap_data(sonpath, base, '_data_star_lw.dat', 'float32', Zt) #we are only going to access the portion of memory required star_fp = io.get_mmap_data(sonpath, base, '_data_star_l.dat', 'float32', shape_star) ##Zt = correct_scans(star_fp, A_fp, TL_fp, dofilt) #phi=1.69 alpha=59 # vertical beam width at 3db theta=35 #opening angle theta # lambertian correction Zt = correct_scans_lambertian(star_fp, A_fp, TL_fp, R_fp, meta['c'], meta['f'], theta, alpha) Zt = np.squeeze(Zt) avg = np.nanmedian(Zt,axis=0) ##avg = median_filter(avg,int(len(avg)/10)) Zt2 = Zt-avg + np.nanmean(avg) Zt2 = Zt2 + np.abs(np.nanmin(Zt2)) try: Zt2 = median_filter(Zt2, (3,3)) except: pass ##Zt2 = np.empty(np.shape(Zt)) ##for kk in range(np.shape(Zt)[1]): ## Zt2[:,kk] = (Zt[:,kk] - avg) + np.nanmean(avg) ##Zt2[Zt<=0] = np.nan ##Zt2[Zt2<=0] = np.nan del Zt # create memory mapped file for Z shape_star = io.set_mmap_data(sonpath, base, '_data_star_la.dat', 'float32', Zt2) del Zt2 #we are only going to access the portion of memory required star_fp = io.get_mmap_data(sonpath, base, '_data_star_la.dat', 'float32', shape_star) ######### port if correct_withwater == 1: Zt = correct_scans(port_fp, A_fp, TL, dofilt) # create memory mapped file for Z) shape_port = io.set_mmap_data(sonpath, base, '_data_port_lw.dat', 'float32', Zt) Zt = remove_water(port_fp, bed, shape_port, dep_m, pix_m, 0, maxW) Zt = np.squeeze(Zt) # create memory mapped file for Z shape_port = io.set_mmap_data(sonpath, base, '_data_port_l.dat', 'float32', Zt) #we are only going to access the portion of memory required port_fp = io.get_mmap_data(sonpath, base, '_data_port_l.dat', 'float32', shape_port) ##Zt = correct_scans(port_fp, A_fp, TL_fp, dofilt) # lambertian correction Zt = correct_scans_lambertian(port_fp, A_fp, TL_fp, R_fp, meta['c'], meta['f'], theta, alpha) Zt = np.squeeze(Zt) Zt2 = Zt-avg + np.nanmean(avg) Zt2 = Zt2 + np.abs(np.nanmin(Zt2)) ##Zt2 = np.empty(np.shape(Zt)) ##for kk in range(np.shape(Zt)[1]): ## Zt2[:,kk] = (Zt[:,kk] - avg) + np.nanmean(avg) ##Zt2[Zt<=0] = np.nan ##Zt2[Zt2<=0] = np.nan del Zt # create memory mapped file for Z shape_port = io.set_mmap_data(sonpath, base, '_data_port_la.dat', 'float32', Zt2) del Zt2 #we are only going to access the portion of memory required port_fp = io.get_mmap_data(sonpath, base, '_data_port_la.dat', 'float32', shape_port) ## do plots of merged scans if doplot==1: if correct_withwater == 1: port_fpw = io.get_mmap_data(sonpath, base, '_data_port_lw.dat', 'float32', shape_port) star_fpw = io.get_mmap_data(sonpath, base, '_data_star_lw.dat', 'float32', shape_star) if len(np.shape(star_fpw))>2: for p in range(len(star_fpw)): plot_merged_scans(port_fpw[p], star_fpw[p], dist_m, shape_port, ft, sonpath, p) else: plot_merged_scans(port_fpw, star_fpw, dist_m, shape_port, ft, sonpath, 0) else: if len(np.shape(star_fp))>2: for p in range(len(star_fp)): plot_merged_scans(port_fp[p], star_fp[p], dist_m, shape_port, ft, sonpath, p) else: plot_merged_scans(port_fp, star_fp, dist_m, shape_port, ft, sonpath, 0) # load memory mapped scans shape_low = np.squeeze(meta['shape_low']) shape_hi = np.squeeze(meta['shape_hi']) if shape_low!='': if os.path.isfile(os.path.normpath(os.path.join(sonpath,base+'_data_dwnlow2.dat'))): try: low_fp = io.get_mmap_data(sonpath, base, '_data_dwnlow2.dat', 'int16', tuple(shape_low)) except: low_fp = io.get_mmap_data(sonpath, base, '_data_dwnlow.dat', 'int16', tuple(shape_low)) finally: low_fp = io.get_mmap_data(sonpath, base, '_data_dwnlow.dat', 'int16', tuple(shape_hi)) #if 'shape_hi' in locals(): # low_fp = io.get_mmap_data(sonpath, base, '_data_dwnlow2.dat', 'int16', tuple(shape_hi)) else: try: low_fp = io.get_mmap_data(sonpath, base, '_data_dwnlow.dat', 'int16', tuple(shape_low)) except: if 'shape_hi' in locals(): low_fp = io.get_mmap_data(sonpath, base, '_data_dwnlow.dat', 'int16', tuple(shape_hi)) shape_hi = np.squeeze(meta['shape_hi']) if shape_hi!='': if os.path.isfile(os.path.normpath(os.path.join(sonpath,base+'_data_dwnhi2.dat'))): try: hi_fp = io.get_mmap_data(sonpath, base, '_data_dwnhi2.dat', 'int16', tuple(shape_hi)) except: hi_fp = io.get_mmap_data(sonpath, base, '_data_dwnhi.dat', 'int16', tuple(shape_hi)) finally: hi_fp = io.get_mmap_data(sonpath, base, '_data_dwnhi.dat', 'int16', tuple(shape_low)) #if 'shape_low' in locals(): # hi_fp = io.get_mmap_data(sonpath, base, '_data_dwnhi2.dat', 'int16', tuple(shape_low)) else: try: hi_fp = io.get_mmap_data(sonpath, base, '_data_dwnhi.dat', 'int16', tuple(shape_hi)) except: if 'shape_low' in locals(): hi_fp = io.get_mmap_data(sonpath, base, '_data_dwnhi.dat', 'int16', tuple(shape_low)) if 'low_fp' in locals(): ######### low Zt = remove_water(low_fp, bed, shape_low, dep_m, pix_m, 0, maxW) Zt = np.squeeze(Zt) # create memory mapped file for Z shape_low = io.set_mmap_data(sonpath, base, '_data_dwnlow_l.dat', 'float32', Zt) del Zt #we are only going to access the portion of memory required low_fp = io.get_mmap_data(sonpath, base, '_data_dwnlow_l.dat', 'float32', shape_low) Zt = correct_scans2(low_fp, TL_fp) # create memory mapped file for Z shape_low = io.set_mmap_data(sonpath, base, '_data_dwnlow_la.dat', 'float32', Zt) del Zt #we are only going to access the lowion of memory required low_fp = io.get_mmap_data(sonpath, base, '_data_dwnlow_la.dat', 'float32', shape_low) if doplot==1: if len(np.shape(low_fp))>2: for p in range(len(low_fp)): plot_dwnlow_scans(low_fp[p], dist_m, shape_low, ft, sonpath, p) else: plot_dwnlow_scans(low_fp, dist_m, shape_low, ft, sonpath, 0) if 'hi_fp' in locals(): ######### hi Zt = remove_water(hi_fp, bed, shape_hi, dep_m, pix_m, 0, maxW) Zt = np.squeeze(Zt) # create memory mapped file for Z shape_hi = io.set_mmap_data(sonpath, base, '_data_dwnhi_l.dat', 'float32', Zt) del Zt #we are only going to access the portion of memory required hi_fp = io.get_mmap_data(sonpath, base, '_data_dwnhi_l.dat', 'float32', shape_hi) Zt = correct_scans2(hi_fp, TL_fp) # create memory mapped file for Z shape_hi = io.set_mmap_data(sonpath, base, '_data_dwnhi_la.dat', 'float32', Zt) del Zt #we are only going to access the hiion of memory required hi_fp = io.get_mmap_data(sonpath, base, '_data_dwnhi_la.dat', 'float32', shape_hi) if doplot==1: if len(np.shape(hi_fp))>2: for p in range(len(hi_fp)): plot_dwnhi_scans(hi_fp[p], dist_m, shape_hi, ft, sonpath, p) else: plot_dwnhi_scans(hi_fp, dist_m, shape_hi, ft, sonpath, 0) if os.name=='posix': # true if linux/mac elapsed = (time.time() - start) else: # windows elapsed = (time.clock() - start) print("Processing took "+ str(elapsed) + "seconds to analyse") print("Done!") print("===================================================")
def texture(humfile, sonpath, win, shift, doplot, density, numclasses, maxscale, notes): ''' Create a texture lengthscale map using the algorithm detailed by Buscombe et al. (2015) This textural lengthscale is not a direct measure of grain size. Rather, it is a statistical representation that integrates over many attributes of bed texture, of which grain size is the most important. The technique is a physically based means to identify regions of texture within a sidescan echogram, and could provide a basis for objective, automated riverbed sediment classification. Syntax ---------- [] = PyHum.texture(humfile, sonpath, win, shift, doplot, density, numclasses, maxscale, notes) Parameters ---------- humfile : str path to the .DAT file sonpath : str path where the *.SON files are win : int, *optional* [Default=100] pixel in pixels of the moving window shift : int, *optional* [Default=10] shift in pixels for moving window operation doplot : int, *optional* [Default=1] if 1, make plots, otherwise do not make plots density : int, *optional* [Default=win/2] echogram will be sampled every 'density' pixels numclasses : int, *optional* [Default=4] number of 'k means' that the texture lengthscale will be segmented into maxscale : int, *optional* [Default=20] Max scale as inverse fraction of data length for wavelet analysis notes : int, *optional* [Default=100] notes per octave for wavelet analysis Returns ------- sonpath+base+'_data_class.dat': memory-mapped file contains the texture lengthscale map sonpath+base+'_data_kclass.dat': memory-mapped file contains the k-means segmented texture lengthscale map References ---------- .. [1] Buscombe, D., Grams, P.E., and Smith, S.M.C., 2015, Automated riverbed sediment classification using low-cost sidescan sonar. Journal of Hydraulic Engineering 10.1061/(ASCE)HY.1943-7900.0001079, 06015019. ''' # prompt user to supply file if no input file given if not humfile: print('An input file is required!!!!!!') Tk().withdraw() # we don't want a full GUI, so keep the root window from appearing humfile = askopenfilename(filetypes=[("DAT files","*.DAT")]) # prompt user to supply directory if no input sonpath is given if not sonpath: print('A *.SON directory is required!!!!!!') Tk().withdraw() # we don't want a full GUI, so keep the root window from appearing sonpath = askdirectory() # print given arguments to screen and convert data type where necessary if humfile: print('Input file is %s' % (humfile)) if sonpath: print('Sonar file path is %s' % (sonpath)) if win: win = np.asarray(win,int) print('Window is %s square pixels' % (str(win))) if shift: shift = np.asarray(shift,int) print('Min shift is %s pixels' % (str(shift))) if density: density = np.asarray(density,int) print('Image will be sampled every %s pixels' % (str(density))) if numclasses: numclasses = np.asarray(numclasses,int) print('Number of sediment classes: %s' % (str(numclasses))) if maxscale: maxscale = np.asarray(maxscale,int) print('Max scale as inverse fraction of data length: %s' % (str(maxscale))) if notes: notes = np.asarray(notes,int) print('Notes per octave: %s' % (str(notes))) if doplot: doplot = int(doplot) if doplot==0: print("Plots will not be made") print('[Default] Number of processors is %s' % (str(cpu_count()))) ######################################################## ######################################################## # start timer if os.name=='posix': # true if linux/mac or cygwin on windows start = time.time() else: # windows start = time.clock() # if son path name supplied has no separator at end, put one on if sonpath[-1]!=os.sep: sonpath = sonpath + os.sep base = humfile.split('.DAT') # get base of file name for output base = base[0].split(os.sep)[-1] # remove underscores, negatives and spaces from basename base = humutils.strip_base(base) meta = loadmat(os.path.normpath(os.path.join(sonpath,base+'meta.mat'))) ft = 1/loadmat(sonpath+base+'meta.mat')['pix_m'] #pix_m = np.squeeze(meta['pix_m']) #dep_m = np.squeeze(meta['dep_m']) dist_m = np.squeeze(meta['dist_m']) ### port print("processing port side ...") # load memory mapped scan ... port shape_port = np.squeeze(meta['shape_port']) if shape_port!='': if os.path.isfile(os.path.normpath(os.path.join(sonpath,base+'_data_port_lar.dat'))): port_fp = io.get_mmap_data(sonpath, base, '_data_port_lar.dat', 'float32', tuple(shape_port)) else: port_fp = io.get_mmap_data(sonpath, base, '_data_port_la.dat', 'float32', tuple(shape_port)) port_fp2 = io.get_mmap_data(sonpath, base, '_data_port_l.dat', 'float32', tuple(shape_port)) ### star print("processing starboard side ...") # load memory mapped scan ... port shape_star = np.squeeze(loadmat(sonpath+base+'meta.mat')['shape_star']) if shape_star!='': if os.path.isfile(os.path.normpath(os.path.join(sonpath,base+'_data_star_lar.dat'))): star_fp = io.get_mmap_data(sonpath, base, '_data_star_lar.dat', 'float32', tuple(shape_star)) else: star_fp = io.get_mmap_data(sonpath, base, '_data_star_la.dat', 'float32', tuple(shape_star)) star_fp2 = io.get_mmap_data(sonpath, base, '_data_star_l.dat', 'float32', tuple(shape_star)) if len(shape_star)>2: shape = shape_port.copy() shape[1] = shape_port[1] + shape_star[1] else: shape = [] shape.append(1) shape.append(shape_port[0]) shape.append(shape_port[1]) shape[1] = shape_port[0] + shape_star[0] # create memory mapped file for Sp with open(os.path.normpath(os.path.join(sonpath,base+'_data_class.dat')), 'w+') as ff: fp = np.memmap(ff, dtype='float32', mode='w+', shape=tuple(shape)) if len(shape_star)>2: #SRT = [] for p in range(len(port_fp)): Z,ind = humutils.sliding_window_sliced(np.vstack((np.flipud(port_fp[p]), star_fp[p])), density, (win,win),(shift,shift)) Snn = get_srt(Z,ind,maxscale, notes, win) #, density) # replace nans using infilling algorithm rn = replace_nans.RN(Snn.astype('float64'),1000,0.01,2,'localmean') Snn = rn.getdata() del rn Ny, Nx = np.shape( np.vstack((np.flipud(port_fp[p]), star_fp[p])) ) Snn = median_filter(Snn,(int(Nx/100),int(Ny/100))) Sp = humutils.im_resize(Snn,Nx,Ny) del Snn Sp[np.isnan(np.vstack((np.flipud(port_fp[p]), star_fp[p])))] = np.nan Sp[np.isnan(np.vstack((np.flipud(port_fp2[p]), star_fp2[p])))] = np.nan #extent = shape_port[1] #Zdist = dist_m[shape_port[-1]*p:shape_port[-1]*(p+1)] #yvec = np.linspace(pix_m,extent*pix_m,extent) #d = dep_m[shape_port[-1]*p:shape_port[-1]*(p+1)] R_fp = io.get_mmap_data(sonpath, base, '_data_range.dat', 'float32', tuple(shape_star)) R = np.vstack((np.flipud(R_fp[0]),R_fp[0])) R = R/np.max(R) #R[R>0.8] = np.nan rn = replace_nans.RN(R.astype('float64'),1000,0.01,2,'localmean') R = rn.getdata() del rn Sp = (Sp**2) * np.cos(np.deg2rad(R)) /shift ##**2 fp[p] = Sp.astype('float32') del Sp del fp # flush data to file class_fp = io.get_mmap_data(sonpath, base, '_data_class.dat', 'float32', tuple(shape)) else: Z,ind = humutils.sliding_window_sliced(np.vstack((np.flipud(port_fp), star_fp)), density, (win,win),(shift,shift)) Snn = get_srt(Z,ind,maxscale, notes, win) #, density) # replace nans using infilling algorithm rn = replace_nans.RN(Snn.astype('float64'),1000,0.01,2,'localmean') Snn = rn.getdata() del rn Ny, Nx = np.shape( np.vstack((np.flipud(port_fp), star_fp)) ) Snn = median_filter(Snn,(int(Nx/100),int(Ny/100))) Sp = humutils.im_resize(Snn,Nx,Ny) del Snn Sp[np.isnan(np.vstack((np.flipud(port_fp), star_fp)))] = np.nan Sp[np.isnan(np.vstack((np.flipud(port_fp2), star_fp2)))] = np.nan #extent = shape_port[0] #Zdist = dist_m #yvec = np.linspace(pix_m,extent*pix_m,extent) #d = dep_m R_fp = io.get_mmap_data(sonpath, base, '_data_range.dat', 'float32', tuple(shape_star)) R = np.vstack((np.flipud(R_fp),R_fp)) R = R/np.max(R) #R[R>0.8] = np.nan rn = replace_nans.RN(R.astype('float64'),1000,0.01,2,'localmean') R = rn.getdata() del rn Sp = (Sp**2) * np.cos(np.deg2rad(R)) / shift ##**2 with open(os.path.normpath(os.path.join(sonpath,base+'_data_class.dat')), 'w+') as ff: np.save(ff, np.squeeze(Sp).astype('float32')) with open(os.path.normpath(os.path.join(sonpath,base+'_data_class.dat')), 'r') as ff: class_fp = np.load(ff) del Sp dist_m = np.squeeze(loadmat(sonpath+base+'meta.mat')['dist_m']) ######################################################## if doplot==1: if len(shape_star)>2: for p in range(len(star_fp)): plot_class(dist_m, shape_port, port_fp[p], star_fp[p], class_fp[p], ft, humfile, sonpath, base, p) else: plot_class(dist_m, shape_port, port_fp, star_fp, class_fp, ft, humfile, sonpath, base, 0) if len(shape_star)>2: for p in range(len(star_fp)): plot_contours(dist_m, shape_port, class_fp[p], ft, humfile, sonpath, base, numclasses, p) else: plot_contours(dist_m, shape_port, class_fp, ft, humfile, sonpath, base, numclasses, 0) ####################################################### # k-means if len(shape_star)>2: with open(os.path.normpath(os.path.join(sonpath,base+'_data_kclass.dat')), 'w+') as ff: fp = np.memmap(ff, dtype='float32', mode='w+', shape=tuple(shape)) for p in range(len(port_fp)): wc = get_kclass(class_fp[p].copy(), numclasses) fp[p] = wc.astype('float32') del wc del fp kclass_fp = io.get_mmap_data(sonpath, base, '_data_kclass.dat', 'float32', tuple(shape)) else: wc = get_kclass(class_fp.copy(), numclasses) with open(os.path.normpath(os.path.join(sonpath,base+'_data_kclass.dat')), 'w+') as ff: np.save(ff, np.squeeze(wc).astype('float32')) del wc with open(os.path.normpath(os.path.join(sonpath,base+'_data_kclass.dat')), 'r') as ff: kclass_fp = np.load(ff) ######################################################## if doplot==1: if len(shape_star)>2: for p in range(len(star_fp)): plot_kmeans(dist_m, shape_port, port_fp[p], star_fp[p], kclass_fp[p], ft, humfile, sonpath, base, p) else: plot_kmeans(dist_m, shape_port, port_fp, star_fp, kclass_fp, ft, humfile, sonpath, base, 0) if os.name=='posix': # true if linux/mac elapsed = (time.time() - start) else: # windows elapsed = (time.clock() - start) print("Processing took "+ str(elapsed)+ "seconds to analyse") print("Done!") print("===================================================")
def texture_slic(humfile, sonpath, doplot=1, numclasses=4, maxscale=20, notes=4): ''' Create a texture lengthscale map using the algorithm detailed by Buscombe et al. (2015) This textural lengthscale is not a direct measure of grain size. Rather, it is a statistical representation that integrates over many attributes of bed texture, of which grain size is the most important. The technique is a physically based means to identify regions of texture within a sidescan echogram, and could provide a basis for objective, automated riverbed sediment classification. Syntax ---------- [] = PyHum.texture(humfile, sonpath, doplot, numclasses, maxscale, notes) Parameters ---------- humfile : str path to the .DAT file sonpath : str path where the *.SON files are doplot : int, *optional* [Default=1] if 1, make plots, otherwise do not make plots numclasses : int, *optional* [Default=4] number of 'k means' that the texture lengthscale will be segmented into maxscale : int, *optional* [Default=20] Max scale as inverse fraction of data length for wavelet analysis notes : int, *optional* [Default=100] notes per octave for wavelet analysis Returns ------- sonpath+base+'_data_class.dat': memory-mapped file contains the texture lengthscale map sonpath+base+'_data_kclass.dat': memory-mapped file contains the k-means segmented texture lengthscale map References ---------- .. [1] Buscombe, D., Grams, P.E., and Smith, S.M.C., 2015, Automated riverbed sediment classification using low-cost sidescan sonar. Journal of Hydraulic Engineering 10.1061/(ASCE)HY.1943-7900.0001079, 06015019. ''' # prompt user to supply file if no input file given if not humfile: print('An input file is required!!!!!!') Tk().withdraw( ) # we don't want a full GUI, so keep the root window from appearing humfile = askopenfilename(filetypes=[("DAT files", "*.DAT")]) # prompt user to supply directory if no input sonpath is given if not sonpath: print('A *.SON directory is required!!!!!!') Tk().withdraw( ) # we don't want a full GUI, so keep the root window from appearing sonpath = askdirectory() # print given arguments to screen and convert data type where necessary if humfile: print('Input file is %s' % (humfile)) if sonpath: print('Sonar file path is %s' % (sonpath)) if numclasses: numclasses = np.asarray(numclasses, int) print('Number of sediment classes: %s' % (str(numclasses))) if maxscale: maxscale = np.asarray(maxscale, int) print('Max scale as inverse fraction of data length: %s' % (str(maxscale))) if notes: notes = np.asarray(notes, int) print('Notes per octave: %s' % (str(notes))) if doplot: doplot = int(doplot) if doplot == 0: print("Plots will not be made") print('[Default] Number of processors is %s' % (str(cpu_count()))) ######################################################## ######################################################## # start timer if os.name == 'posix': # true if linux/mac or cygwin on windows start = time.time() else: # windows start = time.clock() # if son path name supplied has no separator at end, put one on if sonpath[-1] != os.sep: sonpath = sonpath + os.sep base = humfile.split('.DAT') # get base of file name for output base = base[0].split(os.sep)[-1] # remove underscores, negatives and spaces from basename base = humutils.strip_base(base) meta = loadmat(os.path.normpath(os.path.join(sonpath, base + 'meta.mat'))) ft = 1 / loadmat(sonpath + base + 'meta.mat')['pix_m'] #pix_m = np.squeeze(meta['pix_m']) #dep_m = np.squeeze(meta['dep_m']) dist_m = np.squeeze(meta['dist_m']) ### port print("processing port side ...") # load memory mapped scan ... port shape_port = np.squeeze(meta['shape_port']) if shape_port != '': if os.path.isfile( os.path.normpath( os.path.join(sonpath, base + '_data_port_lar.dat'))): port_fp = io.get_mmap_data(sonpath, base, '_data_port_lar.dat', 'float32', tuple(shape_port)) else: port_fp = io.get_mmap_data(sonpath, base, '_data_port_la.dat', 'float32', tuple(shape_port)) #port_fp2 = io.get_mmap_data(sonpath, base, '_data_port_l.dat', 'float32', tuple(shape_port)) ### star print("processing starboard side ...") # load memory mapped scan ... port shape_star = np.squeeze(loadmat(sonpath + base + 'meta.mat')['shape_star']) if shape_star != '': if os.path.isfile( os.path.normpath( os.path.join(sonpath, base + '_data_star_lar.dat'))): star_fp = io.get_mmap_data(sonpath, base, '_data_star_lar.dat', 'float32', tuple(shape_star)) else: star_fp = io.get_mmap_data(sonpath, base, '_data_star_la.dat', 'float32', tuple(shape_star)) #star_fp2 = io.get_mmap_data(sonpath, base, '_data_star_l.dat', 'float32', tuple(shape_star)) if len(shape_star) > 2: shape = shape_port.copy() shape[1] = shape_port[1] + shape_star[1] else: shape = [] shape.append(1) shape.append(shape_port[0]) shape.append(shape_port[1]) shape[1] = shape_port[0] + shape_star[0] #work on the entire scan #im = humutils.rescale(np.vstack((np.flipud(np.hstack(port_fp)), np.hstack(star_fp))),0,1) im = np.vstack((np.flipud(np.hstack(port_fp)), np.hstack(star_fp))) im[np.isnan(im)] = 0 im = humutils.rescale(im, 0, 1) #get SLIC superpixels segments_slic = slic(im, n_segments=int(im.shape[0] / 10), compactness=.1) #pre-allocate texture lengthscale array tl = np.zeros(im.shape, dtype="float64") #cycle through each segment and compute tl for k in np.unique(segments_slic): mask = np.zeros(im.shape[:2], dtype="uint8") mask[segments_slic == k] = 255 cmask, cim = crop_toseg(mask, im) tl[segments_slic == k] = parallel_me(cim, maxscale, notes, np.shape(cim)[0]) R_fp = io.get_mmap_data(sonpath, base, '_data_range.dat', 'float32', tuple(shape_star)) R = np.vstack((np.flipud(np.hstack(R_fp)), np.hstack(R_fp))) R = R / np.max(R) #correct for range and scale tl = tl * np.cos(R) * (1 / ft) tl[im == 0] = np.nan tl[np.isnan(im)] = np.nan # create memory mapped file for Sp with open( os.path.normpath(os.path.join(sonpath, base + '_data_class.dat')), 'w+') as ff: fp = np.memmap(ff, dtype='float32', mode='w+', shape=tuple(shape)) counter = 0 if len(shape_star) > 2: for p in range(len(port_fp)): if p == 0: n, m = np.shape(np.vstack((np.flipud(port_fp[p]), star_fp[p]))) else: n, m = np.shape(np.vstack((np.flipud(port_fp[p]), star_fp[p]))) Sp = tl[:n, counter:counter + m] counter = counter + m fp[p] = Sp.astype('float32') del Sp del fp # flush data to file class_fp = io.get_mmap_data(sonpath, base, '_data_class.dat', 'float32', tuple(shape)) else: with open( os.path.normpath( os.path.join(sonpath, base + '_data_class.dat')), 'w+') as ff: np.save(ff, np.squeeze(Sp).astype('float32')) with open( os.path.normpath( os.path.join(sonpath, base + '_data_class.dat')), 'r') as ff: class_fp = np.load(ff) dist_m = np.squeeze(loadmat(sonpath + base + 'meta.mat')['dist_m']) ######################################################## if doplot == 1: if len(shape_star) > 2: for p in range(len(star_fp)): plot_class(dist_m, shape_port, port_fp[p], star_fp[p], class_fp[p], ft, humfile, sonpath, base, p) else: plot_class(dist_m, shape_port, port_fp, star_fp, class_fp, ft, humfile, sonpath, base, 0) if len(shape_star) > 2: for p in range(len(star_fp)): plot_contours(dist_m, shape_port, class_fp[p], ft, humfile, sonpath, base, numclasses, p) else: plot_contours(dist_m, shape_port, class_fp, ft, humfile, sonpath, base, numclasses, 0) ####################################################### # k-means if len(shape_star) > 2: with open( os.path.normpath( os.path.join(sonpath, base + '_data_kclass.dat')), 'w+') as ff: fp = np.memmap(ff, dtype='float32', mode='w+', shape=tuple(shape)) for p in range(len(port_fp)): wc = get_kclass(class_fp[p].copy(), numclasses) fp[p] = wc.astype('float32') del wc del fp kclass_fp = io.get_mmap_data(sonpath, base, '_data_kclass.dat', 'float32', tuple(shape)) else: wc = get_kclass(class_fp.copy(), numclasses) with open( os.path.normpath( os.path.join(sonpath, base + '_data_kclass.dat')), 'w+') as ff: np.save(ff, np.squeeze(wc).astype('float32')) del wc with open( os.path.normpath( os.path.join(sonpath, base + '_data_kclass.dat')), 'r') as ff: kclass_fp = np.load(ff) ######################################################## if doplot == 1: if len(shape_star) > 2: for p in range(len(star_fp)): plot_kmeans(dist_m, shape_port, port_fp[p], star_fp[p], kclass_fp[p], ft, humfile, sonpath, base, p) else: plot_kmeans(dist_m, shape_port, port_fp, star_fp, kclass_fp, ft, humfile, sonpath, base, 0) if os.name == 'posix': # true if linux/mac elapsed = (time.time() - start) else: # windows elapsed = (time.clock() - start) print("Processing took " + str(elapsed) + "seconds to analyse") print("Done!")
# remove underscores, negatives and spaces from basename base = humutils.strip_base(base) meta = loadmat(os.path.normpath(os.path.join(sonpath, base + 'meta.mat'))) ### port print "processing port side ..." # load memory mapped scan ... port shape_port = np.squeeze(meta['shape_port']) if shape_port != '': if os.path.isfile( os.path.normpath( os.path.join(sonpath, base + '_data_port_lar.dat'))): port_fp = io.get_mmap_data(sonpath, base, '_data_port_lar.dat', 'float32', tuple(shape_port)) else: port_fp = io.get_mmap_data(sonpath, base, '_data_port_la.dat', 'float32', tuple(shape_port)) ### star print "processing starboard side ..." # load memory mapped scan ... port shape_star = np.squeeze(loadmat(sonpath + base + 'meta.mat')['shape_star']) if shape_star != '': if os.path.isfile( os.path.normpath( os.path.join(sonpath, base + '_data_star_lar.dat'))): star_fp = io.get_mmap_data(sonpath, base, '_data_star_lar.dat', 'float32', tuple(shape_star)) else:
def texture_slic(humfile, sonpath, doplot=1, numclasses=4, maxscale=20, notes=4): ''' Create a texture lengthscale map using the algorithm detailed by Buscombe et al. (2015) This textural lengthscale is not a direct measure of grain size. Rather, it is a statistical representation that integrates over many attributes of bed texture, of which grain size is the most important. The technique is a physically based means to identify regions of texture within a sidescan echogram, and could provide a basis for objective, automated riverbed sediment classification. Syntax ---------- [] = PyHum.texture(humfile, sonpath, doplot, numclasses, maxscale, notes) Parameters ---------- humfile : str path to the .DAT file sonpath : str path where the *.SON files are doplot : int, *optional* [Default=1] if 1, make plots, otherwise do not make plots numclasses : int, *optional* [Default=4] number of 'k means' that the texture lengthscale will be segmented into maxscale : int, *optional* [Default=20] Max scale as inverse fraction of data length for wavelet analysis notes : int, *optional* [Default=100] notes per octave for wavelet analysis Returns ------- sonpath+base+'_data_class.dat': memory-mapped file contains the texture lengthscale map sonpath+base+'_data_kclass.dat': memory-mapped file contains the k-means segmented texture lengthscale map References ---------- .. [1] Buscombe, D., Grams, P.E., and Smith, S.M.C., 2015, Automated riverbed sediment classification using low-cost sidescan sonar. Journal of Hydraulic Engineering 10.1061/(ASCE)HY.1943-7900.0001079, 06015019. ''' # prompt user to supply file if no input file given if not humfile: print('An input file is required!!!!!!') Tk().withdraw() # we don't want a full GUI, so keep the root window from appearing humfile = askopenfilename(filetypes=[("DAT files","*.DAT")]) # prompt user to supply directory if no input sonpath is given if not sonpath: print('A *.SON directory is required!!!!!!') Tk().withdraw() # we don't want a full GUI, so keep the root window from appearing sonpath = askdirectory() # print given arguments to screen and convert data type where necessary if humfile: print('Input file is %s' % (humfile)) if sonpath: print('Sonar file path is %s' % (sonpath)) if numclasses: numclasses = np.asarray(numclasses,int) print('Number of sediment classes: %s' % (str(numclasses))) if maxscale: maxscale = np.asarray(maxscale,int) print('Max scale as inverse fraction of data length: %s' % (str(maxscale))) if notes: notes = np.asarray(notes,int) print('Notes per octave: %s' % (str(notes))) if doplot: doplot = int(doplot) if doplot==0: print("Plots will not be made") print('[Default] Number of processors is %s' % (str(cpu_count()))) ######################################################## ######################################################## # start timer if os.name=='posix': # true if linux/mac or cygwin on windows start = time.time() else: # windows start = time.clock() # if son path name supplied has no separator at end, put one on if sonpath[-1]!=os.sep: sonpath = sonpath + os.sep base = humfile.split('.DAT') # get base of file name for output base = base[0].split(os.sep)[-1] # remove underscores, negatives and spaces from basename base = humutils.strip_base(base) meta = loadmat(os.path.normpath(os.path.join(sonpath,base+'meta.mat'))) ft = 1/loadmat(sonpath+base+'meta.mat')['pix_m'] #pix_m = np.squeeze(meta['pix_m']) #dep_m = np.squeeze(meta['dep_m']) dist_m = np.squeeze(meta['dist_m']) ### port print("processing port side ...") # load memory mapped scan ... port shape_port = np.squeeze(meta['shape_port']) if shape_port!='': if os.path.isfile(os.path.normpath(os.path.join(sonpath,base+'_data_port_lar.dat'))): port_fp = io.get_mmap_data(sonpath, base, '_data_port_lar.dat', 'float32', tuple(shape_port)) else: port_fp = io.get_mmap_data(sonpath, base, '_data_port_la.dat', 'float32', tuple(shape_port)) #port_fp2 = io.get_mmap_data(sonpath, base, '_data_port_l.dat', 'float32', tuple(shape_port)) ### star print("processing starboard side ...") # load memory mapped scan ... port shape_star = np.squeeze(loadmat(sonpath+base+'meta.mat')['shape_star']) if shape_star!='': if os.path.isfile(os.path.normpath(os.path.join(sonpath,base+'_data_star_lar.dat'))): star_fp = io.get_mmap_data(sonpath, base, '_data_star_lar.dat', 'float32', tuple(shape_star)) else: star_fp = io.get_mmap_data(sonpath, base, '_data_star_la.dat', 'float32', tuple(shape_star)) #star_fp2 = io.get_mmap_data(sonpath, base, '_data_star_l.dat', 'float32', tuple(shape_star)) if len(shape_star)>2: shape = shape_port.copy() shape[1] = shape_port[1] + shape_star[1] else: shape = [] shape.append(1) shape.append(shape_port[0]) shape.append(shape_port[1]) shape[1] = shape_port[0] + shape_star[0] #work on the entire scan #im = humutils.rescale(np.vstack((np.flipud(np.hstack(port_fp)), np.hstack(star_fp))),0,1) im = np.vstack((np.flipud(np.hstack(port_fp)), np.hstack(star_fp))) im[np.isnan(im)] = 0 im = humutils.rescale(im,0,1) #get SLIC superpixels segments_slic = slic(im, n_segments=int(im.shape[0]/10), compactness=.1) #pre-allocate texture lengthscale array tl = np.zeros(im.shape, dtype = "float64") #cycle through each segment and compute tl for k in np.unique(segments_slic): mask = np.zeros(im.shape[:2], dtype = "uint8") mask[segments_slic == k] = 255 cmask, cim = crop_toseg(mask, im) tl[segments_slic == k] = parallel_me(cim, maxscale, notes, np.shape(cim)[0]) R_fp = io.get_mmap_data(sonpath, base, '_data_range.dat', 'float32', tuple(shape_star)) R = np.vstack((np.flipud(np.hstack(R_fp)), np.hstack(R_fp))) R = R/np.max(R) #correct for range and scale tl = tl * np.cos(R) * (1/ft) tl[im==0] = np.nan tl[np.isnan(im)] = np.nan # create memory mapped file for Sp with open(os.path.normpath(os.path.join(sonpath,base+'_data_class.dat')), 'w+') as ff: fp = np.memmap(ff, dtype='float32', mode='w+', shape=tuple(shape)) counter = 0 if len(shape_star)>2: for p in range(len(port_fp)): if p==0: n,m = np.shape(np.vstack((np.flipud(port_fp[p]), star_fp[p]))) else: n,m = np.shape(np.vstack((np.flipud(port_fp[p]), star_fp[p]))) Sp = tl[:n, counter:counter+m] counter = counter+m fp[p] = Sp.astype('float32') del Sp del fp # flush data to file class_fp = io.get_mmap_data(sonpath, base, '_data_class.dat', 'float32', tuple(shape)) else: with open(os.path.normpath(os.path.join(sonpath,base+'_data_class.dat')), 'w+') as ff: np.save(ff, np.squeeze(Sp).astype('float32')) with open(os.path.normpath(os.path.join(sonpath,base+'_data_class.dat')), 'r') as ff: class_fp = np.load(ff) dist_m = np.squeeze(loadmat(sonpath+base+'meta.mat')['dist_m']) ######################################################## if doplot==1: if len(shape_star)>2: for p in range(len(star_fp)): plot_class(dist_m, shape_port, port_fp[p], star_fp[p], class_fp[p], ft, humfile, sonpath, base, p) else: plot_class(dist_m, shape_port, port_fp, star_fp, class_fp, ft, humfile, sonpath, base, 0) if len(shape_star)>2: for p in range(len(star_fp)): plot_contours(dist_m, shape_port, class_fp[p], ft, humfile, sonpath, base, numclasses, p) else: plot_contours(dist_m, shape_port, class_fp, ft, humfile, sonpath, base, numclasses, 0) ####################################################### # k-means if len(shape_star)>2: with open(os.path.normpath(os.path.join(sonpath,base+'_data_kclass.dat')), 'w+') as ff: fp = np.memmap(ff, dtype='float32', mode='w+', shape=tuple(shape)) for p in range(len(port_fp)): wc = get_kclass(class_fp[p].copy(), numclasses) fp[p] = wc.astype('float32') del wc del fp kclass_fp = io.get_mmap_data(sonpath, base, '_data_kclass.dat', 'float32', tuple(shape)) else: wc = get_kclass(class_fp.copy(), numclasses) with open(os.path.normpath(os.path.join(sonpath,base+'_data_kclass.dat')), 'w+') as ff: np.save(ff, np.squeeze(wc).astype('float32')) del wc with open(os.path.normpath(os.path.join(sonpath,base+'_data_kclass.dat')), 'r') as ff: kclass_fp = np.load(ff) ######################################################## if doplot==1: if len(shape_star)>2: for p in range(len(star_fp)): plot_kmeans(dist_m, shape_port, port_fp[p], star_fp[p], kclass_fp[p], ft, humfile, sonpath, base, p) else: plot_kmeans(dist_m, shape_port, port_fp, star_fp, kclass_fp, ft, humfile, sonpath, base, 0) if os.name=='posix': # true if linux/mac elapsed = (time.time() - start) else: # windows elapsed = (time.clock() - start) print("Processing took "+str(elapsed)+"seconds to analyse") print("Done!")
def texture2(humfile, sonpath, win, doplot, numclasses): ''' Create a texture lengthscale map using the algorithm detailed by Buscombe et al. (2015) This textural lengthscale is not a direct measure of grain size. Rather, it is a statistical representation that integrates over many attributes of bed texture, of which grain size is the most important. The technique is a physically based means to identify regions of texture within a sidescan echogram, and could provide a basis for objective, automated riverbed sediment classification. Syntax ---------- [] = PyHum.texture(humfile, sonpath, win, doplot, numclasses) Parameters ---------- humfile : str path to the .DAT file sonpath : str path where the *.SON files are win : int, *optional* [Default=10] pixel in pixels of the moving window doplot : int, *optional* [Default=1] if 1, make plots, otherwise do not make plots numclasses : int, *optional* [Default=4] number of 'k means' that the texture lengthscale will be segmented into Returns ------- sonpath+base+'_data_class.dat': memory-mapped file contains the texture lengthscale map sonpath+base+'_data_kclass.dat': memory-mapped file contains the k-means segmented texture lengthscale map References ---------- .. [1] Buscombe, D., Grams, P.E., and Smith, S.M.C., 2015, Automated riverbed sediment classification using low-cost sidescan sonar. Journal of Hydraulic Engineering 10.1061/(ASCE)HY.1943-7900.0001079, 06015019. ''' # prompt user to supply file if no input file given if not humfile: print('An input file is required!!!!!!') Tk().withdraw( ) # we don't want a full GUI, so keep the root window from appearing humfile = askopenfilename(filetypes=[("DAT files", "*.DAT")]) # prompt user to supply directory if no input sonpath is given if not sonpath: print('A *.SON directory is required!!!!!!') Tk().withdraw( ) # we don't want a full GUI, so keep the root window from appearing sonpath = askdirectory() # print given arguments to screen and convert data type where necessary if humfile: print('Input file is %s' % (humfile)) if sonpath: print('Sonar file path is %s' % (sonpath)) if win: win = np.asarray(win, int) print('Window is %s square pixels' % (str(win))) if numclasses: numclasses = np.asarray(numclasses, int) print('Number of sediment classes: %s' % (str(numclasses))) if doplot: doplot = int(doplot) if doplot == 0: print("Plots will not be made") print('[Default] Number of processors is %s' % (str(cpu_count()))) ######################################################## ######################################################## # start timer if os.name == 'posix': # true if linux/mac or cygwin on windows start = time.time() else: # windows start = time.clock() # if son path name supplied has no separator at end, put one on if sonpath[-1] != os.sep: sonpath = sonpath + os.sep base = humfile.split('.DAT') # get base of file name for output base = base[0].split(os.sep)[-1] # remove underscores, negatives and spaces from basename base = humutils.strip_base(base) meta = loadmat(os.path.normpath(os.path.join(sonpath, base + 'meta.mat'))) ft = 1 / loadmat(sonpath + base + 'meta.mat')['pix_m'] #pix_m = np.squeeze(meta['pix_m']) #dep_m = np.squeeze(meta['dep_m']) dist_m = np.squeeze(meta['dist_m']) ### port print("processing port side ...") # load memory mapped scan ... port shape_port = np.squeeze(meta['shape_port']) if shape_port != '': if os.path.isfile( os.path.normpath( os.path.join(sonpath, base + '_data_port_lar.dat'))): port_fp = io.get_mmap_data(sonpath, base, '_data_port_lar.dat', 'float32', tuple(shape_port)) else: port_fp = io.get_mmap_data(sonpath, base, '_data_port_la.dat', 'float32', tuple(shape_port)) port_fp2 = io.get_mmap_data(sonpath, base, '_data_port_l.dat', 'float32', tuple(shape_port)) ### star print("processing starboard side ...") # load memory mapped scan ... port shape_star = np.squeeze(loadmat(sonpath + base + 'meta.mat')['shape_star']) if shape_star != '': if os.path.isfile( os.path.normpath( os.path.join(sonpath, base + '_data_star_lar.dat'))): star_fp = io.get_mmap_data(sonpath, base, '_data_star_lar.dat', 'float32', tuple(shape_star)) else: star_fp = io.get_mmap_data(sonpath, base, '_data_star_la.dat', 'float32', tuple(shape_star)) star_fp2 = io.get_mmap_data(sonpath, base, '_data_star_l.dat', 'float32', tuple(shape_star)) if len(shape_star) > 2: shape = shape_port.copy() shape[1] = shape_port[1] + shape_star[1] else: shape = [] shape.append(1) shape.append(shape_port[0]) shape.append(shape_port[1]) shape[1] = shape_port[0] + shape_star[0] # create memory mapped file for Sp #with open(os.path.normpath(os.path.join(sonpath,base+'_data_class.dat')), 'w+') as ff: # fp = np.memmap(ff, dtype='float32', mode='w+', shape=tuple(shape)) fp = np.zeros(tuple(shape), dtype='float32') if len(shape_star) > 2: for p in range(len(port_fp)): merge = np.vstack((np.flipud(port_fp[p]), star_fp[p])) merge = denoise_tv_chambolle(merge.copy(), weight=2, multichannel=False).astype('float32') Snn = std_convoluted(merge, win)[1] del merge try: Snn = medfilt2d(Snn, (win + 1, win + 1)) except: Snn = medfilt2d(Snn, (win, win)) Snn[np.isnan(np.vstack( (np.flipud(port_fp[p]), star_fp[p])))] = np.nan Snn[np.isnan(np.vstack( (np.flipud(port_fp2[p]), star_fp2[p])))] = np.nan R_fp = io.get_mmap_data(sonpath, base, '_data_range.dat', 'float32', tuple(shape_star)) R = np.vstack((np.flipud(R_fp[0]), R_fp[0])) R = R / np.max(R) rn = replace_nans.RN(R.astype('float64'), 1000, 0.01, 2, 'localmean') R = rn.getdata() del rn Sp = (Snn**2) * np.cos(np.deg2rad(R)) / win ##**2 fp[p] = Sp.astype('float32') del Sp #del fp # flush data to file shape = io.set_mmap_data(sonpath, base, '_data_class.dat', 'float32', np.squeeze(fp)) del fp class_fp = io.get_mmap_data(sonpath, base, '_data_class.dat', 'float32', tuple(shape)) else: merge = np.vstack((np.flipud(port_fp), star_fp)) merge = denoise_tv_chambolle(merge.copy(), weight=2, multichannel=False).astype('float32') Snn = std_convoluted(merge, win)[1] del merge try: Snn = medfilt2d(Snn, (win + 1, win + 1)) except: Snn = medfilt2d(Snn, (win, win)) Snn[np.isnan(np.vstack((np.flipud(port_fp), star_fp)))] = np.nan Snn[np.isnan(np.vstack((np.flipud(port_fp2), star_fp2)))] = np.nan R_fp = io.get_mmap_data(sonpath, base, '_data_range.dat', 'float32', tuple(shape_star)) R = np.vstack((np.flipud(R_fp), R_fp)) R = R / np.max(R) rn = replace_nans.RN(R.astype('float64'), 1000, 0.01, 2, 'localmean') R = rn.getdata() del rn Sp = (Snn**2) * np.cos(np.deg2rad(R)) / win ##**2 shape = io.set_mmap_data(sonpath, base, '_data_class.dat', 'float32', np.squeeze(Sp)) #with open(os.path.normpath(os.path.join(sonpath,base+'_data_class.dat')), 'w+') as ff: # np.save(ff, np.squeeze(Sp).astype('float32')) #with open(os.path.normpath(os.path.join(sonpath,base+'_data_class.dat')), 'r') as ff: # class_fp = np.load(ff) #del Sp class_fp = io.get_mmap_data(sonpath, base, '_data_class.dat', 'float32', tuple(shape)) dist_m = np.squeeze(loadmat(sonpath + base + 'meta.mat')['dist_m']) ######################################################## if doplot == 1: if len(shape_star) > 2: for p in range(len(star_fp)): plot_class(dist_m, shape_port, port_fp[p], star_fp[p], class_fp[p], ft, humfile, sonpath, base, p) else: plot_class(dist_m, shape_port, port_fp, star_fp, class_fp, ft, humfile, sonpath, base, 0) if len(shape_star) > 2: for p in range(len(star_fp)): plot_contours(dist_m, shape_port, port_fp[p], star_fp[p], class_fp[p], ft, humfile, sonpath, base, numclasses, p) else: plot_contours(dist_m, shape_port, port_fp, star_fp, class_fp, ft, humfile, sonpath, base, numclasses, 0) ####################################################### # k-means if len(shape_star) > 2: with open( os.path.normpath( os.path.join(sonpath, base + '_data_kclass.dat')), 'w+') as ff: fp = np.memmap(ff, dtype='float32', mode='w+', shape=tuple(shape)) for p in range(len(port_fp)): wc = get_kclass(class_fp[p].copy(), numclasses) fp[p] = wc.astype('float32') del wc del fp kclass_fp = io.get_mmap_data(sonpath, base, '_data_kclass.dat', 'float32', tuple(shape)) else: wc = get_kclass(class_fp.copy(), numclasses) with open( os.path.normpath( os.path.join(sonpath, base + '_data_kclass.dat')), 'w+') as ff: np.save(ff, np.squeeze(wc).astype('float32')) del wc with open( os.path.normpath( os.path.join(sonpath, base + '_data_kclass.dat')), 'r') as ff: kclass_fp = np.load(ff) ######################################################## if doplot == 1: if len(shape_star) > 2: for p in range(len(star_fp)): plot_kmeans(dist_m, shape_port, port_fp[p], star_fp[p], kclass_fp[p], ft, humfile, sonpath, base, p) else: plot_kmeans(dist_m, shape_port, port_fp, star_fp, kclass_fp, ft, humfile, sonpath, base, 0) if os.name == 'posix': # true if linux/mac elapsed = (time.time() - start) else: # windows elapsed = (time.clock() - start) print("Processing took " + str(elapsed) + "seconds to analyse") print("Done!") print("===================================================")
def read(humfile, sonpath, cs2cs_args, c, draft, doplot, t, bedpick, flip_lr, model, calc_bearing, filt_bearing, chunk): #cog = 1, ''' Read a .DAT and associated set of .SON files recorded by a Humminbird(R) instrument. Parse the data into a set of memory mapped files that will subsequently be used by the other functions of the PyHum module. Export time-series data and metadata in other formats. Create a kml file for visualising boat track Syntax ---------- [] = PyHum.read(humfile, sonpath, cs2cs_args, c, draft, doplot, t, bedpick, flip_lr, chunksize, model, calc_bearing, filt_bearing, chunk) Parameters ------------ humfile : str path to the .DAT file sonpath : str path where the *.SON files are cs2cs_args : int, *optional* [Default="epsg:26949"] arguments to create coordinates in a projected coordinate system this argument gets given to pyproj to turn wgs84 (lat/lon) coordinates into any projection supported by the proj.4 libraries c : float, *optional* [Default=1450.0] speed of sound in water (m/s). Defaults to a value of freshwater draft : float, *optional* [Default=0.3] draft from water surface to transducer face (m) doplot : float, *optional* [Default=1] if 1, plots will be made t : float, *optional* [Default=0.108] length of transducer array (m). Default value is that of the 998 series Humminbird(R) bedpick : int, *optional* [Default=1] if 1, bedpicking with be carried out automatically if 0, user will be prompted to pick the bed location on screen flip_lr : int, *optional* [Default=0] if 1, port and starboard scans will be flipped (for situations where the transducer is flipped 180 degrees) model: int, *optional* [Default=998] A 3 or 4 number code indicating the model number Examples: 998, 997, 1198, 1199 calc_bearing : float, *optional* [Default=0] if 1, bearing will be calculated from coordinates filt_bearing : float, *optional* [Default=0] if 1, bearing will be filtered chunk : str, *optional* [Default='d100' (distance, 100 m)] letter, followed by a number. There are the following letter options: 'd' - parse chunks based on distance, then number which is distance in m 'p' - parse chunks based on number of pings, then number which is number of pings 'h' - parse chunks based on change in heading, then number which is the change in heading in degrees '1' - process just 1 chunk Returns --------- sonpath+base+'_data_port.dat': memory-mapped file contains the raw echogram from the port side sidescan sonar (where present) sonpath+base+'_data_port.dat': memory-mapped file contains the raw echogram from the starboard side sidescan sonar (where present) sonpath+base+'_data_dwnhi.dat': memory-mapped file contains the raw echogram from the high-frequency echosounder (where present) sonpath+base+'_data_dwnlow.dat': memory-mapped file contains the raw echogram from the low-frequency echosounder (where present) sonpath+base+"trackline.kml": google-earth kml file contains the trackline of the vessel during data acquisition sonpath+base+'rawdat.csv': comma separated value file contains time-series data. columns corresponding to longitude latitude easting (m) northing (m) depth to bed (m) alongtrack cumulative distance (m) vessel heading (deg.) sonpath+base+'meta.mat': .mat file matlab format file containing a dictionary object holding metadata information. Fields are: e : ndarray, easting (m) n : ndarray, northing (m) es : ndarray, low-pass filtered easting (m) ns : ndarray, low-pass filtered northing (m) lat : ndarray, latitude lon : ndarray, longitude shape_port : tuple, shape of port scans in memory mapped file shape_star : tuple, shape of starboard scans in memory mapped file shape_hi : tuple, shape of high-freq. scans in memory mapped file shape_low : tuple, shape of low-freq. scans in memory mapped file dep_m : ndarray, depth to bed (m) dist_m : ndarray, distance along track (m) heading : ndarray, heading of vessel (deg. N) pix_m: float, size of 1 pixel in across-track dimension (m) bed : ndarray, depth to bed (m) c : float, speed of sound in water (m/s) t : length of sidescan transducer array (m) spd : ndarray, vessel speed (m/s) time_s : ndarray, time elapsed (s) caltime : ndarray, unix epoch time (s) ''' # prompt user to supply file if no input file given if not humfile: print('An input file is required!!!!!!') Tk().withdraw() # we don't want a full GUI, so keep the root window from appearing humfile = askopenfilename(filetypes=[("DAT files","*.DAT")]) # prompt user to supply directory if no input sonpath is given if not sonpath: print('A *.SON directory is required!!!!!!') Tk().withdraw() # we don't want a full GUI, so keep the root window from appearing sonpath = askdirectory() # print given arguments to screen and convert data type where necessary if humfile: print('Input file is %s' % (humfile)) if sonpath: print('Son files are in %s' % (sonpath)) if cs2cs_args: print('cs2cs arguments are %s' % (cs2cs_args)) if draft: draft = float(draft) print('Draft: %s' % (str(draft))) if c: c = float(c) print('Celerity of sound: %s m/s' % (str(c))) if doplot: doplot = int(doplot) if doplot==0: print("Plots will not be made") if flip_lr: flip_lr = int(flip_lr) if flip_lr==1: print("Port and starboard will be flipped") if t: t = np.asarray(t,float) print('Transducer length is %s m' % (str(t))) if bedpick: bedpick = np.asarray(bedpick,int) if bedpick==1: print('Bed picking is auto') elif bedpick==0: print('Bed picking is manual') else: print('User will be prompted per chunk about bed picking method') if chunk: chunk = str(chunk) if chunk[0]=='d': chunkmode=1 chunkval = int(chunk[1:]) print('Chunks based on distance of %s m' % (str(chunkval))) elif chunk[0]=='p': chunkmode=2 chunkval = int(chunk[1:]) print('Chunks based on %s pings' % (str(chunkval))) elif chunk[0]=='h': chunkmode=3 chunkval = int(chunk[1:]) print('Chunks based on heading devation of %s degrees' % (str(chunkval))) elif chunk[0]=='1': chunkmode=4 chunkval = 1 print('Only 1 chunk will be produced') else: print("Chunk mode not understood - should be 'd', 'p', or 'h' - using defaults") chunkmode=1 chunkval = 100 print('Chunks based on distance of %s m' % (str(chunkval))) if model: try: model = int(model) print("Data is from the %s series" % (str(model))) except: if model=='onix': model=0 print("Data is from the ONIX series") elif model=='helix': model=1 print("Data is from the HELIX series") elif model=='mega': model=2 print("Data is from the MEGA series") # if cog: # cog = int(cog) # if cog==1: # print "Heading based on course-over-ground" if calc_bearing: calc_bearing = int(calc_bearing) if calc_bearing==1: print("Bearing will be calculated from coordinates") if filt_bearing: filt_bearing = int(filt_bearing) if filt_bearing==1: print("Bearing will be filtered") ## for debugging #humfile = r"test.DAT"; sonpath = "test_data" #cs2cs_args = "epsg:26949"; doplot = 1; draft = 0 #c=1450; bedpick=1; fliplr=1; chunk = 'd100' #model=998; cog=1; calc_bearing=0; filt_bearing=0 #if model==2: # f = 1000 #else: f = 455 try: print("Checking the epsg code you have chosen for compatibility with Basemap ... ") from mpl_toolkits.basemap import Basemap m = Basemap(projection='merc', epsg=cs2cs_args.split(':')[1], resolution = 'i', llcrnrlon=10, llcrnrlat=10, urcrnrlon=30, urcrnrlat=30) del m print("... epsg code compatible") except (ValueError): print("Error: the epsg code you have chosen is not compatible with Basemap") print("please choose a different epsg code (http://spatialreference.org/)") print("program will now close") sys.exit() # start timer if os.name=='posix': # true if linux/mac or cygwin on windows start = time.time() else: # windows start = time.clock() # if son path name supplied has no separator at end, put one on if sonpath[-1]!=os.sep: sonpath = sonpath + os.sep # get the SON files from this directory sonfiles = glob.glob(sonpath+'*.SON') if not sonfiles: sonfiles = glob.glob(os.getcwd()+os.sep+sonpath+'*.SON') base = humfile.split('.DAT') # get base of file name for output base = base[0].split(os.sep)[-1] # remove underscores, negatives and spaces from basename base = humutils.strip_base(base) print("WARNING: Because files have to be read in byte by byte,") print("this could take a very long time ...") #reading each sonfile in parallel should be faster ... try: o = Parallel(n_jobs = np.min([len(sonfiles), cpu_count()]), verbose=0)(delayed(getscans)(sonfiles[k], humfile, c, model, cs2cs_args) for k in range(len(sonfiles))) X, Y, A, B = zip(*o) for k in range(len(Y)): if Y[k] == 'sidescan_port': dat = A[k] #data.gethumdat() metadat = B[k] #data.getmetadata() if flip_lr==0: data_port = X[k].astype('int16') else: data_star = X[k].astype('int16') elif Y[k] == 'sidescan_starboard': if flip_lr==0: data_star = X[k].astype('int16') else: data_port = X[k].astype('int16') elif Y[k] == 'down_lowfreq': data_dwnlow = X[k].astype('int16') elif Y[k] == 'down_highfreq': data_dwnhi = X[k].astype('int16') elif Y[k] == 'down_vhighfreq': #hopefully this only applies to mega systems data_dwnhi = X[k].astype('int16') del X, Y, A, B, o old_pyread = 0 if 'data_port' not in locals(): data_port = '' print("portside scan not available") if 'data_star' not in locals(): data_star = '' print("starboardside scan not available") if 'data_dwnhi' not in locals(): data_dwnlow = '' print("high-frq. downward scan not available") if 'data_dwnlow' not in locals(): data_dwnlow = '' print("low-frq. downward scan not available") except: # revert back to older version if paralleleised version fails print("something went wrong with the parallelised version of pyread ...") try: import pyread except: from . import pyread data = pyread.pyread(sonfiles, humfile, c, model, cs2cs_args) dat = data.gethumdat() metadat = data.getmetadata() old_pyread = 1 nrec = len(metadat['n']) metadat['instr_heading'] = metadat['heading'][:nrec] #metadat['heading'] = humutils.get_bearing(calc_bearing, filt_bearing, cog, metadat['lat'], metadat['lon'], metadat['instr_heading']) try: es = humutils.runningMeanFast(metadat['e'][:nrec],len(metadat['e'][:nrec])/100) ns = humutils.runningMeanFast(metadat['n'][:nrec],len(metadat['n'][:nrec])/100) except: es = metadat['e'][:nrec] ns = metadat['n'][:nrec] metadat['es'] = es metadat['ns'] = ns try: trans = pyproj.Proj(init=cs2cs_args) except: trans = pyproj.Proj(cs2cs_args.lstrip(), inverse=True) lon, lat = trans(es, ns, inverse=True) metadat['lon'] = lon metadat['lat'] = lat metadat['heading'] = humutils.get_bearing(calc_bearing, filt_bearing, metadat['lat'], metadat['lon'], metadat['instr_heading']) #cog dist_m = humutils.get_dist(lat, lon) metadat['dist_m'] = dist_m if calc_bearing==1: # recalculate speed, m/s ds=np.gradient(np.squeeze(metadat['time_s'])) dx=np.gradient(np.squeeze(metadat['dist_m'])) metadat['spd'] = dx[:nrec]/ds[:nrec] # theta at 3dB in the horizontal theta3dB = np.arcsin(c/(t*(f*1000))) #resolution of 1 sidescan pixel to nadir ft = (np.pi/2)*(1/theta3dB) #/ (f/455) dep_m = humutils.get_depth(metadat['dep_m'][:nrec]) if old_pyread == 1: #older pyread version # port scan try: if flip_lr==0: data_port = data.getportscans().astype('int16') else: data_port = data.getstarscans().astype('int16') except: data_port = '' print("portside scan not available") if data_port!='': Zt, ind_port = makechunks_scan(chunkmode, chunkval, metadat, data_port, 0) del data_port ## create memory mapped file for Z shape_port = io.set_mmap_data(sonpath, base, '_data_port.dat', 'int16', Zt) ##we are only going to access the portion of memory required port_fp = io.get_mmap_data(sonpath, base, '_data_port.dat', 'int16', shape_port) if old_pyread == 1: #older pyread version # starboard scan try: if flip_lr==0: data_star = data.getstarscans().astype('int16') else: data_star = data.getportscans().astype('int16') except: data_star = '' print("starboardside scan not available") if data_star!='': Zt, ind_star = makechunks_scan(chunkmode, chunkval, metadat, data_star, 1) del data_star # create memory mapped file for Z shape_star = io.set_mmap_data(sonpath, base, '_data_star.dat', 'int16', Zt) star_fp = io.get_mmap_data(sonpath, base, '_data_star.dat', 'int16', shape_star) if 'star_fp' in locals() and 'port_fp' in locals(): # check that port and starboard are same size # and trim if not if np.shape(star_fp)!=np.shape(port_fp): print("port and starboard scans are different sizes ... rectifying") if np.shape(port_fp[0])[1] > np.shape(star_fp[0])[1]: tmp = port_fp.copy() tmp2 = np.empty_like(star_fp) for k in range(len(tmp)): tmp2[k] = tmp[k][:,:np.shape(star_fp[k])[1]] del tmp # create memory mapped file for Z shape_port = io.set_mmap_data(sonpath, base, '_data_port2.dat', 'int16', tmp2) #shape_star = shape_port.copy() shape_star = tuple(np.asarray(shape_port).copy()) ##we are only going to access the portion of memory required port_fp = io.get_mmap_data(sonpath, base, '_data_port2.dat', 'int16', shape_port) ind_port = list(ind_port) ind_port[-1] = np.shape(star_fp[0])[1] ind_port = tuple(ind_port) elif np.shape(port_fp[0])[1] < np.shape(star_fp[0])[1]: tmp = star_fp.copy() tmp2 = np.empty_like(port_fp) for k in range(len(tmp)): tmp2[k] = tmp[k][:,:np.shape(port_fp[k])[1]] del tmp # create memory mapped file for Z shape_port = io.set_mmap_data(sonpath, base, '_data_star2.dat', 'int16', tmp2) #shape_star = shape_port.copy() shape_star = tuple(np.asarray(shape_port).copy()) #we are only going to access the portion of memory required star_fp = io.get_mmap_data(sonpath, base, '_data_star2.dat', 'int16', shape_star) ind_star = list(ind_star) ind_star[-1] = np.shape(port_fp[0])[1] ind_star = tuple(ind_star) if old_pyread == 1: #older pyread version # low-freq. sonar try: data_dwnlow = data.getlowscans().astype('int16') except: data_dwnlow = '' print("low-freq. scan not available") if data_dwnlow!='': Zt, ind_low = makechunks_scan(chunkmode, chunkval, metadat, data_dwnlow, 2) del data_dwnlow # create memory mapped file for Z shape_low = io.set_mmap_data(sonpath, base, '_data_dwnlow.dat', 'int16', Zt) ##we are only going to access the portion of memory required dwnlow_fp = io.get_mmap_data(sonpath, base, '_data_dwnlow.dat', 'int16', shape_low) if old_pyread == 1: #older pyread version # hi-freq. sonar try: data_dwnhi = data.gethiscans().astype('int16') except: data_dwnhi = '' print("high-freq. scan not available") if data_dwnhi!='': Zt, ind_hi = makechunks_scan(chunkmode, chunkval, metadat, data_dwnhi, 3) del data_dwnhi # create memory mapped file for Z shape_hi = io.set_mmap_data(sonpath, base, '_data_dwnhi.dat', 'int16', Zt) dwnhi_fp = io.get_mmap_data(sonpath, base, '_data_dwnhi.dat', 'int16', shape_hi) if 'dwnhi_fp' in locals() and 'dwnlow_fp' in locals(): # check that low and high are same size # and trim if not if (np.shape(dwnhi_fp)!=np.shape(dwnlow_fp)) and (chunkmode!=4): print("dwnhi and dwnlow are different sizes ... rectifying") if np.shape(dwnhi_fp[0])[1] > np.shape(dwnlow_fp[0])[1]: tmp = dwnhi_fp.copy() tmp2 = np.empty_like(dwnlow_fp) for k in range(len(tmp)): tmp2[k] = tmp[k][:,:np.shape(dwnlow_fp[k])[1]] del tmp # create memory mapped file for Z shape_low = io.set_mmap_data(sonpath, base, '_data_dwnhi2.dat', 'int16', tmp2) #shape_hi = shape_low.copy() shape_hi = tuple(np.asarray(shape_low).copy()) ##we are only going to access the portion of memory required dwnhi_fp = io.get_mmap_data(sonpath, base, '_data_dwnhi2.dat', 'int16', shape_hi) ind_hi = list(ind_hi) ind_hi[-1] = np.shape(dwnlow_fp[0])[1] ind_hi = tuple(ind_hi) elif np.shape(dwnhi_fp[0])[1] < np.shape(dwnlow_fp[0])[1]: tmp = dwnlow_fp.copy() tmp2 = np.empty_like(dwnhi_fp) for k in range(len(tmp)): tmp2[k] = tmp[k][:,:np.shape(dwnhi_fp[k])[1]] del tmp # create memory mapped file for Z shape_low = io.set_mmap_data(sonpath, base, '_data_dwnlow2.dat', 'int16', tmp2) #shape_hi = shape_low.copy() shape_hi = tuple(np.asarray(shape_low).copy()) ##we are only going to access the portion of memory required dwnlow_fp = io.get_mmap_data(sonpath, base, '_data_dwnlow2.dat', 'int16', shape_low) ind_low = list(ind_low) ind_low[-1] = np.shape(dwnhi_fp[0])[1] ind_low = tuple(ind_low) if old_pyread == 1: #older pyread version del data if ('shape_port' in locals()) and (chunkmode!=4): metadat['shape_port'] = shape_port nrec = metadat['shape_port'][0] * metadat['shape_port'][2] elif ('shape_port' in locals()) and (chunkmode==4): metadat['shape_port'] = shape_port nrec = metadat['shape_port'][1] else: metadat['shape_port'] = '' if ('shape_star' in locals()) and (chunkmode!=4): metadat['shape_star'] = shape_star nrec = metadat['shape_star'][0] * metadat['shape_star'][2] elif ('shape_star' in locals()) and (chunkmode==4): metadat['shape_star'] = shape_star nrec = metadat['shape_star'][1] else: metadat['shape_star'] = '' if ('shape_hi' in locals()) and (chunkmode!=4): metadat['shape_hi'] = shape_hi #nrec = metadat['shape_hi'][0] * metadat['shape_hi'][2] * 2 elif ('shape_hi' in locals()) and (chunkmode==4): metadat['shape_hi'] = shape_hi else: metadat['shape_hi'] = '' if ('shape_low' in locals()) and (chunkmode!=4): metadat['shape_low'] = shape_low #nrec = metadat['shape_low'][0] * metadat['shape_low'][2] * 2 elif ('shape_low' in locals()) and (chunkmode==4): metadat['shape_low'] = shape_low else: metadat['shape_low'] = '' #make kml boat trackline humutils.make_trackline(lon,lat, sonpath, base) if 'port_fp' in locals() and 'star_fp' in locals(): #if not os.path.isfile(os.path.normpath(os.path.join(sonpath,base+'meta.mat'))): if 2>1: if bedpick == 1: # auto x, bed = humutils.auto_bedpick(ft, dep_m, chunkmode, port_fp, c) if len(dist_m)<len(bed): dist_m = np.append(dist_m,dist_m[-1]*np.ones(len(bed)-len(dist_m))) if doplot==1: if chunkmode!=4: for k in range(len(star_fp)): plot_2bedpicks(port_fp[k], star_fp[k], bed[ind_port[-1]*k:ind_port[-1]*(k+1)], dist_m[ind_port[-1]*k:ind_port[-1]*(k+1)], x[ind_port[-1]*k:ind_port[-1]*(k+1)], ft, shape_port, sonpath, k, chunkmode) else: plot_2bedpicks(port_fp, star_fp, bed, dist_m, x, ft, shape_port, sonpath, 0, chunkmode) # 'real' bed is estimated to be the minimum of the two bed = np.min(np.vstack((bed[:nrec],np.squeeze(x[:nrec]))),axis=0) bed = humutils.runningMeanFast(bed, 3) elif bedpick>1: # user prompt x, bed = humutils.auto_bedpick(ft, dep_m, chunkmode, port_fp, c) if len(dist_m)<len(bed): dist_m = np.append(dist_m,dist_m[-1]*np.ones(len(bed)-len(dist_m))) # 'real' bed is estimated to be the minimum of the two bed = np.min(np.vstack((bed[:nrec],np.squeeze(x[:nrec]))),axis=0) bed = humutils.runningMeanFast(bed, 3) # manually intervene fig = plt.figure() ax = plt.gca() if chunkmode !=4: im = ax.imshow(np.hstack(port_fp), cmap = 'gray', origin = 'upper') else: im = ax.imshow(port_fp, cmap = 'gray', origin = 'upper') plt.plot(bed,'r') plt.axis('normal'); plt.axis('tight') pts1 = plt.ginput(n=300, timeout=30) # it will wait for 200 clicks or 60 seconds x1=map(lambda x: x[0],pts1) # map applies the function passed as y1=map(lambda x: x[1],pts1) # first parameter to each element of pts plt.close() del fig if x1 != []: # if x1 is not empty tree = KDTree(zip(np.arange(1,len(bed)), bed)) try: dist, inds = tree.query(zip(x1, y1), k = 100, eps=5, n_jobs=-1) except: dist, inds = tree.query(zip(x1, y1), k = 100, eps=5) b = np.interp(inds,x1,y1) bed2 = bed.copy() bed2[inds] = b bed = bed2 if doplot==1: if chunkmode!=4: for k in range(len(star_fp)): plot_2bedpicks(port_fp[k], star_fp[k], bed[ind_port[-1]*k:ind_port[-1]*(k+1)], dist_m[ind_port[-1]*k:ind_port[-1]*(k+1)], x[ind_port[-1]*k:ind_port[-1]*(k+1)], ft, shape_port, sonpath, k, chunkmode) else: plot_2bedpicks(port_fp, star_fp, bed, dist_m, x, ft, shape_port, sonpath, 0, chunkmode) else: #manual beds=[] if chunkmode!=4: for k in range(len(port_fp)): raw_input("Bed picking "+str(k+1)+" of "+str(len(port_fp))+", are you ready? 30 seconds. Press Enter to continue...") bed={} fig = plt.figure() ax = plt.gca() im = ax.imshow(port_fp[k], cmap = 'gray', origin = 'upper') pts1 = plt.ginput(n=300, timeout=30) # it will wait for 200 clicks or 60 seconds x1=map(lambda x: x[0],pts1) # map applies the function passed as y1=map(lambda x: x[1],pts1) # first parameter to each element of pts bed = np.interp(np.r_[:ind_port[-1]],x1,y1) plt.close() del fig beds.append(bed) extent = np.shape(port_fp[k])[0] bed = np.asarray(np.hstack(beds),'float') else: raw_input("Bed picking - are you ready? 30 seconds. Press Enter to continue...") bed={} fig = plt.figure() ax = plt.gca() im = ax.imshow(port_fp, cmap = 'gray', origin = 'upper') pts1 = plt.ginput(n=300, timeout=30) # it will wait for 200 clicks or 60 seconds x1=map(lambda x: x[0],pts1) # map applies the function passed as y1=map(lambda x: x[1],pts1) # first parameter to each element of pts bed = np.interp(np.r_[:ind_port[-1]],x1,y1) plt.close() del fig beds.append(bed) extent = np.shape(port_fp)[1] bed = np.asarray(np.hstack(beds),'float') # now revise the depth in metres dep_m = (1/ft)*bed if doplot==1: if chunkmode!=4: for k in range(len(star_fp)): plot_bedpick(port_fp[k], star_fp[k], (1/ft)*bed[ind_port[-1]*k:ind_port[-1]*(k+1)], dist_m[ind_port[-1]*k:ind_port[-1]*(k+1)], ft, shape_port, sonpath, k, chunkmode) else: plot_bedpick(port_fp, star_fp, (1/ft)*bed, dist_m, ft, shape_port, sonpath, 0, chunkmode) metadat['bed'] = bed[:nrec] else: metadat['bed'] = dep_m[:nrec]*ft metadat['heading'] = metadat['heading'][:nrec] metadat['lon'] = lon[:nrec] metadat['lat'] = lat[:nrec] metadat['dist_m'] = dist_m[:nrec] metadat['dep_m'] = dep_m[:nrec] metadat['pix_m'] = 1/ft metadat['bed'] = metadat['bed'][:nrec] metadat['c'] = c metadat['t'] = t if model==2: metadat['f'] = f*2 else: metadat['f'] = f metadat['spd'] = metadat['spd'][:nrec] metadat['time_s'] = metadat['time_s'][:nrec] metadat['e'] = metadat['e'][:nrec] metadat['n'] = metadat['n'][:nrec] metadat['es'] = metadat['es'][:nrec] metadat['ns'] = metadat['ns'][:nrec] try: metadat['caltime'] = metadat['caltime'][:nrec] except: metadat['caltime'] = metadat['caltime'] savemat(os.path.normpath(os.path.join(sonpath,base+'meta.mat')), metadat ,oned_as='row') f = open(os.path.normpath(os.path.join(sonpath,base+'rawdat.csv')), 'wt') writer = csv.writer(f) writer.writerow( ('longitude', 'latitude', 'easting', 'northing', 'depth (m)', 'distance (m)', 'instr. heading (deg)', 'heading (deg.)' ) ) for i in range(0, nrec): writer.writerow(( float(lon[i]),float(lat[i]),float(es[i]),float(ns[i]),float(dep_m[i]),float(dist_m[i]), float(metadat['instr_heading'][i]), float(metadat['heading'][i]) )) f.close() del lat, lon, dep_m #, dist_m if doplot==1: plot_pos(sonpath, metadat, es, ns) if 'dwnlow_fp' in locals(): plot_dwnlow(dwnlow_fp, chunkmode, sonpath) if 'dwnhi_fp' in locals(): plot_dwnhi(dwnhi_fp, chunkmode, sonpath) if os.name=='posix': # true if linux/mac elapsed = (time.time() - start) else: # windows elapsed = (time.clock() - start) print("Processing took "+ str(elapsed) + "seconds to analyse") print("Done!") print("===================================================")