def write_points(e, n, t, d, dat_port, dat_star, data_R, pix_m, res,
cs2cs_args, sonpath, p, c, dx):
#trans = pyproj.Proj(init=cs2cs_args)
merge = np.vstack((dat_port, dat_star))
merge[np.isnan(merge)] = 0
merge = merge[:, :len(n)]
## actual along-track resolution is this: dx times dy = Af
tmp = data_R * dx * (c * 0.007 / 2
) #dx = np.arcsin(c/(1000*meta['t']*meta['f']))
res_grid = np.vstack((tmp, tmp))
del tmp
res_grid = res_grid[:np.shape(merge)[0], :np.shape(merge)[1]]
merge = merge - 10 * np.log10(res_grid)
merge[np.isnan(merge)] = 0
merge[merge < 0] = 0
R = np.vstack((np.flipud(data_R), data_R))
R = R[:np.shape(merge)[0], :np.shape(merge)[1]]
# get number pixels in scan line
extent = int(np.shape(merge)[0] / 2)
yvec = np.squeeze(
np.linspace(np.squeeze(pix_m), extent * np.squeeze(pix_m), extent))
X, Y, D, h, t = getXY(e, n, yvec, np.squeeze(d), t, extent)
D[np.isnan(D)] = 0
h[np.isnan(h)] = 0
t[np.isnan(t)] = 0
X = X[np.where(np.logical_not(np.isnan(Y)))]
merge = merge.flatten()[np.where(np.logical_not(np.isnan(Y)))]
res_grid = res_grid.flatten()[np.where(np.logical_not(np.isnan(Y)))]
Y = Y[np.where(np.logical_not(np.isnan(Y)))]
D = D[np.where(np.logical_not(np.isnan(Y)))]
R = R.flatten()[np.where(np.logical_not(np.isnan(Y)))]
h = h[np.where(np.logical_not(np.isnan(Y)))]
t = t[np.where(np.logical_not(np.isnan(Y)))]
Y = Y[np.where(np.logical_not(np.isnan(X)))]
merge = merge.flatten()[np.where(np.logical_not(np.isnan(X)))]
res_grid = res_grid.flatten()[np.where(np.logical_not(np.isnan(X)))]
X = X[np.where(np.logical_not(np.isnan(X)))]
D = D[np.where(np.logical_not(np.isnan(X)))]
R = R.flatten()[np.where(np.logical_not(np.isnan(X)))]
h = h[np.where(np.logical_not(np.isnan(X)))]
t = t[np.where(np.logical_not(np.isnan(X)))]
X = X[np.where(np.logical_not(np.isnan(merge)))]
Y = Y[np.where(np.logical_not(np.isnan(merge)))]
merge = merge[np.where(np.logical_not(np.isnan(merge)))]
res_grid = res_grid.flatten()[np.where(np.logical_not(np.isnan(merge)))]
D = D[np.where(np.logical_not(np.isnan(merge)))]
R = R[np.where(np.logical_not(np.isnan(merge)))]
h = h[np.where(np.logical_not(np.isnan(merge)))]
t = t[np.where(np.logical_not(np.isnan(merge)))]
X = X[np.where(np.logical_not(np.isinf(merge)))]
Y = Y[np.where(np.logical_not(np.isinf(merge)))]
merge = merge[np.where(np.logical_not(np.isinf(merge)))]
res_grid = res_grid.flatten()[np.where(np.logical_not(np.isinf(merge)))]
D = D[np.where(np.logical_not(np.isinf(merge)))]
R = R[np.where(np.logical_not(np.isinf(merge)))]
h = h[np.where(np.logical_not(np.isinf(merge)))]
t = t[np.where(np.logical_not(np.isinf(merge)))]
## write raw bs to file
outfile = os.path.normpath(
os.path.join(sonpath, 'x_y_ss_raw' + str(p) + '.asc'))
write.txtwrite(
outfile,
np.hstack(
(humutils.ascol(X.flatten()), humutils.ascol(Y.flatten()),
humutils.ascol(merge.flatten()), humutils.ascol(D.flatten()),
humutils.ascol(R.flatten()), humutils.ascol(h.flatten()),
humutils.ascol(t.flatten()))))
del D, R, h, t, X, Y, merge, res_grid
X = X[np.where(np.logical_not(np.isnan(merge)))]
Y = Y[np.where(np.logical_not(np.isnan(merge)))]
merge = merge[np.where(np.logical_not(np.isnan(merge)))]
# plot to check
#plt.scatter(X[::20],Y[::20],10,merge[::20], linewidth=0)
print "writing point cloud"
## write raw bs to file
outfile = os.path.normpath(
os.path.join(sonpath, 'x_y_slicsegmentnumber.asc'))
np.savetxt(outfile,
np.hstack(
(humutils.ascol(X.flatten()), humutils.ascol(Y.flatten()),
humutils.ascol(merge.flatten()))),
fmt="%8.6f %8.6f %8.6f")
trans = pyproj.Proj(init="epsg:26949")
humlon, humlat = trans(X, Y, inverse=True)
res = 0.25
orig_def, targ_def, grid_x, grid_y, res, shape = get_griddefs(
np.min(X), np.max(X), np.min(Y), np.max(Y), res, humlon, humlat, trans)
grid_x = grid_x.astype('float32')
grid_y = grid_y.astype('float32')
sigmas = 1 #m
eps = 2
def write_points(e, n, t, d, dat_port, dat_star, data_R, pix_m, res, cs2cs_args, sonpath, p, c, dx):
#trans = pyproj.Proj(init=cs2cs_args)
merge = np.vstack((dat_port,dat_star))
merge[np.isnan(merge)] = 0
merge = merge[:,:len(n)]
## actual along-track resolution is this: dx times dy = Af
tmp = data_R * dx * (c*0.007 / 2) #dx = np.arcsin(c/(1000*meta['t']*meta['f']))
res_grid = np.vstack((tmp, tmp))
del tmp
res_grid = res_grid[:np.shape(merge)[0],:np.shape(merge)[1]]
merge = merge - 10*np.log10(res_grid)
merge[np.isnan(merge)] = 0
merge[merge<0] = 0
R = np.vstack((np.flipud(data_R),data_R))
R = R[:np.shape(merge)[0],:np.shape(merge)[1]]
# get number pixels in scan line
extent = int(np.shape(merge)[0]/2)
yvec = np.squeeze(np.linspace(np.squeeze(pix_m),extent*np.squeeze(pix_m),extent))
X, Y, D, h, t = getXY(e,n,yvec,np.squeeze(d),t,extent)
D[np.isnan(D)] = 0
h[np.isnan(h)] = 0
t[np.isnan(t)] = 0
X = X[np.where(np.logical_not(np.isnan(Y)))]
merge = merge.flatten()[np.where(np.logical_not(np.isnan(Y)))]
res_grid = res_grid.flatten()[np.where(np.logical_not(np.isnan(Y)))]
Y = Y[np.where(np.logical_not(np.isnan(Y)))]
D = D[np.where(np.logical_not(np.isnan(Y)))]
R = R.flatten()[np.where(np.logical_not(np.isnan(Y)))]
h = h[np.where(np.logical_not(np.isnan(Y)))]
t = t[np.where(np.logical_not(np.isnan(Y)))]
Y = Y[np.where(np.logical_not(np.isnan(X)))]
merge = merge.flatten()[np.where(np.logical_not(np.isnan(X)))]
res_grid = res_grid.flatten()[np.where(np.logical_not(np.isnan(X)))]
X = X[np.where(np.logical_not(np.isnan(X)))]
D = D[np.where(np.logical_not(np.isnan(X)))]
R = R.flatten()[np.where(np.logical_not(np.isnan(X)))]
h = h[np.where(np.logical_not(np.isnan(X)))]
t = t[np.where(np.logical_not(np.isnan(X)))]
X = X[np.where(np.logical_not(np.isnan(merge)))]
Y = Y[np.where(np.logical_not(np.isnan(merge)))]
merge = merge[np.where(np.logical_not(np.isnan(merge)))]
res_grid = res_grid.flatten()[np.where(np.logical_not(np.isnan(merge)))]
D = D[np.where(np.logical_not(np.isnan(merge)))]
R = R[np.where(np.logical_not(np.isnan(merge)))]
h = h[np.where(np.logical_not(np.isnan(merge)))]
t = t[np.where(np.logical_not(np.isnan(merge)))]
X = X[np.where(np.logical_not(np.isinf(merge)))]
Y = Y[np.where(np.logical_not(np.isinf(merge)))]
merge = merge[np.where(np.logical_not(np.isinf(merge)))]
res_grid = res_grid.flatten()[np.where(np.logical_not(np.isinf(merge)))]
D = D[np.where(np.logical_not(np.isinf(merge)))]
R = R[np.where(np.logical_not(np.isinf(merge)))]
h = h[np.where(np.logical_not(np.isinf(merge)))]
t = t[np.where(np.logical_not(np.isinf(merge)))]
## write raw bs to file
outfile = os.path.normpath(os.path.join(sonpath,'x_y_ss_raw'+str(p)+'.asc'))
write.txtwrite( outfile, np.hstack((humutils.ascol(X.flatten()),humutils.ascol(Y.flatten()), humutils.ascol(merge.flatten()), humutils.ascol(D.flatten()), humutils.ascol(R.flatten()), humutils.ascol(h.flatten()), humutils.ascol(t.flatten()) )) )
del D, R, h, t, X, Y, merge, res_grid
def make_map(e, n, t, d, dat_port, dat_star, data_R, pix_m, res, cs2cs_args, sonpath, p, mode, nn, numstdevs, c, dx, use_uncorrected, scalemax): #dogrid, influence,dowrite,
thres=5
trans = pyproj.Proj(init=cs2cs_args)
mp = np.nanmean(dat_port)
ms = np.nanmean(dat_star)
if mp>ms:
merge = np.vstack((dat_port,dat_star*(mp/ms)))
else:
merge = np.vstack((dat_port*(ms/mp),dat_star))
del dat_port, dat_star
merge[np.isnan(merge)] = 0
merge = merge[:,:len(n)]
## actual along-track resolution is this: dx times dy = Af
tmp = data_R * dx * (c*0.007 / 2) #dx = np.arcsin(c/(1000*meta['t']*meta['f']))
res_grid = np.sqrt(np.vstack((tmp, tmp)))
del tmp
res_grid = res_grid[:np.shape(merge)[0],:np.shape(merge)[1]]
#if use_uncorrected != 1:
# merge = merge - 10*np.log10(res_grid)
res_grid = res_grid.astype('float32')
merge[np.isnan(merge)] = 0
merge[merge<0] = 0
merge = merge.astype('float32')
merge = denoise_tv_chambolle(merge.copy(), weight=.2, multichannel=False).astype('float32')
R = np.vstack((np.flipud(data_R),data_R))
del data_R
R = R[:np.shape(merge)[0],:np.shape(merge)[1]]
# get number pixels in scan line
extent = int(np.shape(merge)[0]/2)
yvec = np.squeeze(np.linspace(np.squeeze(pix_m),extent*np.squeeze(pix_m),extent))
X, Y, D, h, t = getXY(e,n,yvec,np.squeeze(d),t,extent)
X = X.astype('float32')
Y = Y.astype('float32')
D = D.astype('float32')
h = h.astype('float32')
t = t.astype('float32')
X = X.astype('float32')
D[np.isnan(D)] = 0
h[np.isnan(h)] = 0
t[np.isnan(t)] = 0
X = X[np.where(np.logical_not(np.isnan(Y)))]
merge = merge.flatten()[np.where(np.logical_not(np.isnan(Y)))]
res_grid = res_grid.flatten()[np.where(np.logical_not(np.isnan(Y)))]
Y = Y[np.where(np.logical_not(np.isnan(Y)))]
D = D[np.where(np.logical_not(np.isnan(Y)))]
R = R.flatten()[np.where(np.logical_not(np.isnan(Y)))]
h = h[np.where(np.logical_not(np.isnan(Y)))]
t = t[np.where(np.logical_not(np.isnan(Y)))]
Y = Y[np.where(np.logical_not(np.isnan(X)))]
merge = merge.flatten()[np.where(np.logical_not(np.isnan(X)))]
res_grid = res_grid.flatten()[np.where(np.logical_not(np.isnan(X)))]
X = X[np.where(np.logical_not(np.isnan(X)))]
D = D[np.where(np.logical_not(np.isnan(X)))]
R = R.flatten()[np.where(np.logical_not(np.isnan(X)))]
h = h[np.where(np.logical_not(np.isnan(X)))]
t = t[np.where(np.logical_not(np.isnan(X)))]
X = X[np.where(np.logical_not(np.isnan(merge)))]
Y = Y[np.where(np.logical_not(np.isnan(merge)))]
merge = merge[np.where(np.logical_not(np.isnan(merge)))]
res_grid = res_grid.flatten()[np.where(np.logical_not(np.isnan(merge)))]
D = D[np.where(np.logical_not(np.isnan(merge)))]
R = R[np.where(np.logical_not(np.isnan(merge)))]
h = h[np.where(np.logical_not(np.isnan(merge)))]
t = t[np.where(np.logical_not(np.isnan(merge)))]
X = X[np.where(np.logical_not(np.isinf(merge)))]
Y = Y[np.where(np.logical_not(np.isinf(merge)))]
merge = merge[np.where(np.logical_not(np.isinf(merge)))]
res_grid = res_grid.flatten()[np.where(np.logical_not(np.isinf(merge)))]
D = D[np.where(np.logical_not(np.isinf(merge)))]
R = R[np.where(np.logical_not(np.isinf(merge)))]
h = h[np.where(np.logical_not(np.isinf(merge)))]
t = t[np.where(np.logical_not(np.isinf(merge)))]
print("writing point cloud")
#if dowrite==1:
## write raw bs to file
outfile = os.path.normpath(os.path.join(sonpath,'x_y_ss_raw'+str(p)+'.asc'))
##write.txtwrite( outfile, np.hstack((humutils.ascol(X.flatten()),humutils.ascol(Y.flatten()), humutils.ascol(merge.flatten()), humutils.ascol(D.flatten()), humutils.ascol(R.flatten()), humutils.ascol(h.flatten()), humutils.ascol(t.flatten()) )) )
np.savetxt(outfile, np.hstack((humutils.ascol(X.flatten()),humutils.ascol(Y.flatten()), humutils.ascol(merge.flatten()), humutils.ascol(D.flatten()), humutils.ascol(R.flatten()), humutils.ascol(h.flatten()), humutils.ascol(t.flatten()) )) , fmt="%8.6f %8.6f %8.6f %8.6f %8.6f %8.6f %8.6f")
del D, R, h, t
sigmas = 0.1 #m
eps = 2
print("gridding ...")
#if dogrid==1:
if 2>1:
if res==99:
resg = np.min(res_grid[res_grid>0])/2
print('Gridding at resolution of %s' % str(resg))
else:
resg = res
tree = KDTree(np.c_[X.flatten(),Y.flatten()])
complete=0
while complete==0:
try:
grid_x, grid_y, res = getmesh(np.min(X), np.max(X), np.min(Y), np.max(Y), resg)
longrid, latgrid = trans(grid_x, grid_y, inverse=True)
longrid = longrid.astype('float32')
latgrid = latgrid.astype('float32')
shape = np.shape(grid_x)
## create mask for where the data is not
if pykdtree==1:
dist, _ = tree.query(np.c_[grid_x.ravel(), grid_y.ravel()], k=1)
else:
try:
dist, _ = tree.query(np.c_[grid_x.ravel(), grid_y.ravel()], k=1, n_jobs=cpu_count())
except:
#print ".... update your scipy installation to use faster kd-tree queries"
dist, _ = tree.query(np.c_[grid_x.ravel(), grid_y.ravel()], k=1)
dist = dist.reshape(grid_x.shape)
targ_def = pyresample.geometry.SwathDefinition(lons=longrid.flatten(), lats=latgrid.flatten())
del longrid, latgrid
humlon, humlat = trans(X, Y, inverse=True)
orig_def = pyresample.geometry.SwathDefinition(lons=humlon.flatten(), lats=humlat.flatten())
del humlon, humlat
if 'orig_def' in locals():
complete=1
except:
print("memory error: trying grid resolution of %s" % (str(resg*2)))
resg = resg*2
if mode==1:
complete=0
while complete==0:
try:
try:
dat = pyresample.kd_tree.resample_nearest(orig_def, merge.flatten(), targ_def, radius_of_influence=res*20, fill_value=None, nprocs = cpu_count(), reduce_data=1)
except:
dat = pyresample.kd_tree.resample_nearest(orig_def, merge.flatten(), targ_def, radius_of_influence=res*20, fill_value=None, nprocs = 1, reduce_data=1)
try:
r_dat = pyresample.kd_tree.resample_nearest(orig_def, res_grid.flatten(), targ_def, radius_of_influence=res*20, fill_value=None, nprocs = cpu_count(), reduce_data=1)
except:
r_dat = pyresample.kd_tree.resample_nearest(orig_def, res_grid.flatten(), targ_def, radius_of_influence=res*20, fill_value=None, nprocs = 1, reduce_data=1)
stdev = None
counts = None
if 'dat' in locals():
complete=1
except:
del grid_x, grid_y, targ_def, orig_def
wf = None
humlon, humlat = trans(X, Y, inverse=True)
dat, stdev, counts, resg, complete, shape = getgrid_lm(humlon, humlat, merge, res*10, min(X), max(X), min(Y), max(Y), resg*2, mode, trans, nn, wf, sigmas, eps)
r_dat, stdev, counts, resg, complete, shape = getgrid_lm(humlon, humlat, res_grid, res*10, min(X), max(X), min(Y), max(Y), resg*2, mode, trans, nn, wf, sigmas, eps)
del humlon, humlat
elif mode==2:
# custom inverse distance
wf = lambda r: 1/r**2
complete=0
while complete==0:
try:
try:
dat, stdev, counts = pyresample.kd_tree.resample_custom(orig_def, merge.flatten(),targ_def, radius_of_influence=res*20, neighbours=nn, weight_funcs=wf, fill_value=None, with_uncert = True, nprocs = cpu_count(), reduce_data=1)
except:
dat, stdev, counts = pyresample.kd_tree.resample_custom(orig_def, merge.flatten(),targ_def, radius_of_influence=res*20, neighbours=nn, weight_funcs=wf, fill_value=None, with_uncert = True, nprocs = 1, reduce_data=1)
try:
r_dat = pyresample.kd_tree.resample_custom(orig_def, res_grid.flatten(), targ_def, radius_of_influence=res*20, neighbours=nn, weight_funcs=wf, fill_value=None, with_uncert = False, nprocs = cpu_count(), reduce_data=1)
except:
r_dat = pyresample.kd_tree.resample_custom(orig_def, res_grid.flatten(), targ_def, radius_of_influence=res*20, neighbours=nn, weight_funcs=wf, fill_value=None, with_uncert = False, nprocs = 1, reduce_data=1)
if 'dat' in locals():
complete=1
except:
del grid_x, grid_y, targ_def, orig_def
humlon, humlat = trans(X, Y, inverse=True)
dat, stdev, counts, resg, complete, shape = getgrid_lm(humlon, humlat, merge, res*2, min(X), max(X), min(Y), max(Y), resg*2, mode, trans, nn, wf, sigmas, eps)
r_dat, stdev, counts, resg, complete, shape = getgrid_lm(humlon, humlat, res_grid, res*2, min(X), max(X), min(Y), max(Y), resg*2, mode, trans, nn, wf, sigmas, eps)
del humlat, humlon
del stdev_null, counts_null
elif mode==3:
wf = None
complete=0
while complete==0:
try:
try:
dat, stdev, counts = pyresample.kd_tree.resample_gauss(orig_def, merge.flatten(), targ_def, radius_of_influence=res*20, neighbours=nn, sigmas=sigmas, fill_value=None, with_uncert = True, nprocs = cpu_count(), epsilon = eps, reduce_data=1)
except:
dat, stdev, counts = pyresample.kd_tree.resample_gauss(orig_def, merge.flatten(), targ_def, radius_of_influence=res*20, neighbours=nn, sigmas=sigmas, fill_value=None, with_uncert = True, nprocs = 1, epsilon = eps, reduce_data=1)
try:
r_dat = pyresample.kd_tree.resample_gauss(orig_def, res_grid.flatten(), targ_def, radius_of_influence=res*20, neighbours=nn, sigmas=sigmas, fill_value=None, with_uncert = False, nprocs = cpu_count(), epsilon = eps, reduce_data=1)
except:
r_dat = pyresample.kd_tree.resample_gauss(orig_def, res_grid.flatten(), targ_def, radius_of_influence=res*20, neighbours=nn, sigmas=sigmas, fill_value=None, with_uncert = False, nprocs = 1, epsilon = eps, reduce_data=1)
if 'dat' in locals():
complete=1
except:
del grid_x, grid_y, targ_def, orig_def
humlon, humlat = trans(X, Y, inverse=True)
dat, stdev, counts, resg, complete, shape = getgrid_lm(humlon, humlat, merge, res*10, min(X), max(X), min(Y), max(Y), resg*2, mode, trans, nn, wf, sigmas, eps)
r_dat, stdev_null, counts_null, resg, complete, shape = getgrid_lm(humlon, humlat, res_grid, res*10, min(X), max(X), min(Y), max(Y), resg*2, mode, trans, nn, wf, sigmas, eps)
del humlat, humlon
del stdev_null, counts_null
humlon, humlat = trans(X, Y, inverse=True)
del X, Y, res_grid, merge
dat = dat.reshape(shape)
dat[dist>res*30] = np.nan
del dist
r_dat = r_dat.reshape(shape)
r_dat[r_dat<1] = 1
r_dat[r_dat > 2*np.pi] = 1
r_dat[np.isnan(dat)] = np.nan
dat = dat + r_dat #np.sqrt(np.cos(np.deg2rad(r_dat))) #dat*np.sqrt(r_dat) + dat
del r_dat
if mode>1:
stdev = stdev.reshape(shape)
counts = counts.reshape(shape)
mask = dat.mask.copy()
dat[mask==1] = np.nan
#dat[mask==1] = 0
if mode>1:
dat[(stdev>numstdevs) & (mask!=0)] = np.nan
dat[(counts<nn) & (counts>0)] = np.nan
#if dogrid==1:
dat[dat==0] = np.nan
dat[np.isinf(dat)] = np.nan
dat[dat<thres] = np.nan
datm = np.ma.masked_invalid(dat)
glon, glat = trans(grid_x, grid_y, inverse=True)
#del grid_x, grid_y
try:
from osgeo import gdal,ogr,osr
proj = osr.SpatialReference()
proj.ImportFromEPSG(int(cs2cs_args.split(':')[-1])) #26949)
datout = np.squeeze(np.ma.filled(dat))#.astype('int16')
datout[np.isnan(datout)] = -99
driver = gdal.GetDriverByName('GTiff')
#rows,cols = np.shape(datout)
cols,rows = np.shape(datout)
outFile = os.path.normpath(os.path.join(sonpath,'geotiff_map'+str(p)+'.tif'))
ds = driver.Create( outFile, rows, cols, 1, gdal.GDT_Float32, [ 'COMPRESS=LZW' ] )
if proj is not None:
ds.SetProjection(proj.ExportToWkt())
xmin, ymin, xmax, ymax = [grid_x.min(), grid_y.min(), grid_x.max(), grid_y.max()]
xres = (xmax - xmin) / float(rows)
yres = (ymax - ymin) / float(cols)
geotransform = (xmin, xres, 0, ymax, 0, -yres)
ds.SetGeoTransform(geotransform)
ss_band = ds.GetRasterBand(1)
ss_band.WriteArray(np.flipud(datout)) #datout)
ss_band.SetNoDataValue(-99)
ss_band.FlushCache()
ss_band.ComputeStatistics(False)
del ds
except:
print("error: geotiff could not be created... check your gdal/ogr install")
try:
# =========================================================
print("creating kmz file ...")
## new way to create kml file
pixels = 1024 * 10
fig, ax = humutils.gearth_fig(llcrnrlon=glon.min(),
llcrnrlat=glat.min(),
urcrnrlon=glon.max(),
urcrnrlat=glat.max(),
pixels=pixels)
cs = ax.pcolormesh(glon, glat, datm, vmax=scalemax, cmap='gray')
ax.set_axis_off()
fig.savefig(os.path.normpath(os.path.join(sonpath,'map'+str(p)+'.png')), transparent=True, format='png')
del fig, ax
# =========================================================
fig = plt.figure(figsize=(1.0, 4.0), facecolor=None, frameon=False)
ax = fig.add_axes([0.0, 0.05, 0.2, 0.9])
cb = fig.colorbar(cs, cax=ax)
cb.set_label('Intensity [dB W]', rotation=-90, color='k', labelpad=20)
fig.savefig(os.path.normpath(os.path.join(sonpath,'legend'+str(p)+'.png')), transparent=False, format='png')
del fig, ax, cs, cb
# =========================================================
humutils.make_kml(llcrnrlon=glon.min(), llcrnrlat=glat.min(),
urcrnrlon=glon.max(), urcrnrlat=glat.max(),
figs=[os.path.normpath(os.path.join(sonpath,'map'+str(p)+'.png'))],
colorbar=os.path.normpath(os.path.join(sonpath,'legend'+str(p)+'.png')),
kmzfile=os.path.normpath(os.path.join(sonpath,'GroundOverlay'+str(p)+'.kmz')),
name='Sidescan Intensity')
except:
print("error: map could not be created...")
#y1 = np.min(glat)-0.001
#x1 = np.min(glon)-0.001
#y2 = np.max(glat)+0.001
#x2 = np.max(glon)+0.001
print("drawing and printing map ...")
fig = plt.figure(frameon=False)
map = Basemap(projection='merc', epsg=cs2cs_args.split(':')[1],
resolution = 'i', #h #f
llcrnrlon=np.min(humlon)-0.001, llcrnrlat=np.min(glat)-0.001,
urcrnrlon=np.max(humlon)+0.001, urcrnrlat=np.max(glat)+0.001)
try:
map.arcgisimage(server='http://server.arcgisonline.com/ArcGIS', service='World_Imagery', xpixels=1000, ypixels=None, dpi=300)
except:
map.arcgisimage(server='http://server.arcgisonline.com/ArcGIS', service='ESRI_Imagery_World_2D', xpixels=1000, ypixels=None, dpi=300)
#finally:
# print "error: map could not be created..."
#if dogrid==1:
gx,gy = map.projtran(glon, glat)
ax = plt.Axes(fig, [0., 0., 1., 1.], )
ax.set_axis_off()
fig.add_axes(ax)
#if dogrid==1:
if 2>1:
if datm.size > 25000000:
print("matrix size > 25,000,000 - decimating by factor of 5 for display")
map.pcolormesh(gx[::5,::5], gy[::5,::5], datm[::5,::5], cmap='gray', vmin=np.nanmin(datm), vmax=scalemax) #vmax=np.nanmax(datm)
else:
map.pcolormesh(gx, gy, datm, cmap='gray', vmin=np.nanmin(datm), vmax=scalemax) #vmax=np.nanmax(datm)
del datm, dat
else:
## draw point cloud
x,y = map.projtran(humlon, humlat)
map.scatter(x.flatten(), y.flatten(), 0.5, merge.flatten(), cmap='gray', linewidth = '0')
#map.drawmapscale(x1+0.001, y1+0.001, x1, y1, 200., units='m', barstyle='fancy', labelstyle='simple', fontcolor='k') #'#F8F8FF')
#map.drawparallels(np.arange(y1-0.001, y2+0.001, 0.005),labels=[1,0,0,1], linewidth=0.0, rotation=30, fontsize=8)
#map.drawmeridians(np.arange(x1, x2, 0.002),labels=[1,0,0,1], linewidth=0.0, rotation=30, fontsize=8)
custom_save2(sonpath,'map_imagery'+str(p))
del fig
del humlat, humlon
return res #return the new resolution
def make_map(e, n, t, d, dat_port, dat_star, pix_m, res, cs2cs_args, sonpath, p, dogrid):
trans = pyproj.Proj(init=cs2cs_args)
merge = np.vstack((dat_port,dat_star))
#merge = np.vstack((np.flipud(port_fp[p]),star_fp[p]))
merge[np.isnan(merge)] = 0
merge = merge[:,:len(n)]
# get number pixels in scan line
extent = int(np.shape(merge)[0]/2)
yvec = np.linspace(pix_m,extent*pix_m,extent)
print "getting point cloud ..."
# get the points by rotating the [x,y] vector so it lines up with boat heading, assumed to be the same as the curvature of the [e,n] trace
X=[]; Y=[];
for k in range(len(n)):
x = np.concatenate((np.tile(e[k],extent) , np.tile(e[k],extent)))
#y = np.concatenate((n[k]+yvec, n[k]-yvec))
rangedist = np.sqrt(np.power(yvec, 2.0) - np.power(d[k], 2.0))
y = np.concatenate((n[k]+rangedist, n[k]-rangedist))
# Rotate line around center point
xx = e[k] - ((x - e[k]) * np.cos(t[k])) - ((y - n[k]) * np.sin(t[k]))
yy = n[k] - ((x - e[k]) * np.sin(t[k])) + ((y - n[k]) * np.cos(t[k]))
xx, yy = calc_beam_pos(d[k], t[k], xx, yy)
X.append(xx)
Y.append(yy)
del e, n, t, x, y #, X, Y
# merge flatten and stack
X = np.asarray(X,'float').T
X = X.flatten()
# merge flatten and stack
Y = np.asarray(Y,'float').T
Y = Y.flatten()
X = X[np.where(np.logical_not(np.isnan(Y)))]
merge = merge.flatten()[np.where(np.logical_not(np.isnan(Y)))]
Y = Y[np.where(np.logical_not(np.isnan(Y)))]
Y = Y[np.where(np.logical_not(np.isnan(X)))]
merge = merge.flatten()[np.where(np.logical_not(np.isnan(X)))]
X = X[np.where(np.logical_not(np.isnan(X)))]
X = X[np.where(np.logical_not(np.isnan(merge)))]
Y = Y[np.where(np.logical_not(np.isnan(merge)))]
merge = merge[np.where(np.logical_not(np.isnan(merge)))]
# write raw bs to file
outfile = sonpath+'x_y_ss_raw'+str(p)+'.asc'
with open(outfile, 'w') as f:
np.savetxt(f, np.hstack((humutils.ascol(X.flatten()),humutils.ascol(Y.flatten()), humutils.ascol(merge.flatten()))), delimiter=' ', fmt="%8.6f %8.6f %8.6f")
humlon, humlat = trans(X, Y, inverse=True)
if dogrid==1:
grid_x, grid_y = np.meshgrid( np.arange(np.min(X), np.max(X), res), np.arange(np.min(Y), np.max(Y), res) )
dat = griddata(np.c_[X.flatten(),Y.flatten()], merge.flatten(), (grid_x, grid_y), method='nearest')
## create mask for where the data is not
tree = KDTree(np.c_[X.flatten(),Y.flatten()])
dist, _ = tree.query(np.c_[grid_x.ravel(), grid_y.ravel()], k=1)
dist = dist.reshape(grid_x.shape)
del X, Y #, bearing #, pix_m, yvec
if dogrid==1:
## mask
dat[dist> np.floor(np.sqrt(1/res))-1 ] = np.nan #np.floor(np.sqrt(1/res))-1 ] = np.nan
del dist, tree
dat[dat==0] = np.nan
dat[np.isinf(dat)] = np.nan
datm = np.ma.masked_invalid(dat)
glon, glat = trans(grid_x, grid_y, inverse=True)
del grid_x, grid_y
print "drawing and printing map ..."
fig = plt.figure(frameon=False)
map = Basemap(projection='merc', epsg=cs2cs_args.split(':')[1], #26949,
resolution = 'i', #h #f
llcrnrlon=np.min(humlon)-0.001, llcrnrlat=np.min(humlat)-0.001,
urcrnrlon=np.max(humlon)+0.001, urcrnrlat=np.max(humlat)+0.001)
if dogrid==1:
gx,gy = map.projtran(glon, glat)
ax = plt.Axes(fig, [0., 0., 1., 1.], )
ax.set_axis_off()
fig.add_axes(ax)
if dogrid==1:
map.pcolormesh(gx, gy, datm, cmap='gray', vmin=np.nanmin(dat), vmax=np.nanmax(dat))
del datm, dat
else:
## draw point cloud
x,y = map.projtran(humlon, humlat)
map.scatter(x.flatten(), y.flatten(), 0.5, merge.flatten(), cmap='gray', linewidth = '0')
custom_save(sonpath,'map'+str(p))
del fig
kml = simplekml.Kml()
ground = kml.newgroundoverlay(name='GroundOverlay')
ground.icon.href = sonpath+'map'+str(p)+'.png'
ground.latlonbox.north = np.min(humlat)-0.001
ground.latlonbox.south = np.max(humlat)+0.001
ground.latlonbox.east = np.max(humlon)+0.001
ground.latlonbox.west = np.min(humlon)-0.001
ground.latlonbox.rotation = 0
kml.save(sonpath+'GroundOverlay'+str(p)+'.kml')
del humlat, humlon
def map_texture(humfile, sonpath, cs2cs_args, res, mode, nn, numstdevs): #influence = 10,
'''
Create plots of the texture lengthscale maps made in PyHum.texture module
using the algorithm detailed by Buscombe et al. (2015)
This textural lengthscale is not a direct measure of grain size. Rather, it is a statistical
representation that integrates over many attributes of bed texture, of which grain size is the most important.
The technique is a physically based means to identify regions of texture within a sidescan echogram,
and could provide a basis for objective, automated riverbed sediment classification.
Syntax
----------
[] = PyHum.map_texture(humfile, sonpath, cs2cs_args, res, mode, nn, numstdevs)
Parameters
----------
humfile : str
path to the .DAT file
sonpath : str
path where the *.SON files are
cs2cs_args : int, *optional* [Default="epsg:26949"]
arguments to create coordinates in a projected coordinate system
this argument gets given to pyproj to turn wgs84 (lat/lon) coordinates
into any projection supported by the proj.4 libraries
res : float, *optional* [Default=0.5]
grid resolution of output gridded texture map
mode: int, *optional* [Default=3]
gridding mode. 1 = nearest neighbour
2 = inverse weighted nearest neighbour
3 = Gaussian weighted nearest neighbour
nn: int, *optional* [Default=64]
number of nearest neighbours for gridding (used if mode > 1)
numstdevs: int, *optional* [Default = 4]
Threshold number of standard deviations in texture lengthscale per grid cell up to which to accept
Returns
-------
sonpath+'x_y_class'+str(p)+'.asc' : text file
contains the point cloud of easting, northing, and texture lengthscales
of the pth chunk
sonpath+'class_GroundOverlay'+str(p)+'.kml': kml file
contains gridded (or point cloud) texture lengthscale map for importing into google earth
of the pth chunk
sonpath+'class_map'+str(p)+'.png' :
image overlay associated with the kml file
sonpath+'class_map_imagery'+str(p)+'.png' : png image file
gridded (or point cloud) texture lengthscale map
overlain onto an image pulled from esri image server
References
----------
.. [1] Buscombe, D., Grams, P.E., and Smith, S.M.C., 2015, Automated riverbed sediment
classification using low-cost sidescan sonar. Journal of Hydraulic Engineering 10.1061/(ASCE)HY.1943-7900.0001079, 06015019.
'''
# prompt user to supply file if no input file given
if not humfile:
print('An input file is required!!!!!!')
Tk().withdraw() # we don't want a full GUI, so keep the root window from appearing
humfile = askopenfilename(filetypes=[("DAT files","*.DAT")])
# prompt user to supply directory if no input sonpath is given
if not sonpath:
print('A *.SON directory is required!!!!!!')
Tk().withdraw() # we don't want a full GUI, so keep the root window from appearing
sonpath = askdirectory()
# print given arguments to screen and convert data type where necessary
if humfile:
print('Input file is %s' % (humfile))
if sonpath:
print('Sonar file path is %s' % (sonpath))
if cs2cs_args:
print('cs2cs arguments are %s' % (cs2cs_args))
if res:
res = np.asarray(res,float)
print('Gridding resolution: %s' % (str(res)))
if mode:
mode = int(mode)
print('Mode for gridding: %s' % (str(mode)))
if nn:
nn = int(nn)
print('Number of nearest neighbours for gridding: %s' % (str(nn)))
#if influence:
# influence = int(influence)
# print 'Radius of influence for gridding: %s (m)' % (str(influence))
if numstdevs:
numstdevs = int(numstdevs)
print('Threshold number of standard deviations in texture lengthscale per grid cell up to which to accept: %s' % (str(numstdevs)))
# start timer
if os.name=='posix': # true if linux/mac or cygwin on windows
start = time.time()
else: # windows
start = time.clock()
trans = pyproj.Proj(init=cs2cs_args)
# if son path name supplied has no separator at end, put one on
if sonpath[-1]!=os.sep:
sonpath = sonpath + os.sep
base = humfile.split('.DAT') # get base of file name for output
base = base[0].split(os.sep)[-1]
# remove underscores, negatives and spaces from basename
base = humutils.strip_base(base)
meta = loadmat(os.path.normpath(os.path.join(sonpath,base+'meta.mat')))
esi = np.squeeze(meta['e'])
nsi = np.squeeze(meta['n'])
pix_m = np.squeeze(meta['pix_m'])*1.1
dep_m = np.squeeze(meta['dep_m'])
c = np.squeeze(meta['c'])
#dist_m = np.squeeze(meta['dist_m'])
theta = np.squeeze(meta['heading'])/(180/np.pi)
# load memory mapped scans
shape_port = np.squeeze(meta['shape_port'])
if shape_port!='':
if os.path.isfile(os.path.normpath(os.path.join(sonpath,base+'_data_port_lar.dat'))):
port_fp = io.get_mmap_data(sonpath, base, '_data_port_lar.dat', 'float32', tuple(shape_port))
else:
port_fp = io.get_mmap_data(sonpath, base, '_data_port_la.dat', 'float32', tuple(shape_port))
shape_star = np.squeeze(meta['shape_star'])
if shape_star!='':
if os.path.isfile(os.path.normpath(os.path.join(sonpath,base+'_data_star_lar.dat'))):
star_fp = io.get_mmap_data(sonpath, base, '_data_star_lar.dat', 'float32', tuple(shape_star))
else:
star_fp = io.get_mmap_data(sonpath, base, '_data_star_la.dat', 'float32', tuple(shape_star))
if len(shape_star)>2:
shape = shape_port.copy()
shape[1] = shape_port[1] + shape_star[1]
class_fp = io.get_mmap_data(sonpath, base, '_data_class.dat', 'float32', tuple(shape))
#with open(os.path.normpath(os.path.join(sonpath,base+'_data_class.dat')), 'r') as ff:
# class_fp = np.memmap(ff, dtype='float32', mode='r', shape=tuple(shape))
else:
with open(os.path.normpath(os.path.join(sonpath,base+'_data_class.dat')), 'r') as ff:
class_fp = np.load(ff)
tvg = ((8.5*10**-5)+(3/76923)+((8.5*10**-5)/4))*c
dist_tvg = ((np.tan(np.radians(25)))*dep_m)-(tvg)
if len(shape_star)>2:
for p in range(len(class_fp)):
e = esi[shape_port[-1]*p:shape_port[-1]*(p+1)]
n = nsi[shape_port[-1]*p:shape_port[-1]*(p+1)]
t = theta[shape_port[-1]*p:shape_port[-1]*(p+1)]
d = dist_tvg[shape_port[-1]*p:shape_port[-1]*(p+1)]
len_n = len(n)
merge = class_fp[p].copy()
merge[np.isnan(merge)] = 0
merge[np.isnan(np.vstack((np.flipud(port_fp[p]),star_fp[p])))] = 0
extent = shape_port[1]
R1 = merge[extent:,:]
R2 = np.flipud(merge[:extent,:])
merge = np.vstack((R2,R1))
del R1, R2
# get number pixels in scan line
extent = int(np.shape(merge)[0]/2)
yvec = np.linspace(pix_m,extent*pix_m,extent)
X, Y = getXY(e,n,yvec,d,t,extent)
merge[merge==0] = np.nan
if len(merge.flatten()) != len(X):
merge = merge[:,:len_n]
merge = merge.T.flatten()
index = np.where(np.logical_not(np.isnan(merge)))[0]
X, Y, merge = trim_xys(X, Y, merge, index)
X = X.astype('float32')
Y = Y.astype('float32')
merge = merge.astype('float32')
# write raw bs to file
outfile = os.path.normpath(os.path.join(sonpath,'x_y_class'+str(p)+'.asc'))
with open(outfile, 'w') as f:
np.savetxt(f, np.hstack((humutils.ascol(X),humutils.ascol(Y), humutils.ascol(merge))), delimiter=' ', fmt="%8.6f %8.6f %8.6f")
humlon, humlat = trans(X, Y, inverse=True)
#if dogrid==1:
orig_def, targ_def, grid_x, grid_y, res, shape = get_griddefs(np.min(X), np.max(X), np.min(Y), np.max(Y), res, humlon, humlat, trans)
grid_x = grid_x.astype('float32')
grid_y = grid_y.astype('float32')
sigmas = 1 #m
eps = 2
dat, res = get_grid(mode, orig_def, targ_def, merge, res*10, np.min(X), np.max(X), np.min(Y), np.max(Y), res, nn, sigmas, eps, shape, numstdevs, trans, humlon, humlat)
del merge
dat[dat==0] = np.nan
dat[np.isinf(dat)] = np.nan
datm = np.ma.masked_invalid(dat)
del dat
glon, glat = trans(grid_x, grid_y, inverse=True)
del grid_x, grid_y
vmin=np.nanmin(datm)+0.1
vmax=np.nanmax(datm)-0.1
if vmin > vmax:
vmin=np.nanmin(datm)
vmax=np.nanmax(datm)
print_map(cs2cs_args, glon, glat, datm, sonpath, p, vmin=vmin, vmax=vmax)
else: #just 1 chunk
e = esi
n = nsi
t = theta
d = dist_tvg
len_n = len(n)
merge = class_fp.copy()
merge[np.isnan(merge)] = 0
merge[np.isnan(np.vstack((np.flipud(port_fp),star_fp)))] = 0
extent = shape_port[0]
R1 = merge[extent:,:]
R2 = np.flipud(merge[:extent,:])
merge = np.vstack((R2,R1))
del R1, R2
# get number pixels in scan line
extent = int(np.shape(merge)[0]/2)
yvec = np.linspace(pix_m,extent*pix_m,extent)
X, Y = getXY(e,n,yvec,d,t,extent)
merge[merge==0] = np.nan
if len(merge.flatten()) != len(X):
merge = merge[:,:len_n]
merge = merge.T.flatten()
index = np.where(np.logical_not(np.isnan(merge)))[0]
X, Y, merge = trim_xys(X, Y, merge, index)
# write raw bs to file
outfile = os.path.normpath(os.path.join(sonpath,'x_y_class'+str(0)+'.asc'))
with open(outfile, 'w') as f:
np.savetxt(f, np.hstack((humutils.ascol(X),humutils.ascol(Y), humutils.ascol(merge))), delimiter=' ', fmt="%8.6f %8.6f %8.6f")
humlon, humlat = trans(X, Y, inverse=True)
#if dogrid==1:
if 2>1:
orig_def, targ_def, grid_x, grid_y, res, shape = get_griddefs(np.min(X), np.max(X), np.min(Y), np.max(Y), res, humlon, humlat, trans)
## create mask for where the data is not
tree = KDTree(np.c_[X.flatten(),Y.flatten()])
if pykdtree==1:
dist, _ = tree.query(np.c_[grid_x.ravel(), grid_y.ravel()], k=1)
else:
try:
dist, _ = tree.query(np.c_[grid_x.ravel(), grid_y.ravel()], k=1, n_jobs=cpu_count())
except:
#print ".... update your scipy installation to use faster kd-tree queries"
dist, _ = tree.query(np.c_[grid_x.ravel(), grid_y.ravel()], k=1)
dist = dist.reshape(grid_x.shape)
sigmas = 1 #m
eps = 2
dat, res = get_grid(mode, orig_def, targ_def, merge, res*10, np.min(X), np.max(X), np.min(Y), np.max(Y), res, nn, sigmas, eps, shape, numstdevs, trans, humlon, humlat)
del merge
#if dogrid==1:
if 2>1:
dat[dat==0] = np.nan
dat[np.isinf(dat)] = np.nan
dat[dist>res*2] = np.nan
del dist
datm = np.ma.masked_invalid(dat)
glon, glat = trans(grid_x, grid_y, inverse=True)
del grid_x, grid_y
vmin=np.nanmin(datm)+0.1
vmax=np.nanmax(datm)-0.1
if vmin > vmax:
vmin=np.nanmin(datm)
vmax=np.nanmax(datm)
Parallel(n_jobs = 2, verbose=0)(delayed(doplots)(k, humlon, humlat, cs2cs_args, glon, glat, datm, sonpath, 0, vmin=vmin, vmax=vmax) for k in range(2))
#print_map(cs2cs_args, glon, glat, datm, sonpath, 0, vmin=vmin, vmax=vmax)
#print_contour_map(cs2cs_args, humlon, humlat, glon, glat, datm, sonpath, 0, vmin=vmin, vmax=vmax)
if os.name=='posix': # true if linux/mac
elapsed = (time.time() - start)
else: # windows
elapsed = (time.clock() - start)
print("Processing took "+str(elapsed)+"seconds to analyse")
print("Done!")
print("===================================================")
def map_texture(humfile, sonpath, cs2cs_args, dogrid, calc_bearing, filt_bearing, res, cog, dowrite):
'''
Create plots of the texture lengthscale maps made in PyHum.texture module
using the algorithm detailed by Buscombe et al. (forthcoming)
This textural lengthscale is not a direct measure of grain size. Rather, it is a statistical
representation that integrates over many attributes of bed texture, of which grain size is the most important.
The technique is a physically based means to identify regions of texture within a sidescan echogram,
and could provide a basis for objective, automated riverbed sediment classification.
Syntax
----------
[] = PyHum.map_texture(humfile, sonpath, cs2cs_args, dogrid, calc_bearing, filt_bearing, res, cog, dowrite)
Parameters
----------
humfile : str
path to the .DAT file
sonpath : str
path where the *.SON files are
cs2cs_args : int, *optional* [Default="epsg:26949"]
arguments to create coordinates in a projected coordinate system
this argument gets given to pyproj to turn wgs84 (lat/lon) coordinates
into any projection supported by the proj.4 libraries
dogrid : float, *optional* [Default=1]
if 1, textures will be gridded with resolution 'res'.
Otherwise, point cloud will be plotted
calc_bearing : float, *optional* [Default=0]
if 1, bearing will be calculated from coordinates
filt_bearing : float, *optional* [Default=0]
if 1, bearing will be filtered
res : float, *optional* [Default=0.1]
grid resolution of output gridded texture map
cog : int, *optional* [Default=1]
if 1, heading calculated assuming GPS course-over-ground rather than
using a compass
dowrite: int, *optional* [Default=1]
if 1, point cloud data from each chunk is written to ascii file
if 0, processing times are speeded up considerably but point clouds are not available for further analysis
Returns
-------
sonpath+'x_y_class'+str(p)+'.asc' : text file
contains the point cloud of easting, northing, and texture lengthscales
of the pth chunk
sonpath+'class_GroundOverlay'+str(p)+'.kml': kml file
contains gridded (or point cloud) texture lengthscale map for importing into google earth
of the pth chunk
sonpath+'class_map'+str(p)+'.png' :
image overlay associated with the kml file
sonpath+'class_map_imagery'+str(p)+'.png' : png image file
gridded (or point cloud) texture lengthscale map
overlain onto an image pulled from esri image server
References
----------
.. [1] Buscombe, D., Grams, P.E., and Smith, S.M.C., Automated riverbed sediment
classification using low-cost sidescan sonar. submitted to
Journal of Hydraulic Engineering
'''
# prompt user to supply file if no input file given
if not humfile:
print 'An input file is required!!!!!!'
Tk().withdraw() # we don't want a full GUI, so keep the root window from appearing
inputfile = askopenfilename(filetypes=[("DAT files","*.DAT")])
# prompt user to supply directory if no input sonpath is given
if not sonpath:
print 'A *.SON directory is required!!!!!!'
Tk().withdraw() # we don't want a full GUI, so keep the root window from appearing
sonpath = askdirectory()
# print given arguments to screen and convert data type where necessary
if humfile:
print 'Input file is %s' % (humfile)
if sonpath:
print 'Sonar file path is %s' % (sonpath)
if cs2cs_args:
print 'cs2cs arguments are %s' % (cs2cs_args)
if dogrid:
dogrid = int(dogrid)
if dogrid==1:
print "Data will be gridded"
if calc_bearing:
calc_bearing = int(calc_bearing)
if calc_bearing==1:
print "Bearing will be calculated from coordinates"
if filt_bearing:
filt_bearing = int(filt_bearing)
if filt_bearing==1:
print "Bearing will be filtered"
if res:
res = np.asarray(res,float)
print 'Gridding resolution: %s' % (str(res))
if cog:
cog = int(cog)
if cog==1:
print "Heading based on course-over-ground"
if dowrite:
dowrite = int(dowrite)
if dowrite==0:
print "Point cloud data will be written to ascii file"
if not cs2cs_args:
# arguments to pass to cs2cs for coordinate transforms
cs2cs_args = "epsg:26949"
print '[Default] cs2cs arguments are %s' % (cs2cs_args)
if not dogrid:
if dogrid != 0:
dogrid = 1
print "[Default] Data will be gridded"
if not calc_bearing:
if calc_bearing != 1:
calc_bearing = 0
print "[Default] Heading recorded by instrument will be used"
if not filt_bearing:
if filt_bearing != 1:
filt_bearing = 0
print "[Default] Heading will not be filtered"
if not res:
res = 0.5
print '[Default] Grid resolution is %s m' % (str(res))
if not cog:
if cog != 0:
cog = 1
print "[Default] Heading based on course-over-ground"
if not dowrite:
if dowrite != 0:
dowrite = 1
print "[Default] Point cloud data will be written to ascii file"
trans = pyproj.Proj(init=cs2cs_args)
# if son path name supplied has no separator at end, put one on
if sonpath[-1]!=os.sep:
sonpath = sonpath + os.sep
base = humfile.split('.DAT') # get base of file name for output
base = base[0].split(os.sep)[-1]
# remove underscores, negatives and spaces from basename
if base.find('_')>-1:
base = base[:base.find('_')]
if base.find('-')>-1:
base = base[:base.find('-')]
if base.find(' ')>-1:
base = base[:base.find(' ')]
esi = np.squeeze(loadmat(sonpath+base+'meta.mat')['e'])
nsi = np.squeeze(loadmat(sonpath+base+'meta.mat')['n'])
pix_m = np.squeeze(loadmat(sonpath+base+'meta.mat')['pix_m'])
dep_m = np.squeeze(loadmat(sonpath+base+'meta.mat')['dep_m'])
c = np.squeeze(loadmat(sonpath+base+'meta.mat')['c'])
dist_m = np.squeeze(loadmat(sonpath+base+'meta.mat')['dist_m'])
# over-ride measured bearing and calc from positions
if calc_bearing==1:
lat = np.squeeze(loadmat(sonpath+base+'meta.mat')['lat'])
lon = np.squeeze(loadmat(sonpath+base+'meta.mat')['lon'])
#point-to-point bearing
bearing = np.zeros(len(lat))
for k in xrange(len(lat)-1):
bearing[k] = bearingBetweenPoints(lat[k], lat[k+1], lon[k], lon[k+1])
del lat, lon
else:
# reported bearing by instrument (Kalman filtered?)
bearing = np.squeeze(loadmat(sonpath+base+'meta.mat')['heading'])
## bearing can only be observed modulo 2*pi, therefore phase unwrap
#bearing = np.unwrap(bearing)
# if stdev in heading is large, there's probably noise that needs to be filtered out
if np.std(bearing)>180:
print "WARNING: large heading stdev - attempting filtering"
from sklearn.cluster import MiniBatchKMeans
# can have two modes
data = np.column_stack([bearing, bearing])
k_means = MiniBatchKMeans(2)
# fit the model
k_means.fit(data)
values = k_means.cluster_centers_.squeeze()
labels = k_means.labels_
if np.sum(labels==0) > np.sum(labels==1):
bearing[labels==1] = np.nan
else:
bearing[labels==0] = np.nan
nans, y= humutils.nan_helper(bearing)
bearing[nans]= np.interp(y(nans), y(~nans), bearing[~nans])
if filt_bearing ==1:
bearing = humutils.runningMeanFast(bearing, len(bearing)/100)
theta = np.asarray(bearing, 'float')/(180/np.pi)
# this is standard course over ground
if cog==1:
#course over ground is given as a compass heading (ENU) from True north, or Magnetic north.
#To get this into NED (North-East-Down) coordinates, you need to rotate the ENU
# (East-North-Up) coordinate frame.
#Subtract pi/2 from your heading
theta = theta - np.pi/2
# (re-wrap to Pi to -Pi)
theta = np.unwrap(-theta)
# load memory mapped scans
shape_port = np.squeeze(loadmat(sonpath+base+'meta.mat')['shape_port'])
if shape_port!='':
port_fp = np.memmap(sonpath+base+'_data_port_l.dat', dtype='float32', mode='r', shape=tuple(shape_port))
shape_star = np.squeeze(loadmat(sonpath+base+'meta.mat')['shape_star'])
if shape_star!='':
star_fp = np.memmap(sonpath+base+'_data_star_l.dat', dtype='float32', mode='r', shape=tuple(shape_star))
shape = shape_port.copy()
shape[1] = shape_port[1] + shape_star[1]
class_fp = np.memmap(sonpath+base+'_data_class.dat', dtype='float32', mode='r', shape=tuple(shape))
tvg = ((8.5*10**-5)+(3/76923)+((8.5*10**-5)/4))*c
dist_tvg = ((np.tan(np.radians(25)))*dep_m)-(tvg)
for p in xrange(len(class_fp)):
e = esi[shape_port[-1]*p:shape_port[-1]*(p+1)]
n = nsi[shape_port[-1]*p:shape_port[-1]*(p+1)]
t = theta[shape_port[-1]*p:shape_port[-1]*(p+1)]
d = dist_tvg[shape_port[-1]*p:shape_port[-1]*(p+1)]
len_n = len(n)
merge = class_fp[p].copy()
merge[np.isnan(merge)] = 0
merge[np.isnan(np.vstack((np.flipud(port_fp[p]),star_fp[p])))] = 0
extent = shape_port[1]
R1 = merge[extent:,:]
R2 = np.flipud(merge[:extent,:])
merge = np.vstack((R2,R1))
del R1, R2
# get number pixels in scan line
extent = int(np.shape(merge)[0]/2)
yvec = np.linspace(pix_m,extent*pix_m,extent)
print "getting point cloud ..."
# get the points by rotating the [x,y] vector so it lines up with boat heading
X=[]; Y=[];
for k in range(len(n)):
x = np.concatenate((np.tile(e[k],extent) , np.tile(e[k],extent)))
#y = np.concatenate((n[k]+yvec, n[k]-yvec))
rangedist = np.sqrt(np.power(yvec, 2.0) - np.power(d[k], 2.0))
y = np.concatenate((n[k]+rangedist, n[k]-rangedist))
# Rotate line around center point
xx = e[k] - ((x - e[k]) * np.cos(t[k])) - ((y - n[k]) * np.sin(t[k]))
yy = n[k] - ((x - e[k]) * np.sin(t[k])) + ((y - n[k]) * np.cos(t[k]))
xx, yy = calc_beam_pos(d[k], t[k], xx, yy)
X.append(xx)
Y.append(yy)
del e, n, t, x, y
# merge flatten and stack
X = np.asarray(X,'float')
X = X.flatten()
# merge flatten and stack
Y = np.asarray(Y,'float')
Y = Y.flatten()
merge[merge==0] = np.nan
if len(merge.flatten()) != len(X):
merge = merge[:,:len_n]
merge = merge.T.flatten()
index = np.where(np.logical_not(np.isnan(merge)))[0]
X = X.flatten()[index]
Y = Y.flatten()[index]
merge = merge.flatten()[index]
X = X[np.where(np.logical_not(np.isnan(Y)))]
merge = merge.flatten()[np.where(np.logical_not(np.isnan(Y)))]
Y = Y[np.where(np.logical_not(np.isnan(Y)))]
Y = Y[np.where(np.logical_not(np.isnan(X)))]
merge = merge.flatten()[np.where(np.logical_not(np.isnan(X)))]
X = X[np.where(np.logical_not(np.isnan(X)))]
X = X[np.where(np.logical_not(np.isnan(merge)))]
Y = Y[np.where(np.logical_not(np.isnan(merge)))]
merge = merge[np.where(np.logical_not(np.isnan(merge)))]
if dowrite==1:
# write raw bs to file
outfile = sonpath+'x_y_class'+str(p)+'.asc'
with open(outfile, 'w') as f:
np.savetxt(f, np.hstack((humutils.ascol(X),humutils.ascol(Y), humutils.ascol(merge))), delimiter=' ', fmt="%8.6f %8.6f %8.6f")
humlon, humlat = trans(X, Y, inverse=True)
if dogrid==1:
grid_x, grid_y = np.meshgrid( np.arange(np.min(X), np.max(X), res), np.arange(np.min(Y), np.max(Y), res) )
dat = griddata(np.c_[X.flatten(),Y.flatten()], merge.flatten(), (grid_x, grid_y), method='nearest')
## create mask for where the data is not
tree = KDTree(np.c_[X.flatten(),Y.flatten()])
dist, _ = tree.query(np.c_[grid_x.ravel(), grid_y.ravel()], k=1)
dist = dist.reshape(grid_x.shape)
del X, Y
if dogrid==1:
## mask
dat[dist> 1 ] = np.nan
del dist, tree
dat[dat==0] = np.nan
dat[np.isinf(dat)] = np.nan
datm = np.ma.masked_invalid(dat)
glon, glat = trans(grid_x, grid_y, inverse=True)
del grid_x, grid_y
try:
print "drawing and printing map ..."
fig = plt.figure(frameon=False)
map = Basemap(projection='merc', epsg=cs2cs_args.split(':')[1], #26949,
resolution = 'i', #h #f
llcrnrlon=np.min(humlon)-0.001, llcrnrlat=np.min(humlat)-0.001,
urcrnrlon=np.max(humlon)+0.001, urcrnrlat=np.max(humlat)+0.001)
if dogrid==1:
gx,gy = map.projtran(glon, glat)
ax = plt.Axes(fig, [0., 0., 1., 1.], )
ax.set_axis_off()
fig.add_axes(ax)
if dogrid==1:
map.pcolormesh(gx, gy, datm, cmap='YlOrRd', vmin=0.5, vmax=2)
del dat
else:
## draw point cloud
x,y = map.projtran(humlon, humlat)
map.scatter(x.flatten(), y.flatten(), 0.5, merge.flatten(), cmap='YlOrRd', linewidth = '0')
custom_save(sonpath,'class_map'+str(p))
del fig
except:
print "error: map could not be created..."
kml = simplekml.Kml()
ground = kml.newgroundoverlay(name='GroundOverlay')
ground.icon.href = 'class_map'+str(p)+'.png'
ground.latlonbox.north = np.min(humlat)-0.001
ground.latlonbox.south = np.max(humlat)+0.001
ground.latlonbox.east = np.max(humlon)+0.001
ground.latlonbox.west = np.min(humlon)-0.001
ground.latlonbox.rotation = 0
kml.save(sonpath+'class_GroundOverlay'+str(p)+'.kml')
if dowrite==1:
X = []; Y = []; S = [];
for p in xrange(len(class_fp)):
dat = np.genfromtxt(sonpath+'x_y_class'+str(p)+'.asc', delimiter=' ')
X.append(dat[:,0])
Y.append(dat[:,1])
S.append(dat[:,2])
del dat
# merge flatten and stack
X = np.asarray(np.hstack(X),'float')
X = X.flatten()
# merge flatten and stack
Y = np.asarray(np.hstack(Y),'float')
Y = Y.flatten()
# merge flatten and stack
S = np.asarray(np.hstack(S),'float')
S = S.flatten()
humlon, humlat = trans(X, Y, inverse=True)
if dogrid==1:
grid_x, grid_y = np.meshgrid( np.arange(np.min(X), np.max(X), res), np.arange(np.min(Y), np.max(Y), res) )
dat = griddata(np.c_[X.flatten(),Y.flatten()], S.flatten(), (grid_x, grid_y), method='nearest')
## create mask for where the data is not
tree = KDTree(np.c_[X.flatten(),Y.flatten()])
dist, _ = tree.query(np.c_[grid_x.ravel(), grid_y.ravel()], k=1)
dist = dist.reshape(grid_x.shape)
del X, Y
if dogrid==1:
## mask
dat[dist> 1 ] = np.nan
del dist, tree
dat[dat==0] = np.nan
dat[np.isinf(dat)] = np.nan
datm = np.ma.masked_invalid(dat)
glon, glat = trans(grid_x, grid_y, inverse=True)
del grid_x, grid_y
levels = [0.5,0.75,1.25,1.5,1.75,2,3]
try:
print "drawing and printing map ..."
fig = plt.figure()
map = Basemap(projection='merc', epsg=cs2cs_args.split(':')[1],
resolution = 'i',
llcrnrlon=np.min(humlon)-0.001, llcrnrlat=np.min(humlat)-0.001,
urcrnrlon=np.max(humlon)+0.001, urcrnrlat=np.max(humlat)+0.001)
map.arcgisimage(server='http://server.arcgisonline.com/ArcGIS', service='World_Imagery', xpixels=1000, ypixels=None, dpi=300)
if dogrid==1:
gx,gy = map.projtran(glon, glat)
if dogrid==1:
map.contourf(gx, gy, datm, levels, cmap='YlOrRd')
else:
## draw point cloud
x,y = map.projtran(humlon, humlat)
map.scatter(x.flatten(), y.flatten(), 0.5, S.flatten(), cmap='YlOrRd', linewidth = '0')
custom_save2(sonpath,'class_map_imagery'+str(p))
del fig
except:
print "error: map could not be created..."
def map_texture(humfile, sonpath, cs2cs_args = "epsg:26949", res = 0.5, mode=3, nn = 64, numstdevs=5): #influence = 10,
'''
Create plots of the texture lengthscale maps made in PyHum.texture module
using the algorithm detailed by Buscombe et al. (2015)
This textural lengthscale is not a direct measure of grain size. Rather, it is a statistical
representation that integrates over many attributes of bed texture, of which grain size is the most important.
The technique is a physically based means to identify regions of texture within a sidescan echogram,
and could provide a basis for objective, automated riverbed sediment classification.
Syntax
----------
[] = PyHum.map_texture(humfile, sonpath, cs2cs_args, res, mode, nn, numstdevs)
Parameters
----------
humfile : str
path to the .DAT file
sonpath : str
path where the *.SON files are
cs2cs_args : int, *optional* [Default="epsg:26949"]
arguments to create coordinates in a projected coordinate system
this argument gets given to pyproj to turn wgs84 (lat/lon) coordinates
into any projection supported by the proj.4 libraries
res : float, *optional* [Default=0.5]
grid resolution of output gridded texture map
mode: int, *optional* [Default=3]
gridding mode. 1 = nearest neighbour
2 = inverse weighted nearest neighbour
3 = Gaussian weighted nearest neighbour
nn: int, *optional* [Default=64]
number of nearest neighbours for gridding (used if mode > 1)
numstdevs: int, *optional* [Default = 4]
Threshold number of standard deviations in texture lengthscale per grid cell up to which to accept
Returns
-------
sonpath+'x_y_class'+str(p)+'.asc' : text file
contains the point cloud of easting, northing, and texture lengthscales
of the pth chunk
sonpath+'class_GroundOverlay'+str(p)+'.kml': kml file
contains gridded (or point cloud) texture lengthscale map for importing into google earth
of the pth chunk
sonpath+'class_map'+str(p)+'.png' :
image overlay associated with the kml file
sonpath+'class_map_imagery'+str(p)+'.png' : png image file
gridded (or point cloud) texture lengthscale map
overlain onto an image pulled from esri image server
References
----------
.. [1] Buscombe, D., Grams, P.E., and Smith, S.M.C., 2015, Automated riverbed sediment
classification using low-cost sidescan sonar. Journal of Hydraulic Engineering 10.1061/(ASCE)HY.1943-7900.0001079, 06015019.
'''
# prompt user to supply file if no input file given
if not humfile:
print 'An input file is required!!!!!!'
Tk().withdraw() # we don't want a full GUI, so keep the root window from appearing
humfile = askopenfilename(filetypes=[("DAT files","*.DAT")])
# prompt user to supply directory if no input sonpath is given
if not sonpath:
print 'A *.SON directory is required!!!!!!'
Tk().withdraw() # we don't want a full GUI, so keep the root window from appearing
sonpath = askdirectory()
# print given arguments to screen and convert data type where necessary
if humfile:
print 'Input file is %s' % (humfile)
if sonpath:
print 'Sonar file path is %s' % (sonpath)
if cs2cs_args:
print 'cs2cs arguments are %s' % (cs2cs_args)
if res:
res = np.asarray(res,float)
print 'Gridding resolution: %s' % (str(res))
if mode:
mode = int(mode)
print 'Mode for gridding: %s' % (str(mode))
if nn:
nn = int(nn)
print 'Number of nearest neighbours for gridding: %s' % (str(nn))
#if influence:
# influence = int(influence)
# print 'Radius of influence for gridding: %s (m)' % (str(influence))
if numstdevs:
numstdevs = int(numstdevs)
print 'Threshold number of standard deviations in texture lengthscale per grid cell up to which to accept: %s' % (str(numstdevs))
# start timer
if os.name=='posix': # true if linux/mac or cygwin on windows
start = time.time()
else: # windows
start = time.clock()
trans = pyproj.Proj(init=cs2cs_args)
# if son path name supplied has no separator at end, put one on
if sonpath[-1]!=os.sep:
sonpath = sonpath + os.sep
base = humfile.split('.DAT') # get base of file name for output
base = base[0].split(os.sep)[-1]
# remove underscores, negatives and spaces from basename
base = humutils.strip_base(base)
meta = loadmat(os.path.normpath(os.path.join(sonpath,base+'meta.mat')))
esi = np.squeeze(meta['e'])
nsi = np.squeeze(meta['n'])
pix_m = np.squeeze(meta['pix_m'])
dep_m = np.squeeze(meta['dep_m'])
c = np.squeeze(meta['c'])
#dist_m = np.squeeze(meta['dist_m'])
theta = np.squeeze(meta['heading'])/(180/np.pi)
# load memory mapped scans
shape_port = np.squeeze(meta['shape_port'])
if shape_port!='':
if os.path.isfile(os.path.normpath(os.path.join(sonpath,base+'_data_port_lar.dat'))):
port_fp = io.get_mmap_data(sonpath, base, '_data_port_lar.dat', 'float32', tuple(shape_port))
else:
port_fp = io.get_mmap_data(sonpath, base, '_data_port_la.dat', 'float32', tuple(shape_port))
shape_star = np.squeeze(meta['shape_star'])
if shape_star!='':
if os.path.isfile(os.path.normpath(os.path.join(sonpath,base+'_data_star_lar.dat'))):
star_fp = io.get_mmap_data(sonpath, base, '_data_star_lar.dat', 'float32', tuple(shape_star))
else:
star_fp = io.get_mmap_data(sonpath, base, '_data_star_la.dat', 'float32', tuple(shape_star))
if len(shape_star)>2:
shape = shape_port.copy()
shape[1] = shape_port[1] + shape_star[1]
class_fp = io.get_mmap_data(sonpath, base, '_data_class.dat', 'float32', tuple(shape))
#with open(os.path.normpath(os.path.join(sonpath,base+'_data_class.dat')), 'r') as ff:
# class_fp = np.memmap(ff, dtype='float32', mode='r', shape=tuple(shape))
else:
with open(os.path.normpath(os.path.join(sonpath,base+'_data_class.dat')), 'r') as ff:
class_fp = np.load(ff)
tvg = ((8.5*10**-5)+(3/76923)+((8.5*10**-5)/4))*c
dist_tvg = ((np.tan(np.radians(25)))*dep_m)-(tvg)
if len(shape_star)>2:
for p in xrange(len(class_fp)):
e = esi[shape_port[-1]*p:shape_port[-1]*(p+1)]
n = nsi[shape_port[-1]*p:shape_port[-1]*(p+1)]
t = theta[shape_port[-1]*p:shape_port[-1]*(p+1)]
d = dist_tvg[shape_port[-1]*p:shape_port[-1]*(p+1)]
len_n = len(n)
merge = class_fp[p].copy()
merge[np.isnan(merge)] = 0
merge[np.isnan(np.vstack((np.flipud(port_fp[p]),star_fp[p])))] = 0
extent = shape_port[1]
R1 = merge[extent:,:]
R2 = np.flipud(merge[:extent,:])
merge = np.vstack((R2,R1))
del R1, R2
# get number pixels in scan line
extent = int(np.shape(merge)[0]/2)
yvec = np.linspace(pix_m,extent*pix_m,extent)
X, Y = getXY(e,n,yvec,d,t,extent)
merge[merge==0] = np.nan
if len(merge.flatten()) != len(X):
merge = merge[:,:len_n]
merge = merge.T.flatten()
index = np.where(np.logical_not(np.isnan(merge)))[0]
X, Y, merge = trim_xys(X, Y, merge, index)
X = X.astype('float32')
Y = Y.astype('float32')
merge = merge.astype('float32')
# write raw bs to file
outfile = os.path.normpath(os.path.join(sonpath,'x_y_class'+str(p)+'.asc'))
with open(outfile, 'w') as f:
np.savetxt(f, np.hstack((humutils.ascol(X),humutils.ascol(Y), humutils.ascol(merge))), delimiter=' ', fmt="%8.6f %8.6f %8.6f")
humlon, humlat = trans(X, Y, inverse=True)
#if dogrid==1:
orig_def, targ_def, grid_x, grid_y, res, shape = get_griddefs(np.min(X), np.max(X), np.min(Y), np.max(Y), res, humlon, humlat, trans)
grid_x = grid_x.astype('float32')
grid_y = grid_y.astype('float32')
sigmas = 1 #m
eps = 2
dat, res = get_grid(mode, orig_def, targ_def, merge, res*10, np.min(X), np.max(X), np.min(Y), np.max(Y), res, nn, sigmas, eps, shape, numstdevs, trans, humlon, humlat)
del merge
dat[dat==0] = np.nan
dat[np.isinf(dat)] = np.nan
datm = np.ma.masked_invalid(dat)
del dat
glon, glat = trans(grid_x, grid_y, inverse=True)
del grid_x, grid_y
vmin=np.nanmin(datm)+0.1
vmax=np.nanmax(datm)-0.1
if vmin > vmax:
vmin=np.nanmin(datm)
vmax=np.nanmax(datm)
print_map(cs2cs_args, glon, glat, datm, sonpath, p, vmin=vmin, vmax=vmax)
else: #just 1 chunk
e = esi
n = nsi
t = theta
d = dist_tvg
len_n = len(n)
merge = class_fp.copy()
merge[np.isnan(merge)] = 0
merge[np.isnan(np.vstack((np.flipud(port_fp),star_fp)))] = 0
extent = shape_port[0]
R1 = merge[extent:,:]
R2 = np.flipud(merge[:extent,:])
merge = np.vstack((R2,R1))
del R1, R2
# get number pixels in scan line
extent = int(np.shape(merge)[0]/2)
yvec = np.linspace(pix_m,extent*pix_m,extent)
X, Y = getXY(e,n,yvec,d,t,extent)
merge[merge==0] = np.nan
if len(merge.flatten()) != len(X):
merge = merge[:,:len_n]
merge = merge.T.flatten()
index = np.where(np.logical_not(np.isnan(merge)))[0]
X, Y, merge = trim_xys(X, Y, merge, index)
# write raw bs to file
outfile = os.path.normpath(os.path.join(sonpath,'x_y_class'+str(0)+'.asc'))
with open(outfile, 'w') as f:
np.savetxt(f, np.hstack((humutils.ascol(X),humutils.ascol(Y), humutils.ascol(merge))), delimiter=' ', fmt="%8.6f %8.6f %8.6f")
humlon, humlat = trans(X, Y, inverse=True)
#if dogrid==1:
if 2>1:
orig_def, targ_def, grid_x, grid_y, res, shape = get_griddefs(np.min(X), np.max(X), np.min(Y), np.max(Y), res, humlon, humlat, trans)
## create mask for where the data is not
tree = KDTree(np.c_[X.flatten(),Y.flatten()])
if pykdtree==1:
dist, _ = tree.query(np.c_[grid_x.ravel(), grid_y.ravel()], k=1)
else:
try:
dist, _ = tree.query(np.c_[grid_x.ravel(), grid_y.ravel()], k=1, n_jobs=cpu_count())
except:
#print ".... update your scipy installation to use faster kd-tree queries"
dist, _ = tree.query(np.c_[grid_x.ravel(), grid_y.ravel()], k=1)
dist = dist.reshape(grid_x.shape)
sigmas = 1 #m
eps = 2
dat, res = get_grid(mode, orig_def, targ_def, merge, res*10, np.min(X), np.max(X), np.min(Y), np.max(Y), res, nn, sigmas, eps, shape, numstdevs, trans, humlon, humlat)
del merge
#if dogrid==1:
if 2>1:
dat[dat==0] = np.nan
dat[np.isinf(dat)] = np.nan
dat[dist>res*2] = np.nan
del dist
datm = np.ma.masked_invalid(dat)
glon, glat = trans(grid_x, grid_y, inverse=True)
del grid_x, grid_y
vmin=np.nanmin(datm)+0.1
vmax=np.nanmax(datm)-0.1
if vmin > vmax:
vmin=np.nanmin(datm)
vmax=np.nanmax(datm)
Parallel(n_jobs = 2, verbose=0)(delayed(doplots)(k, humlon, humlat, cs2cs_args, glon, glat, datm, sonpath, 0, vmin=vmin, vmax=vmax) for k in xrange(2))
#print_map(cs2cs_args, glon, glat, datm, sonpath, 0, vmin=vmin, vmax=vmax)
#print_contour_map(cs2cs_args, humlon, humlat, glon, glat, datm, sonpath, 0, vmin=vmin, vmax=vmax)
if os.name=='posix': # true if linux/mac
elapsed = (time.time() - start)
else: # windows
elapsed = (time.clock() - start)
print "Processing took ", elapsed , "seconds to analyse"
print "Done!"
def make_map(e, n, t, d, dat_port, dat_star, data_R, pix_m, res, cs2cs_args, sonpath, p, dogrid, dowrite, mode, nn, influence, numstdevs):
trans = pyproj.Proj(init=cs2cs_args)
merge = np.vstack((dat_port,dat_star))
#merge = np.vstack((np.flipud(port_fp[p]),star_fp[p]))
merge[np.isnan(merge)] = 0
merge = merge[:,:len(n)]
R = np.vstack((np.flipud(data_R),data_R))
R = R[:np.shape(merge)[0],:np.shape(merge)[1]]
# get number pixels in scan line
extent = int(np.shape(merge)[0]/2)
yvec = np.linspace(pix_m,extent*pix_m,extent)
X, Y, D, h, t = getXY(e,n,yvec,d,t,extent)
D[np.isnan(D)] = 0
h[np.isnan(h)] = 0
t[np.isnan(t)] = 0
X = X[np.where(np.logical_not(np.isnan(Y)))]
merge = merge.flatten()[np.where(np.logical_not(np.isnan(Y)))]
Y = Y[np.where(np.logical_not(np.isnan(Y)))]
D = D[np.where(np.logical_not(np.isnan(Y)))]
R = R.flatten()[np.where(np.logical_not(np.isnan(Y)))]
h = h[np.where(np.logical_not(np.isnan(Y)))]
t = t[np.where(np.logical_not(np.isnan(Y)))]
Y = Y[np.where(np.logical_not(np.isnan(X)))]
merge = merge.flatten()[np.where(np.logical_not(np.isnan(X)))]
X = X[np.where(np.logical_not(np.isnan(X)))]
D = D[np.where(np.logical_not(np.isnan(X)))]
R = R.flatten()[np.where(np.logical_not(np.isnan(X)))]
h = h[np.where(np.logical_not(np.isnan(X)))]
t = t[np.where(np.logical_not(np.isnan(X)))]
X = X[np.where(np.logical_not(np.isnan(merge)))]
Y = Y[np.where(np.logical_not(np.isnan(merge)))]
merge = merge[np.where(np.logical_not(np.isnan(merge)))]
D = D[np.where(np.logical_not(np.isnan(merge)))]
R = R[np.where(np.logical_not(np.isnan(merge)))]
h = h[np.where(np.logical_not(np.isnan(merge)))]
t = t[np.where(np.logical_not(np.isnan(merge)))]
if dowrite==1:
## write raw bs to file
outfile = os.path.normpath(os.path.join(sonpath,'x_y_ss_raw'+str(p)+'.asc'))
write.txtwrite( outfile, np.hstack((humutils.ascol(X.flatten()),humutils.ascol(Y.flatten()), humutils.ascol(merge.flatten()), humutils.ascol(D.flatten()), humutils.ascol(np.cos(R.flatten())), humutils.ascol(h.flatten()), humutils.ascol(t.flatten()) )) )
del D, R, h, t
humlon, humlat = trans(X, Y, inverse=True)
if dogrid==1:
grid_x, grid_y = np.meshgrid( np.arange(np.min(X), np.max(X), res), np.arange(np.min(Y), np.max(Y), res) )
#del X, Y
longrid, latgrid = trans(grid_x, grid_y, inverse=True)
shape = np.shape(grid_x)
#del grid_y, grid_x
targ_def = pyresample.geometry.SwathDefinition(lons=longrid.flatten(), lats=latgrid.flatten())
del longrid, latgrid
orig_def = pyresample.geometry.SwathDefinition(lons=humlon.flatten(), lats=humlat.flatten())
#del humlat, humlon
#influence = 1 #m
if mode==1:
try:
# nearest neighbour
dat = pyresample.kd_tree.resample_nearest(orig_def, merge.flatten(), targ_def, radius_of_influence=influence, fill_value=None, nprocs = cpu_count())
except:
print "Memory error: trying a grid resolution twice as big"
res = res*2
grid_x, grid_y = np.meshgrid( np.arange(np.min(X), np.max(X), res), np.arange(np.min(Y), np.max(Y), res) )
#del X, Y
longrid, latgrid = trans(grid_x, grid_y, inverse=True)
shape = np.shape(grid_x)
#del grid_y, grid_x
targ_def = pyresample.geometry.SwathDefinition(lons=longrid.flatten(), lats=latgrid.flatten())
del longrid, latgrid
orig_def = pyresample.geometry.SwathDefinition(lons=humlon.flatten(), lats=humlat.flatten())
dat = pyresample.kd_tree.resample_nearest(orig_def, merge.flatten(), targ_def, radius_of_influence=influence, fill_value=None, nprocs = cpu_count())
elif mode==2:
# custom inverse distance
wf = lambda r: 1/r**2
try:
dat, stdev, counts = pyresample.kd_tree.resample_custom(orig_def, merge.flatten(),targ_def, radius_of_influence=influence, neighbours=nn, weight_funcs=wf, fill_value=None, with_uncert = True, nprocs = cpu_count())
except:
print "Memory error: trying a grid resolution twice as big"
res = res*2
grid_x, grid_y = np.meshgrid( np.arange(np.min(X), np.max(X), res), np.arange(np.min(Y), np.max(Y), res) )
#del X, Y
longrid, latgrid = trans(grid_x, grid_y, inverse=True)
shape = np.shape(grid_x)
#del grid_y, grid_x
targ_def = pyresample.geometry.SwathDefinition(lons=longrid.flatten(), lats=latgrid.flatten())
del longrid, latgrid
orig_def = pyresample.geometry.SwathDefinition(lons=humlon.flatten(), lats=humlat.flatten())
dat, stdev, counts = pyresample.kd_tree.resample_custom(orig_def, merge.flatten(),targ_def, radius_of_influence=influence, neighbours=nn, weight_funcs=wf, fill_value=None, with_uncert = True, nprocs = cpu_count())
elif mode==3:
sigmas = 1 #m
eps = 2
try:
dat, stdev, counts = pyresample.kd_tree.resample_gauss(orig_def, merge.flatten(), targ_def, radius_of_influence=influence, neighbours=nn, sigmas=sigmas, fill_value=None, with_uncert = np.nan, nprocs = cpu_count(), epsilon = eps)
except:
print "Memory error: trying a grid resolution twice as big"
res = res*2
grid_x, grid_y = np.meshgrid( np.arange(np.min(X), np.max(X), res), np.arange(np.min(Y), np.max(Y), res) )
#del X, Y
longrid, latgrid = trans(grid_x, grid_y, inverse=True)
shape = np.shape(grid_x)
#del grid_y, grid_x
targ_def = pyresample.geometry.SwathDefinition(lons=longrid.flatten(), lats=latgrid.flatten())
del longrid, latgrid
orig_def = pyresample.geometry.SwathDefinition(lons=humlon.flatten(), lats=humlat.flatten())
dat, stdev, counts = pyresample.kd_tree.resample_gauss(orig_def, merge.flatten(), targ_def, radius_of_influence=influence, neighbours=nn, sigmas=sigmas, fill_value=None, with_uncert = np.nan, nprocs = cpu_count(), epsilon = eps)
del X, Y
dat = dat.reshape(shape)
if mode>1:
stdev = stdev.reshape(shape)
counts = counts.reshape(shape)
mask = dat.mask.copy()
dat[mask==1] = 0
if mode>1:
dat[(stdev>numstdevs) & (mask!=0)] = np.nan
dat[(counts<nn) & (counts>0)] = np.nan
dat2 = replace_nans.RN(dat.astype('float64'),1000,0.01,2,'localmean').getdata()
dat2[dat==0] = np.nan
# get a new mask
mask = np.isnan(dat2)
mask = ~binary_dilation(binary_erosion(~mask,structure=np.ones((15,15))), structure=np.ones((15,15)))
#mask = binary_fill_holes(mask, structure=np.ones((15,15)))
#mask = ~binary_fill_holes(~mask, structure=np.ones((15,15)))
dat2[mask==1] = np.nan
dat2[dat2<1] = np.nan
del dat
dat = dat2
del dat2
if dogrid==1:
### mask
#if np.floor(np.sqrt(1/res))-1 > 0.0:
# dat[dist> np.floor(np.sqrt(1/res))-1 ] = np.nan #np.floor(np.sqrt(1/res))-1 ] = np.nan
#else:
# dat[dist> np.sqrt(1/res) ] = np.nan #np.floor(np.sqrt(1/res))-1 ] = np.nan
#del dist, tree
dat[dat==0] = np.nan
dat[np.isinf(dat)] = np.nan
datm = np.ma.masked_invalid(dat)
glon, glat = trans(grid_x, grid_y, inverse=True)
del grid_x, grid_y
try:
print "drawing and printing map ..."
fig = plt.figure(frameon=False)
map = Basemap(projection='merc', epsg=cs2cs_args.split(':')[1], #26949,
resolution = 'i', #h #f
llcrnrlon=np.min(humlon)-0.00001, llcrnrlat=np.min(humlat)-0.00001,
urcrnrlon=np.max(humlon)+0.00001, urcrnrlat=np.max(humlat)+0.00001)
if dogrid==1:
gx,gy = map.projtran(glon, glat)
ax = plt.Axes(fig, [0., 0., 1., 1.], )
ax.set_axis_off()
fig.add_axes(ax)
if dogrid==1:
map.pcolormesh(gx, gy, datm, cmap='gray', vmin=np.nanmin(datm), vmax=np.nanmax(datm))
del datm, dat
else:
## draw point cloud
x,y = map.projtran(humlon, humlat)
map.scatter(x.flatten(), y.flatten(), 0.5, merge.flatten(), cmap='gray', linewidth = '0')
custom_save(sonpath,'map'+str(p))
del fig
except:
print "error: map could not be created..."
kml = simplekml.Kml()
ground = kml.newgroundoverlay(name='GroundOverlay')
ground.icon.href = 'map'+str(p)+'.png'
ground.latlonbox.north = np.min(humlat)-0.00001
ground.latlonbox.south = np.max(humlat)+0.00001
ground.latlonbox.east = np.max(humlon)+0.00001
ground.latlonbox.west = np.min(humlon)-0.00001
ground.latlonbox.rotation = 0
#kml.save(sonpath+'GroundOverlay'+str(p)+'.kml')
kml.save(os.path.normpath(os.path.join(sonpath,'GroundOverlay'+str(p)+'.kml')))
del humlat, humlon
def map_texture(humfile, sonpath, cs2cs_args = "epsg:26949", dogrid = 1, res = 0.5, dowrite = 0, mode=3, nn = 64, influence = 1, numstdevs=5):
'''
Create plots of the texture lengthscale maps made in PyHum.texture module
using the algorithm detailed by Buscombe et al. (forthcoming)
This textural lengthscale is not a direct measure of grain size. Rather, it is a statistical
representation that integrates over many attributes of bed texture, of which grain size is the most important.
The technique is a physically based means to identify regions of texture within a sidescan echogram,
and could provide a basis for objective, automated riverbed sediment classification.
Syntax
----------
[] = PyHum.map_texture(humfile, sonpath, cs2cs_args, dogrid, res, dowrite, mode, nn, influence, numstdevs)
Parameters
----------
humfile : str
path to the .DAT file
sonpath : str
path where the *.SON files are
cs2cs_args : int, *optional* [Default="epsg:26949"]
arguments to create coordinates in a projected coordinate system
this argument gets given to pyproj to turn wgs84 (lat/lon) coordinates
into any projection supported by the proj.4 libraries
dogrid : float, *optional* [Default=1]
if 1, textures will be gridded with resolution 'res'.
Otherwise, point cloud will be plotted
res : float, *optional* [Default=0.5]
grid resolution of output gridded texture map
dowrite: int, *optional* [Default=0]
if 1, point cloud data from each chunk is written to ascii file
if 0, processing times are speeded up considerably but point clouds are not available for further analysis
mode: int, *optional* [Default=3]
gridding mode. 1 = nearest neighbour
2 = inverse weighted nearest neighbour
3 = Gaussian weighted nearest neighbour
nn: int, *optional* [Default=64]
number of nearest neighbours for gridding (used if mode > 1)
influence: float, *optional* [Default=1]
Radius of influence used in gridding. Cut off distance in meters
numstdevs: int, *optional* [Default = 4]
Threshold number of standard deviations in texture lengthscale per grid cell up to which to accept
Returns
-------
sonpath+'x_y_class'+str(p)+'.asc' : text file
contains the point cloud of easting, northing, and texture lengthscales
of the pth chunk
sonpath+'class_GroundOverlay'+str(p)+'.kml': kml file
contains gridded (or point cloud) texture lengthscale map for importing into google earth
of the pth chunk
sonpath+'class_map'+str(p)+'.png' :
image overlay associated with the kml file
sonpath+'class_map_imagery'+str(p)+'.png' : png image file
gridded (or point cloud) texture lengthscale map
overlain onto an image pulled from esri image server
References
----------
.. [1] Buscombe, D., Grams, P.E., and Smith, S.M.C., 2015, Automated riverbed sediment
classification using low-cost sidescan sonar. Journal of Hydraulic Engineering, accepted
'''
# prompt user to supply file if no input file given
if not humfile:
print 'An input file is required!!!!!!'
Tk().withdraw() # we don't want a full GUI, so keep the root window from appearing
inputfile = askopenfilename(filetypes=[("DAT files","*.DAT")])
# prompt user to supply directory if no input sonpath is given
if not sonpath:
print 'A *.SON directory is required!!!!!!'
Tk().withdraw() # we don't want a full GUI, so keep the root window from appearing
sonpath = askdirectory()
# print given arguments to screen and convert data type where necessary
if humfile:
print 'Input file is %s' % (humfile)
if sonpath:
print 'Sonar file path is %s' % (sonpath)
if cs2cs_args:
print 'cs2cs arguments are %s' % (cs2cs_args)
if dogrid:
dogrid = int(dogrid)
if dogrid==1:
print "Data will be gridded"
if res:
res = np.asarray(res,float)
print 'Gridding resolution: %s' % (str(res))
if dowrite:
dowrite = int(dowrite)
if dowrite==0:
print "Point cloud data will be written to ascii file"
if mode:
mode = int(mode)
print 'Mode for gridding: %s' % (str(mode))
if nn:
nn = int(nn)
print 'Number of nearest neighbours for gridding: %s' % (str(nn))
if influence:
influence = int(influence)
print 'Radius of influence for gridding: %s (m)' % (str(influence))
if numstdevs:
numstdevs = int(numstdevs)
print 'Threshold number of standard deviations in texture lengthscale per grid cell up to which to accept: %s' % (str(numstdevs))
trans = pyproj.Proj(init=cs2cs_args)
# if son path name supplied has no separator at end, put one on
if sonpath[-1]!=os.sep:
sonpath = sonpath + os.sep
base = humfile.split('.DAT') # get base of file name for output
base = base[0].split(os.sep)[-1]
# remove underscores, negatives and spaces from basename
if base.find('_')>-1:
base = base[:base.find('_')]
if base.find('-')>-1:
base = base[:base.find('-')]
if base.find(' ')>-1:
base = base[:base.find(' ')]
meta = loadmat(os.path.normpath(os.path.join(sonpath,base+'meta.mat')))
esi = np.squeeze(meta['e'])
nsi = np.squeeze(meta['n'])
pix_m = np.squeeze(meta['pix_m'])
dep_m = np.squeeze(meta['dep_m'])
c = np.squeeze(meta['c'])
dist_m = np.squeeze(meta['dist_m'])
theta = np.squeeze(meta['heading'])/(180/np.pi)
# load memory mapped scans
shape_port = np.squeeze(meta['shape_port'])
if shape_port!='':
#port_fp = np.memmap(sonpath+base+'_data_port_l.dat', dtype='float32', mode='r', shape=tuple(shape_port))
if os.path.isfile(os.path.normpath(os.path.join(sonpath,base+'_data_port_lar.dat'))):
with open(os.path.normpath(os.path.join(sonpath,base+'_data_port_lar.dat')), 'r') as ff:
port_fp = np.memmap(ff, dtype='float32', mode='r', shape=tuple(shape_port))
else:
with open(os.path.normpath(os.path.join(sonpath,base+'_data_port_la.dat')), 'r') as ff:
port_fp = np.memmap(ff, dtype='float32', mode='r', shape=tuple(shape_port))
shape_star = np.squeeze(meta['shape_star'])
if shape_star!='':
#star_fp = np.memmap(sonpath+base+'_data_star_l.dat', dtype='float32', mode='r', shape=tuple(shape_star))
if os.path.isfile(os.path.normpath(os.path.join(sonpath,base+'_data_star_lar.dat'))):
with open(os.path.normpath(os.path.join(sonpath,base+'_data_star_lar.dat')), 'r') as ff:
star_fp = np.memmap(ff, dtype='float32', mode='r', shape=tuple(shape_star))
else:
with open(os.path.normpath(os.path.join(sonpath,base+'_data_star_la.dat')), 'r') as ff:
star_fp = np.memmap(ff, dtype='float32', mode='r', shape=tuple(shape_star))
shape = shape_port.copy()
shape[1] = shape_port[1] + shape_star[1]
#class_fp = np.memmap(sonpath+base+'_data_class.dat', dtype='float32', mode='r', shape=tuple(shape))
with open(os.path.normpath(os.path.join(sonpath,base+'_data_class.dat')), 'r') as ff:
class_fp = np.memmap(ff, dtype='float32', mode='r', shape=tuple(shape))
tvg = ((8.5*10**-5)+(3/76923)+((8.5*10**-5)/4))*c
dist_tvg = ((np.tan(np.radians(25)))*dep_m)-(tvg)
for p in xrange(len(class_fp)):
e = esi[shape_port[-1]*p:shape_port[-1]*(p+1)]
n = nsi[shape_port[-1]*p:shape_port[-1]*(p+1)]
t = theta[shape_port[-1]*p:shape_port[-1]*(p+1)]
d = dist_tvg[shape_port[-1]*p:shape_port[-1]*(p+1)]
len_n = len(n)
merge = class_fp[p].copy()
merge[np.isnan(merge)] = 0
merge[np.isnan(np.vstack((np.flipud(port_fp[p]),star_fp[p])))] = 0
extent = shape_port[1]
R1 = merge[extent:,:]
R2 = np.flipud(merge[:extent,:])
merge = np.vstack((R2,R1))
del R1, R2
# get number pixels in scan line
extent = int(np.shape(merge)[0]/2)
yvec = np.linspace(pix_m,extent*pix_m,extent)
X, Y = getXY(e,n,yvec,d,t,extent)
merge[merge==0] = np.nan
if len(merge.flatten()) != len(X):
merge = merge[:,:len_n]
merge = merge.T.flatten()
index = np.where(np.logical_not(np.isnan(merge)))[0]
X = X.flatten()[index]
Y = Y.flatten()[index]
merge = merge.flatten()[index]
X = X[np.where(np.logical_not(np.isnan(Y)))]
merge = merge.flatten()[np.where(np.logical_not(np.isnan(Y)))]
Y = Y[np.where(np.logical_not(np.isnan(Y)))]
Y = Y[np.where(np.logical_not(np.isnan(X)))]
merge = merge.flatten()[np.where(np.logical_not(np.isnan(X)))]
X = X[np.where(np.logical_not(np.isnan(X)))]
X = X[np.where(np.logical_not(np.isnan(merge)))]
Y = Y[np.where(np.logical_not(np.isnan(merge)))]
merge = merge[np.where(np.logical_not(np.isnan(merge)))]
if dowrite==1:
# write raw bs to file
#outfile = sonpath+'x_y_class'+str(p)+'.asc'
outfile = os.path.normpath(os.path.join(sonpath,'x_y_class'+str(p)+'.asc'))
with open(outfile, 'w') as f:
np.savetxt(f, np.hstack((humutils.ascol(X),humutils.ascol(Y), humutils.ascol(merge))), delimiter=' ', fmt="%8.6f %8.6f %8.6f")
humlon, humlat = trans(X, Y, inverse=True)
if dogrid==1:
grid_x, grid_y = np.meshgrid( np.arange(np.min(X), np.max(X), res), np.arange(np.min(Y), np.max(Y), res) )
del X, Y
longrid, latgrid = trans(grid_x, grid_y, inverse=True)
shape = np.shape(grid_x)
#del grid_y, grid_x
targ_def = pyresample.geometry.SwathDefinition(lons=longrid.flatten(), lats=latgrid.flatten())
del longrid, latgrid
orig_def = pyresample.geometry.SwathDefinition(lons=humlon.flatten(), lats=humlat.flatten())
#del humlat, humlon
#influence = 1 #m
#numneighbours = 64
if mode==1:
try:
# nearest neighbour
dat = pyresample.kd_tree.resample_nearest(orig_def, merge.flatten(), targ_def, radius_of_influence=influence, fill_value=None, nprocs = cpu_count())
except:
# nearest neighbour
dat, stdev, counts = pyresample.kd_tree.resample_nearest(orig_def, merge.flatten(), targ_def, radius_of_influence=influence, fill_value=None, with_uncert = True, nprocs = 1)
elif mode==2:
# custom inverse distance
wf = lambda r: 1/r**2
try:
dat, stdev, counts = pyresample.kd_tree.resample_custom(orig_def, merge.flatten(),targ_def, radius_of_influence=influence, neighbours=nn, weight_funcs=wf, fill_value=None, with_uncert = True, nprocs = cpu_count())
except:
dat, stdev, counts = pyresample.kd_tree.resample_custom(orig_def, merge.flatten(),targ_def, radius_of_influence=influence, neighbours=nn, weight_funcs=wf, fill_value=None, with_uncert = True, nprocs = 1)
elif mode==3:
sigmas = 1 #m
eps = 2
try:
dat, stdev, counts = pyresample.kd_tree.resample_gauss(orig_def, merge.flatten(), targ_def, radius_of_influence=influence, neighbours=nn, sigmas=sigmas, fill_value=None, with_uncert = np.nan, nprocs = cpu_count(), epsilon = eps)
except:
dat, stdev, counts = pyresample.kd_tree.resample_gauss(orig_def, merge.flatten(), targ_def, radius_of_influence=influence, neighbours=nn, sigmas=sigmas, fill_value=None, with_uncert = np.nan, nprocs = 1, epsilon = eps)
dat = dat.reshape(shape)
if mode>1:
stdev = stdev.reshape(shape)
counts = counts.reshape(shape)
mask = dat.mask.copy()
dat[mask==1] = 0
if mode>1:
dat[(stdev>3) & (mask!=0)] = np.nan
dat[(counts<nn) & (counts>0)] = np.nan
dat2 = replace_nans.RN(dat.astype('float64'),1000,0.01,2,'localmean').getdata()
dat2[dat==0] = np.nan
# get a new mask
mask = np.isnan(dat2)
mask = ~binary_dilation(binary_erosion(~mask,structure=np.ones((15,15))), structure=np.ones((15,15)))
#mask = binary_fill_holes(mask, structure=np.ones((15,15)))
#mask = ~binary_fill_holes(~mask, structure=np.ones((15,15)))
dat2[mask==1] = np.nan
dat2[dat2<1] = np.nan
del dat
dat = dat2
del dat2
if dogrid==1:
## mask
#dat[dist> 1 ] = np.nan
#del dist, tree
dat[dat==0] = np.nan
dat[np.isinf(dat)] = np.nan
datm = np.ma.masked_invalid(dat)
glon, glat = trans(grid_x, grid_y, inverse=True)
del grid_x, grid_y
try:
print "drawing and printing map ..."
fig = plt.figure(frameon=False)
map = Basemap(projection='merc', epsg=cs2cs_args.split(':')[1], #26949,
resolution = 'i', #h #f
llcrnrlon=np.min(humlon)-0.0001, llcrnrlat=np.min(humlat)-0.0001,
urcrnrlon=np.max(humlon)+0.0001, urcrnrlat=np.max(humlat)+0.0001)
if dogrid==1:
gx,gy = map.projtran(glon, glat)
ax = plt.Axes(fig, [0., 0., 1., 1.], )
ax.set_axis_off()
fig.add_axes(ax)
if dogrid==1:
map.pcolormesh(gx, gy, datm, cmap='YlOrRd', vmin=0.25, vmax=2)
del dat
else:
## draw point cloud
x,y = map.projtran(humlon, humlat)
map.scatter(x.flatten(), y.flatten(), 0.5, merge.flatten(), cmap='YlOrRd', linewidth = '0')
custom_save(sonpath,'class_map'+str(p))
del fig
except:
print "error: map could not be created..."
kml = simplekml.Kml()
ground = kml.newgroundoverlay(name='GroundOverlay')
ground.icon.href = 'class_map'+str(p)+'.png'
ground.latlonbox.north = np.min(humlat)-0.00001
ground.latlonbox.south = np.max(humlat)+0.00001
ground.latlonbox.east = np.max(humlon)+0.00001
ground.latlonbox.west = np.min(humlon)-0.00001
ground.latlonbox.rotation = 0
#kml.save(sonpath+'class_GroundOverlay'+str(p)+'.kml')
kml.save(os.path.normpath(os.path.join(sonpath,'class_GroundOverlay'+str(p)+'.kml')))
if dowrite==1:
X = []; Y = []; S = [];
for p in xrange(len(class_fp)):
#dat = np.genfromtxt(sonpath+'x_y_class'+str(p)+'.asc', delimiter=' ')
dat = np.genfromtxt(os.path.normpath(os.path.join(sonpath,'x_y_class'+str(p)+'.asc')), delimiter=' ')
X.append(dat[:,0])
Y.append(dat[:,1])
S.append(dat[:,2])
del dat
# merge flatten and stack
X = np.asarray(np.hstack(X),'float')
X = X.flatten()
# merge flatten and stack
Y = np.asarray(np.hstack(Y),'float')
Y = Y.flatten()
# merge flatten and stack
S = np.asarray(np.hstack(S),'float')
S = S.flatten()
humlon, humlat = trans(X, Y, inverse=True)
if dogrid==1:
grid_x, grid_y = np.meshgrid( np.arange(np.min(X), np.max(X), res), np.arange(np.min(Y), np.max(Y), res) )
del X, Y
longrid, latgrid = trans(grid_x, grid_y, inverse=True)
shape = np.shape(grid_x)
#del grid_y, grid_x
targ_def = pyresample.geometry.SwathDefinition(lons=longrid.flatten(), lats=latgrid.flatten())
del longrid, latgrid
orig_def = pyresample.geometry.SwathDefinition(lons=humlon.flatten(), lats=humlat.flatten())
#del humlat, humlon
#influence = 1 #m
#numneighbours = 64
if mode==1:
try:
# nearest neighbour
dat = pyresample.kd_tree.resample_nearest(orig_def, merge.flatten(), targ_def, radius_of_influence=influence, fill_value=None, nprocs = cpu_count())
except:
# nearest neighbour
dat, stdev, counts = pyresample.kd_tree.resample_nearest(orig_def, merge.flatten(), targ_def, radius_of_influence=influence, fill_value=None, with_uncert = True, nprocs = 1)
elif mode==2:
# custom inverse distance
wf = lambda r: 1/r**2
try:
dat, stdev, counts = pyresample.kd_tree.resample_custom(orig_def, merge.flatten(),targ_def, radius_of_influence=influence, neighbours=nn, weight_funcs=wf, fill_value=None, with_uncert = True, nprocs = cpu_count())
except:
dat, stdev, counts = pyresample.kd_tree.resample_custom(orig_def, merge.flatten(),targ_def, radius_of_influence=influence, neighbours=nn, weight_funcs=wf, fill_value=None, with_uncert = True, nprocs = 1)
elif mode==3:
sigmas = 1 #m
eps = 2
try:
dat, stdev, counts = pyresample.kd_tree.resample_gauss(orig_def, merge.flatten(), targ_def, radius_of_influence=influence, neighbours=nn, sigmas=sigmas, fill_value=None, with_uncert = np.nan, nprocs = cpu_count(), epsilon = eps)
except:
dat, stdev, counts = pyresample.kd_tree.resample_gauss(orig_def, merge.flatten(), targ_def, radius_of_influence=influence, neighbours=nn, sigmas=sigmas, fill_value=None, with_uncert = np.nan, nprocs = 1, epsilon = eps)
dat = dat.reshape(shape)
if mode>1:
stdev = stdev.reshape(shape)
counts = counts.reshape(shape)
mask = dat.mask.copy()
dat[mask==1] = 0
if mode>1:
dat[(stdev>5) & (mask!=0)] = np.nan
dat[(counts<nn) & (counts>0)] = np.nan
dat2 = replace_nans.RN(dat.astype('float64'),1000,0.01,2,'localmean').getdata()
dat2[dat==0] = np.nan
# get a new mask
mask = np.isnan(dat2)
mask = ~binary_dilation(binary_erosion(~mask,structure=np.ones((15,15))), structure=np.ones((15,15)))
#mask = binary_fill_holes(mask, structure=np.ones((15,15)))
#mask = ~binary_fill_holes(~mask, structure=np.ones((15,15)))
dat2[mask==1] = np.nan
dat2[dat2<1] = np.nan
del dat
dat = dat2
del dat2
if dogrid==1:
## mask
#dat[dist> 1 ] = np.nan
#el dist, tree
dat[dat==0] = np.nan
dat[np.isinf(dat)] = np.nan
datm = np.ma.masked_invalid(dat)
glon, glat = trans(grid_x, grid_y, inverse=True)
del grid_x, grid_y
levels = [0.5,0.75,1.25,1.5,1.75,2,3]
try:
print "drawing and printing map ..."
fig = plt.figure()
map = Basemap(projection='merc', epsg=cs2cs_args.split(':')[1],
resolution = 'i',
llcrnrlon=np.min(humlon)-0.00001, llcrnrlat=np.min(humlat)-0.00001,
urcrnrlon=np.max(humlon)+0.00001, urcrnrlat=np.max(humlat)+0.00001)
map.arcgisimage(server='http://server.arcgisonline.com/ArcGIS', service='World_Imagery', xpixels=1000, ypixels=None, dpi=300)
if dogrid==1:
gx,gy = map.projtran(glon, glat)
if dogrid==1:
map.contourf(gx, gy, datm, levels, cmap='YlOrRd')
else:
## draw point cloud
x,y = map.projtran(humlon, humlat)
map.scatter(x.flatten(), y.flatten(), 0.5, S.flatten(), cmap='YlOrRd', linewidth = '0')
custom_save2(sonpath,'class_map_imagery'+str(p))
del fig
except:
print "error: map could not be created..."
Y = Y[np.where(np.logical_not(np.isnan(X)))]
merge = merge.flatten()[np.where(np.logical_not(np.isnan(X)))]
X = X[np.where(np.logical_not(np.isnan(X)))]
X = X[np.where(np.logical_not(np.isnan(merge)))]
Y = Y[np.where(np.logical_not(np.isnan(merge)))]
merge = merge[np.where(np.logical_not(np.isnan(merge)))]
# plot to check
#plt.scatter(X[::20],Y[::20],10,merge[::20], linewidth=0)
print "writing point cloud"
## write raw bs to file
outfile = os.path.normpath(os.path.join(sonpath,'x_y_slicsegmentnumber.asc'))
np.savetxt(outfile, np.hstack((humutils.ascol(X.flatten()),humutils.ascol(Y.flatten()), humutils.ascol(merge.flatten()) )) , fmt="%8.6f %8.6f %8.6f")
trans = pyproj.Proj(init="epsg:26949")
humlon, humlat = trans(X, Y, inverse=True)
res = 0.25
orig_def, targ_def, grid_x, grid_y, res, shape = get_griddefs(np.min(X), np.max(X), np.min(Y), np.max(Y), res, humlon, humlat, trans)
grid_x = grid_x.astype('float32')
grid_y = grid_y.astype('float32')
sigmas = 1 #m
eps = 2
mode = 1
print 'Now Gridding slic superpixels...'