def main(args):
'''
Core functionality. Called by qc_wrap.py and __file__
'''
try:
pargs = parser.parse_args(args[1:])
except Exception as error_msg:
print(str(error_msg))
return 1
kmname = constants.get_tilename(pargs.las_file)
print('Running %s on block: %s, %s' % (PROGNAME, kmname, time.asctime()))
if not os.path.exists(pargs.outdir):
os.mkdir(pargs.outdir)
path = pargs.las_file
filename = os.path.basename(path)
out_path = os.path.join(pargs.outdir, filename)
las_in = laspy.read(path)
las_out = laspy.LasData(las_in.header)
points = las_in.points
las_out.points = points
xy = np.column_stack((las_in.x, las_in.y))
geoid = grid.fromGDAL(GEOID_GRID, upcast=True)
geoid_offset = geoid.interpolate(xy)
# Apply vertical offset from geoid grid
las_out.z -= geoid_offset
las_out.write(out_path)
return 0
def main(args):
'''
Main function that can be called from either the command line or via qc_wrap.py
'''
try:
pargs = parser.parse_args(args[1:])
except Exception as error_msg:
print(str(error_msg))
return 1
kmname = constants.get_tilename(pargs.las_file)
print("Running %s on block: %s, %s" % (PROGNAME, kmname, time.asctime()))
if pargs.below_poly:
below_poly = True
ptype = "below_poly"
else:
below_poly = False
if pargs.type is not None:
ptype = pargs.type
else:
ptype = "undefined"
if below_poly:
print("Only using points which lie below polygon mean z!")
pc = pointcloud.fromAny(pargs.las_file)
print("Classes in pointcloud: %s" % pc.get_classes())
try:
extent = np.asarray(constants.tilename_to_extent(kmname))
except Exception:
print("Could not get extent from tilename.")
extent = None
polygons = vector_io.get_geometries(pargs.ref_data, pargs.layername,
pargs.layersql, extent)
feature_count = 0
use_local = pargs.use_local
if pargs.schema is not None:
report.set_schema(pargs.schema)
reporter = report.ReportClassCheck(use_local)
for polygon in polygons:
if below_poly:
if polygon.GetCoordinateDimension() < 3:
print(
"Error: polygon not 3D - below_poly does not make sense!")
continue
a_polygon3d = array_geometry.ogrpoly2array(polygon,
flatten=False)[0]
#warping loop here....
if pargs.toE:
geoid = grid.fromGDAL(GEOID_GRID, upcast=True)
print("Using geoid from %s to warp to ellipsoidal heights." %
GEOID_GRID)
toE = geoid.interpolate(a_polygon3d[:, :2].copy())
mask = toE == geoid.nd_val
if mask.any():
raise Warning(
"Warping to ellipsoidal heights produced no-data values!"
)
a_polygon3d[:, 2] += toE
mean_z = a_polygon3d[:, 2].mean()
if mean_z < SENSIBLE_Z_MIN or mean_z > SENSIBLE_Z_MAX:
msg = "Warning: This feature seems to have unrealistic mean z value: {0:.2f} m"
print(msg.format(mean_z))
continue
del a_polygon3d
else:
mean_z = -1
polygon.FlattenTo2D()
feature_count += 1
separator = "-" * 70
print("%s\nFeature %d\n%s" % (separator, feature_count, separator))
a_polygon = array_geometry.ogrpoly2array(polygon)
pc_in_poly = pc.cut_to_polygon(a_polygon)
if below_poly:
pc_in_poly = pc_in_poly.cut_to_z_interval(-999, mean_z)
n_all = pc_in_poly.get_size()
freqs = [0] * (len(constants.classes) + 1) #list of frequencies...
if n_all > 0:
c_all = pc_in_poly.get_classes()
if below_poly and DEBUG:
print("Mean z of polygon is: %.2f m" % mean_z)
print("Mean z of points below is: %.2f m" %
pc_in_poly.z.mean())
print("Number of points in polygon: %d" % n_all)
print("Classes in polygon: %s" % str(c_all))
# important for reporting that the order here is the same as in the table
# definition in report.py!!
n_found = 0
for i, cls in enumerate(constants.classes):
if cls in c_all:
pcc = pc_in_poly.cut_to_class(cls)
n_c = pcc.get_size()
f_c = n_c / float(n_all)
n_found += n_c
print("Class %d::" % cls)
print(" #Points: %d" % n_c)
print(" Fraction: %.3f" % f_c)
freqs[i] = f_c
f_other = (n_all - n_found) / float(n_all)
freqs[-1] = f_other
send_args = [kmname] + freqs + [n_all, ptype]
reporter.report(*send_args, ogr_geom=polygon)
def main(args):
'''
Main processing function
'''
pargs = parser.parse_args(args[1:])
lasname = pargs.las_file
kmname = constants.get_tilename(lasname)
layer_def = pargs.layer_def
fargs = dict.fromkeys(NAMES, None)
if pargs.layer_def is not None:
if layer_def.endswith(".json"):
with open(layer_def) as layer_def_file:
jargs = json.load(layer_def_file)
else:
jargs = json.loads(layer_def)
fargs.update(jargs)
for name in NAMES:
res_type = NAMES[name]
if fargs[name] is not None:
try:
fargs[name] = res_type(fargs[name])
except TypeError as error_msg:
print(str(error_msg))
print(name + " must be convertable to %s" % repr(res_type))
try:
extent = np.asarray(constants.tilename_to_extent(kmname))
except (ValueError, AttributeError) as error_msg:
print("Exception: %s" % str(error_msg))
print("Bad 1km formatting of las file: %s" % lasname)
return 1
extent_buf = extent + (-BUFBUF, -BUFBUF, BUFBUF, BUFBUF)
cell_buf_extent = np.array([-CELL_BUF, -CELL_BUF, CELL_BUF, CELL_BUF], dtype=np.float64)
grid_buf = (extent + cell_buf_extent * pargs.cell_size)
buf_georef = [grid_buf[0], pargs.cell_size, 0, grid_buf[3], 0, -pargs.cell_size]
#move these to a method in e.g. grid.py
ncols = int(ceil((grid_buf[2] - grid_buf[0]) / pargs.cell_size))
nrows = int(ceil((grid_buf[3] - grid_buf[1]) / pargs.cell_size))
assert (extent_buf[:2] < grid_buf[:2]).all()
assert modf((extent[2] - extent[0]) / pargs.cell_size)[0] == 0.0
if not os.path.exists(pargs.output_dir):
os.mkdir(pargs.output_dir)
terrainname = os.path.join(pargs.output_dir, "dtm_" + kmname + ".tif")
surfacename = os.path.join(pargs.output_dir, "dsm_" + kmname + ".tif")
terrain_exists = os.path.exists(terrainname)
surface_exists = os.path.exists(surfacename)
if pargs.dsm:
do_dsm = pargs.overwrite or (not surface_exists)
else:
do_dsm = False
if do_dsm:
do_dtm = True
else:
do_dtm = pargs.dtm and (pargs.overwrite or (not terrain_exists))
if not (do_dtm or do_dsm):
print("dtm already exists: %s" % terrain_exists)
print("dsm already exists: %s" % surface_exists)
print("Nothing to do - exiting...")
return 2
#### warn on smoothing #####
if pargs.smooth_rad > CELL_BUF:
print("Warning: smoothing radius is larger than grid buffer")
tiles = get_neighbours(pargs.tile_cstr, kmname, pargs.rowcol_sql, pargs.tile_sql, pargs.remove_bridges_in_dtm)
bufpc = None
geoid = grid.fromGDAL(GEOID_GRID, upcast=True)
for path, ground_cls, surf_cls, h_system in tiles:
if os.path.exists(path):
#check sanity
assert set(ground_cls).issubset(set(surf_cls))
assert h_system in ["dvr90", "E"]
tile_pc = pointcloud.fromAny(path, include_return_number=True)
tile_pc = tile_pc.cut_to_box(*extent_buf)
tile_pc = tile_pc.cut_to_class(surf_cls)
if tile_pc.get_size() > 0:
mask = np.zeros((tile_pc.get_size(),), dtype=np.bool)
#reclass hack
for cls in ground_cls:
mask |= (tile_pc.c == cls)
tile_pc.c[mask] = SYNTH_TERRAIN
#warping to hsys
if h_system != pargs.hsys and not pargs.nowarp:
if pargs.hsys == "E":
tile_pc.toE(geoid)
else:
tile_pc.toH(geoid)
if bufpc is None:
bufpc = tile_pc
else:
bufpc.extend(tile_pc)
del tile_pc
else:
print("Neighbour " + path + " does not exist.")
if bufpc is None:
return 3
if bufpc.get_size() <= 3:
return 3
rc1 = 0
rc2 = 0
dtm = None
dsm = None
triangle_mask = np.ndarray(0)
water_mask, lake_raster, sea_mask, build_mask = setup_masks(fargs, nrows, ncols, buf_georef)
# Remove terrain points in buildings
if pargs.clean_buildings and (build_mask is not None) and build_mask.any():
bmask_shrink = image.morphology.binary_erosion(build_mask)
mask = bufpc.get_grid_mask(bmask_shrink, buf_georef)
#validate thoroughly
testpc1 = bufpc.cut(mask) # only building points(?)
testpc2 = None
try:
testpc1.sort_spatially(2)
mask &= (bufpc.c == SYNTH_TERRAIN)
testpc2 = bufpc.cut(mask) # building points and terrain(?)
in_building = ((testpc1.max_filter(2, xy=testpc2.xy, nd_val=ND_VAL) - testpc2.z) > 1)
cut_buildings = np.zeros_like(mask, dtype=np.bool)
if in_building.any():
cut_buildings[mask] = in_building
#so see if these are really, really inside buildings
bufpc = bufpc.cut(np.logical_not(cut_buildings))
except:
pass
finally:
if testpc1 is not None:
del testpc1
if testpc2 is not None:
del testpc2
if do_dtm:
terr_pc = bufpc.cut_to_class(SYNTH_TERRAIN)
if terr_pc.get_size() > 3:
dtm, trig_grid = gridit(terr_pc, grid_buf, pargs.cell_size, None, doround=pargs.round)
else:
rc1 = 3
if dtm and not rc1:
assert dtm.grid.shape == (nrows, ncols) #else something is horribly wrong...
# Create a mask with triangles larger than triangle_limit.
# Small triangles will be ignored and possibly filled out later.
triangle_mask = trig_grid.grid > pargs.triangle_limit
else:
rc1 = 3
if triangle_mask.any() and water_mask.any() and not rc1:
if not pargs.no_expand_water:
water_mask = expand_water(triangle_mask, water_mask)
if build_mask is not None:
water_mask &= np.logical_not(build_mask) #xor
# Filling in large triangles
mask = np.logical_and(triangle_mask, water_mask)
zlow = array_geometry.tri_filter_low(
terr_pc.z,
terr_pc.triangulation.vertices,
terr_pc.triangulation.ntrig,
pargs.zlim)
if pargs.debug:
debug_difference = terr_pc.z - zlow
print(debug_difference.mean(), (debug_difference != 0).sum())
terr_pc.z = zlow
dtm_low, trig_grid = gridit(terr_pc, grid_buf, pargs.cell_size, None, doround=pargs.round)
dtm.grid[mask] = dtm_low.grid[mask]
del dtm_low
# Smooth water
if pargs.flatten:
flat_grid = array_geometry.masked_mean_filter(dtm.grid, mask, 4)
dtm.grid[triangle_mask] = flat_grid[triangle_mask]
#FIX THIS PART
if pargs.smooth_rad > 0 and build_mask is not None and triangle_mask.any():
# Smoothing below houses (probably)...
mask = np.logical_and(triangle_mask, build_mask)
dilated_mask = image.morphology.binary_dilation(mask)
dilated_mask &= np.logical_not(water_mask)
flat_grid = array_geometry.masked_mean_filter(dtm.grid, dilated_mask, pargs.smooth_rad)
mask &= np.logical_not(water_mask)
dtm.grid[mask] = flat_grid[mask]
del flat_grid
del trig_grid
del dilated_mask
del mask
if pargs.burn_sea and (sea_mask is not None) and not rc1:
dtm = burn_sea(dtm, sea_mask, triangle_mask, pargs.sea_z, pargs.sea_tolerance)
# Burn lakes
if lake_raster is not None and not rc1:
burn_lakes(dtm, lake_raster, triangle_mask, pargs.lake_tolerance_dtm)
if pargs.dtm and (pargs.overwrite or (not terrain_exists)):
dtm.shrink(CELL_BUF).save(terrainname, dco=TIF_CREATION_OPTIONS, srs=SRS_WKT)
del triangle_mask
del terr_pc
if do_dsm:
surf_pc = bufpc.cut_to_return_number(1)
del bufpc
if surf_pc.get_size() > 3:
dsm, trig_grid = gridit(surf_pc, grid_buf, pargs.cell_size, None, doround=pargs.round)
else:
rc2 = 3
if dsm and not rc2:
triangle_mask = trig_grid.grid > pargs.triangle_limit
else:
rc2 = 3
#now we are in a position to handle water...
if dtm and water_mask.any() and triangle_mask.any() and not rc2:
# Fill large triangles
mask = np.logical_and(triangle_mask, water_mask)
dsm.grid[mask] = dtm.grid[mask]
if pargs.debug:
print(dsm.grid.shape)
t_name = os.path.join(
pargs.output_dir, "triangles_" + kmname + ".tif")
trig_grid.shrink(CELL_BUF).save(
t_name,
dco=["TILED=YES", "COMPRESS=LZW"])
w_name = os.path.join(
pargs.output_dir, "water_" + kmname + ".tif")
water_grid = grid.Grid(water_mask, dsm.geo_ref, 0)
water_grid.shrink(CELL_BUF).save(w_name, dco=["TILED=YES", "COMPRESS=LZW"])
if pargs.burn_sea and (sea_mask is not None):
dsm = burn_sea(dsm, sea_mask, triangle_mask, pargs.sea_z, pargs.sea_tolerance)
# Burn lakes
if lake_raster is not None:
burn_lakes(dsm, lake_raster, triangle_mask, pargs.lake_tolerance_dsm)
del triangle_mask
dsm.shrink(CELL_BUF).save(surfacename, dco=TIF_CREATION_OPTIONS, srs=SRS_WKT)
del surf_pc
return max(rc1, rc2)
def main(args):
try:
pargs = parser.parse_args(args[1:])
except Exception as e:
print(str(e))
return 1
kmname = constants.get_tilename(pargs.las_file)
print("Running %s on block: %s, %s" % (progname, kmname, time.asctime()))
lasname = pargs.las_file
pointname = pargs.ref_data
use_local = pargs.use_local
if pargs.schema is not None:
report.set_schema(pargs.schema)
reporter = report.ReportZcheckAbs(use_local)
try:
extent = np.asarray(constants.tilename_to_extent(kmname))
except Exception as e:
print("Could not get extent from tilename.")
extent = None
pc_ref = None #base reference pointcloud
pc_refs = [] #list of possibly 'cropped' pointclouds...
if pargs.multipoints:
ftype = "multipoints"
explode = False
elif pargs.lines:
ftype = "lines"
explode = True
geoms = vector_io.get_geometries(pointname,
pargs.layername,
pargs.layersql,
extent,
explode=explode)
for geom in geoms:
xyz = array_geometry.ogrgeom2array(geom, flatten=False)
if xyz.shape[0] > 0:
pc_refs.append(pointcloud.Pointcloud(xyz[:, :2], xyz[:, 2]))
print("Found %d non-empty geometries" % len(pc_refs))
if len(pc_refs) == 0:
print("No input geometries in intersection...")
if pargs.ftype is not None:
ftype = pargs.ftype
cut_input_to = pargs.cut_to
print("Cutting input pointcloud to class %d" % cut_input_to)
pc = pointcloud.fromAny(lasname).cut_to_class(
cut_input_to) #what to cut to here...??
#warping loop here....
if (pargs.toE):
geoid = grid.fromGDAL(GEOID_GRID, upcast=True)
print("Using geoid from %s to warp to ellipsoidal heights." %
GEOID_GRID)
for i in range(len(pc_refs)):
toE = geoid.interpolate(pc_refs[i].xy)
M = (toE == geoid.nd_val)
if (M.any()):
print(
"Warping to ellipsoidal heights produced no-data values!")
M = np.logical_not(M)
toE = toE[M]
pc_refs[i] = pc_refs[i].cut(M)
pc_refs[i].z += toE
#Remove empty pointsets
not_empty = []
for pc_r in pc_refs:
if pc_r.get_size() > 0:
not_empty.append(pc_r) #dont worry, just a pointer...
else:
raise Warning("Empty input set...")
print("Checking %d point sets" % len(not_empty))
#Loop over strips#
for id in pc.get_pids():
print("%s\n" % ("+" * 70))
print("Strip id: %d" % id)
pc_c = pc.cut_to_strip(id)
if pc_c.get_size() < 50:
print("Not enough points...")
continue
might_intersect = []
for i, pc_r in enumerate(not_empty):
if pc_c.might_overlap(pc_r):
might_intersect.append(i)
if len(might_intersect) == 0:
print("Strip does not intersect any point 'patch'...")
continue
pc_c.triangulate()
pc_c.calculate_validity_mask(angle_tolerance, xy_tolerance,
z_tolerance)
#now loop over the patches which might intersect this strip...
any_checked = False
for i in might_intersect:
pc_r = not_empty[i].cut_to_box(*(pc_c.get_bounds()))
if pc_r.get_size == 0:
continue
any_checked = True
print("Stats for check of 'patch/set' %d against strip %d:" %
(i, id))
stats = check_points(pc_c, pc_r)
if stats is None:
print("Not enough points in proper triangles...")
continue
m, sd, n = stats
cm_x, cm_y = pc_r.xy.mean(axis=0)
cm_z = pc_r.z.mean()
print("Center of mass: %.2f %.2f %.2f" % (cm_x, cm_y, cm_z))
cm_geom = ogr.Geometry(ogr.wkbPoint25D)
cm_geom.SetPoint(0, cm_x, cm_y, cm_z)
#what geometry should be reported, bounding box??
reporter.report(kmname, id, ftype, m, sd, n, ogr_geom=cm_geom)
if not any_checked:
print("Strip did not intersect any point 'patch'...")