class TestRealGridValid(unittest.TestCase):
_test_dir = 'test_tmp_dir'
_test_data_dir = 'testdata'
_test_tile_idx = [101, 101]
_test_file_name = 'C_43FN1_1_1.LAZ'
_min_x = -113107.8100
_min_y = 214783.8700
_max_x = 398892.1900
_max_y = 726783.87
_n_tiles_sides = 256
plot = False
def setUp(self):
self.grid = Grid()
self.grid.setup(min_x=self._min_x,
min_y=self._min_y,
max_x=self._max_x,
max_y=self._max_y,
n_tiles_side=self._n_tiles_sides)
self._test_data_path = Path(self._test_data_dir).joinpath(self._test_file_name)
self.points = _read_points_from_file(str(self._test_data_path))
def test_isPointInTile(self):
x_pts, y_pts = self.points.T
mask_valid_points = self.grid.is_point_in_tile(x_pts, y_pts,
*self._test_tile_idx)
self.assertTrue(np.all(mask_valid_points))
def __init__(self, input_file=None, label=None):
self.pipeline = ('set_grid', 'split_and_redistribute', 'validate')
self.grid = Grid()
if input_file is not None:
self.input_path = input_file
if label is not None:
self.label = label
def setUp(self):
self.grid = Grid()
self.grid.setup(min_x=self._min_x,
min_y=self._min_y,
max_x=self._max_x,
max_y=self._max_y,
n_tiles_side=self._n_tiles_sides)
self._test_data_path = Path(self._test_data_dir).joinpath(self._test_file_name)
self.points = _read_points_from_file(str(self._test_data_path))
def __init__(self, input=None, label=None, tile_index=(None, None)):
self.pipeline = ('add_custom_feature', 'load', 'normalize',
'apply_filter', 'export_point_cloud',
'generate_targets', 'extract_features',
'export_targets', 'clear_cache')
self.point_cloud = create_point_cloud([], [], [])
self.targets = create_point_cloud([], [], [])
self.grid = Grid()
self.filter = DictToObj({
f.__name__: f
for f in
[select_above, select_below, select_equal, select_polygon]
})
self.extractors = DictToObj(_get_extractor_dict())
self._features = None
self._tile_index = tile_index
if input is not None:
self.input_path = input
if label is not None:
self.label = label
class TestValidGridSetup(unittest.TestCase):
def setUp(self):
self.grid = Grid()
self.grid.setup(0., 0., 20., 20., 5)
def test_gridMins(self):
np.testing.assert_allclose(self.grid.grid_mins, [0., 0.])
def test_gridMaxs(self):
np.testing.assert_allclose(self.grid.grid_maxs, [20., 20.])
def test_gridWidth(self):
np.testing.assert_allclose(self.grid.grid_width, 20.)
def test_tileWidth(self):
np.testing.assert_allclose(self.grid.tile_width, 4.)
def test_tileIndexForPoint(self):
np.testing.assert_array_equal(self.grid.get_tile_index(0.1, 0.2),
(0, 0))
def test_tileIndexForArray(self):
np.testing.assert_array_equal(self.grid.get_tile_index((0.1, 19.9),
(0.2, 19.8)),
((0, 0), (4, 4)))
def test_tileBoundsForPoint(self):
np.testing.assert_array_equal(self.grid.get_tile_bounds(0, 0),
((0., 0.), (4., 4.)))
def test_tileBoundsForArray(self):
np.testing.assert_array_equal(self.grid.get_tile_bounds((0, 0),
(0, 1)),
(((0., 0.), (0., 4.)),
((4., 4.), (4., 8.))))
def test_rectangularGrid(self):
with self.assertRaises(ValueError):
grid = Grid()
grid.setup(0., 0., 10., 20., 5)
def test_zeroWidthGrid(self):
with self.assertRaises(ValueError):
grid = Grid()
grid.setup(0., 0., 0., 20., 5)
def test_zeroNumberOfTilesGrid(self):
with self.assertRaises(ValueError):
grid = Grid()
grid.setup(0., 0., 20., 20., 0)
def setUp(self):
self.grid = Grid()
self.grid.setup(0., 0., 20., 20., 5)
class Retiler(PipelineRemoteData):
""" Split point cloud data into smaller tiles on a regular grid. """
def __init__(self, input_file=None, label=None):
self.pipeline = ('set_grid', 'split_and_redistribute', 'validate')
self.grid = Grid()
if input_file is not None:
self.input_path = input_file
if label is not None:
self.label = label
def set_grid(self, min_x, min_y, max_x, max_y, n_tiles_side):
"""
Setup the grid to which the input file is retiled.
:param min_x: min x value of tiling schema
:param min_y: max y value of tiling schema
:param max_x: min x value of tiling schema
:param max_y: max y value of tiling schema
:param n_tiles_side: number of tiles along axis. Tiling MUST be square
(enforced)
"""
logger.info('Setting up the target grid')
self.grid.setup(min_x, min_y, max_x, max_y, n_tiles_side)
return self
def split_and_redistribute(self):
"""
Split the input file using PDAL and organize the tiles in subfolders
using the location on the input grid as naming scheme.
"""
self._check_input()
logger.info('Splitting file {} with PDAL ...'.format(self.input_path))
_run_PDAL_splitter(self.input_path, self.output_folder,
self.grid.grid_mins, self.grid.grid_maxs,
self.grid.n_tiles_side)
logger.info('... splitting completed.')
tiles = [
f for f in self.output_folder.iterdir()
if (f.is_file() and f.suffix.lower() == self.input_path.suffix.
lower() and f.stem.startswith(self.input_path.stem)
and f.name != self.input_path.name)
]
logger.info('Redistributing files to tiles ...')
for tile in tiles:
(_, tile_mins, tile_maxs, _, _) = _get_details_pc_file(str(tile))
# Get central point to identify associated tile
cpX = tile_mins[0] + ((tile_maxs[0] - tile_mins[0]) / 2.)
cpY = tile_mins[1] + ((tile_maxs[1] - tile_mins[1]) / 2.)
tile_id = _get_tile_name(*self.grid.get_tile_index(cpX, cpY))
retiled_folder = self.output_folder.joinpath(tile_id)
check_dir_exists(retiled_folder, should_exist=True, mkdir=True)
logger.info('... file {} to {}'.format(tile.name, tile_id))
tile.rename(retiled_folder.joinpath(tile.name))
logger.info('... redistributing completed.')
return self
def validate(self, write_record_to_file=True):
"""
Validate the produced output by checking consistency in the number
of input and output points.
"""
self._check_input()
logger.info('Validating split ...')
(parent_points, _, _, _,
_) = _get_details_pc_file(self.input_path.as_posix())
logger.info('... {} points in parent file'.format(parent_points))
valid_split = False
split_points = 0
redistributed_to = []
tiles = self.output_folder.glob('tile_*/{}*'.format(
self.input_path.stem))
for tile in tiles:
if tile.is_file():
(tile_points, _, _, _,
_) = _get_details_pc_file(tile.as_posix())
logger.info('... {} points in {}'.format(
tile_points, tile.name))
split_points += tile_points
redistributed_to.append(tile.parent.name)
if parent_points == split_points:
logger.info('... split validation completed.')
valid_split = True
else:
logger.error('Number of points in parent and child tiles differ!')
retile_record = {
'file': self.input_path.as_posix(),
'redistributed_to': redistributed_to,
'validated': valid_split
}
if write_record_to_file:
_write_record(self.input_path.stem, self.output_folder,
retile_record)
return self
def _check_input(self):
if not self.grid.is_set:
raise ValueError('The grid has not been set!')
check_file_exists(self.input_path, should_exist=True)
check_dir_exists(self.output_folder, should_exist=True)
class DataProcessing(PipelineRemoteData):
""" Read, process and write point cloud data using laserchicken. """
def __init__(self, input=None, label=None, tile_index=(None, None)):
self.pipeline = ('add_custom_feature', 'load', 'normalize',
'apply_filter', 'export_point_cloud',
'generate_targets', 'extract_features',
'export_targets', 'clear_cache')
self.point_cloud = create_point_cloud([], [], [])
self.targets = create_point_cloud([], [], [])
self.grid = Grid()
self.filter = DictToObj({
f.__name__: f
for f in
[select_above, select_below, select_equal, select_polygon]
})
self.extractors = DictToObj(_get_extractor_dict())
self._features = None
self._tile_index = tile_index
if input is not None:
self.input_path = input
if label is not None:
self.label = label
@property
def features(self):
self._features = DictToObj(list_feature_names())
return self._features
def add_custom_feature(self, extractor_name, **parameters):
"""
Add customized feature to be computed with laserchicken.
For information on the available extractors and the corresponding
parameters:
$ laserfarm data_processing extractors --help
$ laserfarm data_processing extractors <extractor_name> --help
:param extractor_name: Name of the (customizable) extractor
:param parameters: Extractor-specific parameters
"""
extractor = _get_attribute(self.extractors, extractor_name)
_check_parameters_for_extractor(extractor, parameters)
logger.info('Setting up feature extractor {}'.format(extractor_name))
register_new_feature_extractor(extractor(**parameters))
return self
def load(self, **load_opts):
"""
Read point cloud from disk.
:param load_opts: Arguments passed to the laserchicken load function
"""
check_path_exists(self.input_path, should_exist=True)
input_file_list = _get_input_file_list(self.input_path)
logger.info('Loading point cloud data ...')
for file in input_file_list:
logger.info('... loading {}'.format(file))
add_to_point_cloud(self.point_cloud, load(file, **load_opts))
logger.info('... loading completed.')
return self
def normalize(self, cell_size):
"""
Normalize point cloud heights.
:param cell_size: Size of the side of the cell employed for
normalization (in m)
:return:
"""
if not cell_size > 0.:
raise ValueError('Cell size should be > 0.!')
_check_point_cloud_is_not_empty(self.point_cloud)
logger.info('Normalizing point-cloud heights ...')
normalize(self.point_cloud, cell_size)
logger.info('... normalization completed.')
return self
def apply_filter(self, filter_type, **filter_input):
"""
Apply a filter to the environment point cloud.
For information on filter_types and the corresponding input:
$ laserfarm data_processing filter --help
$ laserfarm data_processing filter <filter_type> --help
:param filter_type: Type of filter to apply.
:param filter_input: Filter-specific input.
"""
_check_point_cloud_is_not_empty(self.point_cloud)
filter = _get_attribute(self.filter, filter_type)
logger.info('Filtering point-cloud data')
self.point_cloud = filter(self.point_cloud, **filter_input)
return self
def export_point_cloud(self, filename='', attributes='all', **export_opts):
"""
Write environment point cloud to disk.
:param filename: optional filename where to write point-cloud data
:param attributes: List of attributes to be written in the output file
:param export_opts: Optional arguments passed to the laserchicken
export function
"""
expath = self._get_export_path(filename)
logger.info('Exporting environment point-cloud ...')
self._export(self.point_cloud,
expath,
attributes,
multi_band_files=True,
**export_opts)
logger.info('... exporting completed.')
return self
def generate_targets(self,
min_x,
min_y,
max_x,
max_y,
n_tiles_side,
tile_mesh_size,
validate=True,
validate_precision=None):
"""
Generate the target point cloud.
:param min_x: Min x value of the tiling schema
:param min_y: Min y value of the tiling schema
:param max_x: Max x value of the tiling schema
:param max_y: Max y value of the tiling schema
:param n_tiles_side: Number of tiles along X and Y (tiling MUST be
square)
:param tile_mesh_size: Spacing between target points (in m). The tiles'
width must be an integer times this spacing
:param validate: If True, check if all points in the point-cloud belong
to the same tile
:param validate_precision: Optional precision threshold to determine
whether point belong to tile
"""
logger.info('Setting up the target grid')
self.grid.setup(min_x, min_y, max_x, max_y, n_tiles_side)
if any([idx is None for idx in self._tile_index]):
raise RuntimeError('Tile index not set!')
if validate:
logger.info('Checking whether points belong to cell '
'({},{})'.format(*self._tile_index))
x_all, y_all, _ = get_point(self.point_cloud, ...)
mask = self.grid.is_point_in_tile(x_all, y_all,
self._tile_index[0],
self._tile_index[1],
validate_precision)
assert np.all(mask), ('{} points belong to (a) different tile(s)'
'!'.format(len(x_all[~mask])))
logger.info('Generating target point mesh with '
'{}m spacing '.format(tile_mesh_size))
x_trgts, y_trgts = self.grid.generate_tile_mesh(
self._tile_index[0], self._tile_index[1], tile_mesh_size)
self.targets = create_point_cloud(x_trgts, y_trgts,
np.zeros_like(x_trgts))
return self
def extract_features(self,
volume_type,
volume_size,
feature_names,
sample_size=None):
"""
Extract point-cloud features and assign them to the specified target
point cloud.
:param volume_type: Type of volume used to construct neighborhoods
:param volume_size: Size of the volume-related parameter (in m)
:param feature_names: List of the feature names to be computed
:param sample_size: Sample neighborhoods with a random subset of points
"""
logger.info('Building volume of type {}'.format(volume_type))
volume = build_volume(volume_type, volume_size)
logger.info('Constructing neighborhoods')
neighborhoods = compute_neighborhoods(self.point_cloud,
self.targets,
volume,
sample_size=sample_size)
logger.info('Starting feature extraction ...')
compute_features(self.point_cloud, neighborhoods, self.targets,
feature_names, volume)
logger.info('... feature extraction completed.')
return self
def export_targets(self,
filename='',
attributes='all',
multi_band_files=True,
**export_opts):
"""
Write target point cloud to disk.
:param filename: optional filename where to write point-cloud data
:param attributes: List of attributes to be written in the output file
:param multi_band_files: If true, write all attributes in one file
:param export_opts: Optional arguments passed to the laserchicken
export function
"""
expath = self._get_export_path(filename)
file_handle = 'tile_{}_{}'.format(*self._tile_index)
logger.info('Exporting target point-cloud ...')
self._export(self.targets, expath, attributes, multi_band_files,
file_handle, **export_opts)
logger.info('... exporting completed.')
return self
def clear_cache(self):
""" Clear KDTree's cached by Laserchicken. """
logger.info('Clearing cached KDTrees ...')
initialize_cache()
return self
@staticmethod
def _export(point_cloud,
path,
attributes='all',
multi_band_files=True,
file_handle='point_cloud',
**export_opts):
"""
Write generic point-cloud data to disk.
:param path: Path where to write point-cloud data
:param attributes: List of attributes to be written in the output file
:param multi_band_files: If true, write all attributes in one file
:param file_handle: Stem for the file name(s) if path is a directory
:param export_opts: Optional arguments passed to the laserchicken
export function
"""
features = [
f for f in point_cloud[laserchicken.keys.point].keys()
if f not in 'xyz'
] if attributes == 'all' else attributes
for file, feature_set in _get_output_file_dict(path, file_handle,
features,
multi_band_files,
**export_opts).items():
logger.info('... exporting {}'.format(file))
export(point_cloud, file, attributes=feature_set, **export_opts)
def _get_export_path(self, filename=''):
check_dir_exists(self.output_folder, should_exist=True)
if pathlib.Path(filename).parent.name:
raise IOError('filename should not include path!')
return self.output_folder.joinpath(filename).as_posix()