def test_create_training_dataset():
"""Test the creation of a training dataset for implementing a supervised
learning algorithm.
One tests the case where there is no sampled dataset to concatenate (pandas
raises a ValueError), and a classic case where some label-related samples
are available.
"""
EMPTY_EXPERIMENT = "test"
with pytest.raises(ValueError):
dataset = create_training_dataset(
DATADIR, EMPTY_EXPERIMENT, NEIGHBORS, LABELS
)
VALID_EXPERIMENT = "b9"
ground_data = io.read_ply(DATADIR / "input" / "b9_ground.ply")
vegetation_data = io.read_ply(DATADIR / "input" / "b9_vegetation.ply")
roof_data = io.read_ply(DATADIR / "input" / "b9_roof.ply")
dataset = create_training_dataset(
DATADIR, VALID_EXPERIMENT, NEIGHBORS, LABELS
)
assert dataset.shape[0] == (
len(ground_data) + len(vegetation_data) + len(roof_data)
)
assert dataset.shape[1] == (
1 # labels
+ len(NEIGHBORS) * N_FEATURES # geometric features
)
def main(opts):
"""Load a point cloud file and compute a kd-tree
"""
input_path = Path(opts.datapath, "input", opts.input_file)
if not input_path.is_file():
logger.error("No such file '%s'.", input_path)
sys.exit(1)
logger.info("Load data...")
if input_path.suffix == ".xyz":
data = read_xyz(str(input_path))
elif input_path.suffix == ".las":
data = read_las(str(input_path))
elif input_path.suffix == ".ply":
data = read_ply(str(input_path))
else:
raise ValueError("Wrong file extension, please send xyz or las file.")
tree_file = opts.tree_file
leaf_size = opts.kdtree_leafs
if not tree_file:
experiment = opts.input_file.split(".")[0]
fname = "kd-tree-leaf-{}.pkl".format(leaf_size)
output_path = Path(opts.datapath, "output", experiment)
output_path.mkdir(parents=True, exist_ok=True)
tree_file = output_path / fname
logger.info("Compute tree...")
tree = compute_tree(data[:, :3], leaf_size)
logger.info("Dump tree into %s...", tree_file)
with open(tree_file, 'wb') as fobj:
pickle.dump(tree, fobj)
def main(opts):
input_path = Path(opts.datapath, "input", opts.input_file)
if not input_path.is_file():
logger.error("No such file '%s'.", input_path)
sys.exit(1)
output_path = Path(opts.datapath, "output", opts.input_file.split(".")[0])
kdtrees = [f.name for f in output_path.glob("*.pkl")]
features = [f.name for f in Path(output_path, "features").glob("*.h5")]
pred = [f.name for f in Path(output_path, "prediction").glob("*.xyz")]
pred = pred + [f.name for f in Path(output_path, "prediction").glob("*.las")]
if input_path.suffix == ".las":
header = laspy.header.Header()
reader = laspy.base.FileManager(input_path, mode="r")
hm = laspy.header.HeaderManager(header, reader)
nb_points = hm.point_records_count
feature_list = [s.name for s in reader.point_format.specs]
xmin, ymin, zmin = hm.min
xmax, ymax, zmax = hm.max
elif input_path.suffix == ".xyz":
logger.warning(
"The file is read; it may take time regarding the file size"
)
data = read_xyz(input_path)
nb_points = len(data)
feature_list = "x, y, z, r, g, b"
xmin, ymin, zmin = np.min(data[:, :3], axis=0)
xmax, ymax, zmax = np.max(data[:, :3], axis=0)
elif input_path.suffix == ".ply":
logger.warning(
"The file is read; it may take time regarding the file size"
)
data = read_ply(input_path)
nb_points = len(data)
feature_list = "x, y, z"
xmin, ymin, zmin = np.min(data, axis=0)
xmax, ymax, zmax = np.max(data, axis=0)
else:
logger.error("Unknown file extension, '%s' not supported", input_path.suffix)
sys.exit(1)
info_string = (
f"File : {opts.input_file}"
f"\nFolder : {input_path.parent}"
f"\nNb points : {nb_points}"
f"\nFeature list : {feature_list}"
f"\nx range : {xmax-xmin:.2f}\t[{xmin:.2f}; {xmax:.2f}]"
f"\ny range : {ymax-ymin:.2f}\t[{ymin:.2f}; {ymax:.2f}]"
f"\nz range : {zmax-zmin:.2f}\t[{zmin:.2f}; {zmax:.2f}]"
f"\nkd-tree : {kdtrees}"
f"\nfeatures : {features}"
f"\npredictions : {pred}"
)
logger.info("Generate info on an input file...\n%s", info_string)
def main(opts):
input_path = Path(opts.datapath, "input", opts.input_file)
if not input_path.is_file():
logger.error("No such file '%s'.", input_path)
sys.exit(1)
if input_path.suffix == ".xyz":
data = read_xyz(str(input_path))
elif input_path.suffix == ".las":
data = read_las(str(input_path))
elif input_path.suffix == ".ply":
data = read_ply(str(input_path))
else:
raise ValueError("Wrong file extension, please send xyz or las file.")
if opts.extra_columns is not None:
if len(opts.extra_columns) + 3 != data.shape[1]:
logger.warning(
"%d extra fields are expected for the provided input data, however you enter %d field names (%s).",
data.shape[1] - 3, len(opts.extra_columns), opts.extra_columns)
raise ValueError(
"The given input columns does not match data shape, i.e. x,y,z plus extra columns."
)
experiment = experiment_folder_name(opts.input_file, opts.label_scene)
tree_file = opts.tree_file
if not tree_file:
tree_file = Path(opts.datapath, "output", experiment,
"kd-tree-leaf-" + str(opts.kdtree_leafs) + ".pkl")
if not tree_file.exists():
logger.info(
"No serialized kd-tree with leaf size = %s. Please index your "
"point cloud with the 'index' command, then use the "
"--tree-file or the -t/--kdtree-leafs option.",
opts.kdtree_leafs)
sys.exit(0)
with open(tree_file, 'rb') as fobj:
logger.info("Load kd-tree from file %s...", tree_file)
tree = pickle.load(fobj)
# the shape must be the same between the data read from file
# and the data stored in the kd-tree file, except for the '--label-scene'
# option (where you have a sample of the scene)
if not opts.label_scene and tree.data.shape[0] != data.shape[0]:
logger.info("Input data and data stored in the kd-tree "
"do not have the same length")
sys.exit(0)
output_path = Path(opts.datapath, "output", experiment, "features")
output_path.mkdir(parents=True, exist_ok=True)
if opts.label_scene:
hdf5name = "features_" + label_from_file(opts.input_file) + ".h5"
else:
hdf5name = "features.h5" # complete scene
output_file = output_path / hdf5name
neighbors = _check_neighbors_feature_file(output_file, opts.neighbors)
neighbors = list(sorted(neighbors))
if len(neighbors) == 0:
logger.info("Features was extracted for all neighboorhoods %s.",
opts.neighbors)
logger.info("Check the file '%s'", output_file)
logger.info("Exit program")
sys.exit(0)
extra_columns = (tuple(opts.extra_columns)
if opts.extra_columns is not None else tuple())
extract(data, tree, output_file, neighbors, opts.nb_process, extra_columns,
opts.chunksize)
logger.info("Results in %s", output_file)
def test_read_ply():
"""Test the reading of a .ply file
"""
fpath = str(PLYFILE)
data = io.read_ply(fpath)
assert data.shape == (22300, 3)