def transform(arg, transformation_matrix):
matrix_string = " ".join(
[str(e) for trans_row in transformation_matrix for e in trans_row])
if isinstance(arg, str):
json = u"""
{
"pipeline": [
\"""" + arg + """\",
{
"type":"filters.transformation",
"matrix":\"""" + matrix_string + """\"
}
]
}"""
pipeline = pdal.Pipeline(json)
else:
json = u"""
{
"pipeline": [
{
"type":"filters.transformation",
"matrix":\"""" + matrix_string + """\"
}
]
}"""
pipeline = pdal.Pipeline(json, arrays=arg)
pipeline.validate()
pipeline.loglevel = 8
pipeline.execute()
return pipeline.arrays
def test_merged_arrays(self):
"""Can we fetch multiple point views from merged PDAL data """
json = self.fetch_json('/data/filters/chip.json')
r = pdal.Pipeline(json)
r.execute()
arrays = r.arrays
self.assertEqual(len(arrays), 43)
def test_read_arrays(self):
"""Can we read and filter data from a list of arrays to PDAL"""
if Version(pdal.info.version) < Version('1.8'):
return True
# just some dummy data
x_vals = [1.0, 2.0, 3.0, 4.0, 5.0]
y_vals = [6.0, 7.0, 8.0, 9.0, 10.0]
z_vals = [1.5, 3.5, 5.5, 7.5, 9.5]
test_data = np.array([(x, y, z)
for x, y, z in zip(x_vals, y_vals, z_vals)],
dtype=[('X', np.float), ('Y', np.float),
('Z', np.float)])
pipeline = """
{
"pipeline": [
{
"type":"filters.range",
"limits":"X[2.5:4.5]"
}
]
}
"""
p = pdal.Pipeline(pipeline, arrays=[
test_data,
])
p.loglevel = 8
count = p.execute()
arrays = p.arrays
self.assertEqual(count, 2)
self.assertEqual(len(arrays), 1)
def test_schema(self):
"""Fetching a schema works"""
json = self.fetch_json('sort.json')
r = pdal.Pipeline(json)
r.validate()
r.execute()
self.assertEqual(r.schema['schema']['dimensions'][0]['name'], 'X')
def worker(mapped):
"""Multiprocessing worker function to be used by the
p.map function to map objects to, and then start
multiple times in parallel on separate CPU cores.
In this case the worker function instances ground
filter one file each, and return the resulting log,
metadata and the point cloud data array itself.
"""
print("PID {} starting to ground filter file {}".format(
os.getpid(), mapped[2]))
config, fpath, write = mapped[0], mapped[1] + mapped[2], mapped[3]
if write == True:
tag = mapped[4]
config = ('[\n\t"' + fpath + '",\n' + config +
',\n\t"' + fpath[:-4] + '_' + tag + '.las"\n]')
else: config = ('[\n\t"' + fpath + '",\n' + config + '\n]')
pipeline = pdal.Pipeline(config)
start = time()
pipeline.execute()
end = time()
print("PID {} finished ground filtering.".format(os.getpid()),
"Time elapsed: {} sec.".format(round(end - start, 2)))
log = pipeline.log
metadata = pipeline.metadata
arrays = pipeline.arrays
return log, metadata, arrays
def las_2_dataframe(las_path):
inputfile = str(las_path)
pipe_LASreader =\
{
"pipeline":[
{
"type":"readers.las",
"filename":inputfile,
"use_eb_vlr": "true"
}
]
}
# """%lasinput
#print(pipe_reader)
pipeline = pdal.Pipeline(json.dumps(pipe_LASreader))
pipeline.validate()
print(pipeline.validate())
# n_points = pipeline.execute()
#%time
start_time = time.time()
n_points = pipeline.execute()
elapsed_time_fl = (time.time() - start_time)
print('Time taken ', elapsed_time_fl,' seconds')
lidar_df = pd.DataFrame(pipeline.arrays[0])
print("Number of points in LiDAR:", n_points)
#print(lidar_df)
lidar_df.head()
return lidar_df
def tile_lidar(folder_with_las, spacing, output_folder, buffer_distance):
creating_json = {
"pipeline": [{
"type": "readers.las",
"filename": f"{folder_with_las}/*",
"spatialreference": "EPSG:25830"
}, {
"type": "filters.splitter",
"length": f"{spacing}",
"buffer": f"{buffer_distance}"
}, {
"type": "writer.las",
"srs": "EPSG:25830",
"filename": f"{output_folder}"
}]
}
consulta = json.dumps(creating_json, indent=4)
print(consulta)
pipeline = pdal.Pipeline(consulta)
pipeline.validate() # Check if json options are good
pipeline.loglevel = 8
count = pipeline.execute()
print(count)
def writeLAZBatches(path, data):
""" Write a numpy array with batch number for each point into laz file
Input:
String path, filepath
Numpy Array data, with at least 4 dimensions for X,Y,Z and BatchID
"""
# create correct structured numpy array
data = np.array(list(map(tuple, data)),
dtype=[('X', np.float64), ('Y', np.float64),
('Z', np.float64), ('ClusterID', np.int32)])
# calc offset from first data point
x_offset = int(str(data[0]['X'])[:5]) * 1000.0
y_offset = int(str(data[0]['Y'])[:5]) * 100.0
pipeline = {
"pipeline": [{
"type": "writers.las",
"dataformat_id": 0,
"compression": "laszip",
"offset_x": x_offset,
"offset_y": y_offset,
"extra_dims": "ClusterID=int32",
"filename": path
}]
}
r = pdal.Pipeline(json.dumps(pipeline), [data])
r.validate()
r.execute()
def get_points(lote):
s = lote
bounds = ([s.bounds[0], s.bounds[2]], [s.bounds[1], s.bounds[3]])
ept = {
"pipeline":[
{
"type": "readers.ept",
"filename": "https://ept-m3dc-pmsp.s3-sa-east-1.amazonaws.com/ept.json",
"bounds": str(bounds)
},
{
"type":"filters.crop",
"polygon":s.wkt
},
{
"type":"filters.hag_delaunay"
}
]}
pipeline = pdal.Pipeline(json.dumps(ept))
pipeline.validate()
n_points = pipeline.execute()
arr = pipeline.arrays[0]
df = pd.DataFrame(arr)
return df
def test_reference_counting(self):
"""Can we read and filter data from a list of arrays to PDAL"""
if Version(pdal.info.version) < Version("1.8"):
return True
# just some dummy data
x_vals = [1.0, 2.0, 3.0, 4.0, 5.0]
y_vals = [6.0, 7.0, 8.0, 9.0, 10.0]
z_vals = [1.5, 3.5, 5.5, 7.5, 9.5]
test_data = np.array(
[(x, y, z) for x, y, z in zip(x_vals, y_vals, z_vals)],
dtype=[("X", np.float), ("Y", np.float), ("Z", np.float)],
)
pipeline = """
{
"pipeline": [
{
"type":"filters.range",
"limits":"X[2.5:4.5]"
}
]
}
"""
p = pdal.Pipeline(pipeline, arrays=[test_data])
p.loglevel = 8
count = p.execute()
self.assertEqual(count, 2)
self.assertEqual(1, sys.getrefcount(p.arrays[0]),
"Reference count should only be 1 in this case")
def work(row):
items = row.rstrip().split(',')
try:
lat = float(items[0])
lon = float(items[1])
except:
# print('Input line in wrong format (possibly a header)')
return None
x, y = proj.transform(lat, lon)
x_min, x_max = (x + offset for offset in (-50, 50))
y_min, y_max = (y + offset for offset in (-50, 50))
json_def = json_tmpl.format(x_min=x_min,
x_max=x_max,
y_min=y_min,
y_max=y_max,
EPT_JSON=EPT_JSON)
#print(json_def)
pipeline = pdal.Pipeline(json_def)
pipeline.validate() # check if our JSON and options were good
pipeline.loglevel = 0 #really noisy
count = pipeline.execute()
arrays = pipeline.arrays
return count
def _run_PDAL_splitter(filename, tiled_temp_folder, tiling_mins, tiling_maxs,
n_tiles_side):
length_PDAL_tile = ((tiling_maxs[0] - tiling_mins[0]) /
float(n_tiles_side))
outfile_with_placeholder = "_#".join([filename.stem, filename.suffix])
outfilepath = tiled_temp_folder.joinpath(outfile_with_placeholder)
PDAL_pipeline_dict = {
"pipeline": [
filename.as_posix(), {
"type": "filters.splitter",
"origin_x": "{}".format(tiling_mins[0]),
"origin_y": "{}".format(tiling_mins[1]),
"length": "{}".format(length_PDAL_tile)
}, {
"type": "writers.las",
"filename": outfilepath.as_posix(),
"forward": ["scale_x", "scale_y", "scale_z"],
"offset_x": "auto",
"offset_y": "auto",
"offset_z": "auto"
}
]
}
_print_PDAL_pipeline_dict(PDAL_pipeline_dict)
PDAL_pipeline = pdal.Pipeline(json.dumps(PDAL_pipeline_dict))
logger.debug("... running PDAL:")
PDAL_pipeline.execute()
def get_points(ept_path, bounds, wkt):
READ_PIPELINE = """
{{
"pipeline": [
{{
"type": "readers.ept",
"filename": "{path}",
"bounds": "{bounds}"
}},
{{
"type":"filters.crop",
"polygon":"{wkt}"
}}
]
}}
"""
pipeline = pdal.Pipeline(
READ_PIPELINE.format(path=ept_path, bounds=bounds, wkt=wkt))
pipeline.validate()
pipeline.execute()
point_cloud = pipeline.arrays[0]
return point_cloud
def load_from_file(self,
filename, # type: str
proj, # type: str
zone, # type: str
ellps, # type: str
datum # type: str
): # type: (...) -> None
pipe_json = json.dumps([
{
'type': 'readers.sbet',
'filename': filename
}
])
sbet_pipeline = pdal.Pipeline(pipe_json)
sbet_pipeline.validate()
self._num_records = sbet_pipeline.execute()
data = sbet_pipeline.arrays[0]
self._record_length = len(data.dtype.names)
# convert structured data array to dict
self._nav_data = {convert_to_snake_case(name): data[name] for name in data.dtype.names}
if not zone:
self._projection = pyproj.Proj(proj=proj, ellps=ellps, datum=datum, preserve_units=True)
else:
self._projection = pyproj.Proj(proj=proj, zone=zone, ellps=ellps, datum=datum, preserve_units=True)
def readlasfile(lasfile):
"""
Run a PDAL pipeline. Input is a JSON declaration to
deliver to PDAL. Output is a labelled numpy array.
Data are filtered to compute height above ground using nearest ground point neighbours (TIN method arriving soon) and sort by morton order. Any unused dimensions are also trimmed.
"""
pipeline = {
"pipeline": [
{
"type": "readers.las",
"filename": lasfile
},
{
"type": "filters.hag"
},
{
"type": "filters.mortonorder"
}
]
}
#create a pipeline object
pipeline = pdal.Pipeline(json.dumps(pipeline))
# execute the pipeline
count = pipeline.execute()
#read points into a numpy structured array
arrays = pipeline.arrays
#return the numpy array to operate on
return(arrays)
def get_pipeline(filename):
with open(os.path.join(DATADIRECTORY, filename), "r") as f:
if filename.endswith(".json"):
pipeline = pdal.Pipeline(f.read())
elif filename.endswith(".py"):
pipeline = eval(f.read(), vars(pdal))
return pipeline
def read_file(filename, bounds=None):
if bounds is None:
json = u"""
{
"pipeline": [
\"""" + filename + """\"
]
}"""
else:
json = u"""
{
"pipeline": [
\"""" + filename + """\",
{
"type":"filters.crop",
"bounds":"([""" + str(bounds[0][0]) + """,""" \
+ str(bounds[0][1]) + """],[""" \
+ str(bounds[1][0]) + """,""" \
+ str(bounds[1][1]) + """])"
}
]
}"""
pipeline = pdal.Pipeline(json)
pipeline.validate()
pipeline.loglevel = 8
pipeline.execute()
return pipeline.arrays
def test_reference_counting(self):
"""Can we read and filter data from a list of arrays to PDAL"""
# just some dummy data
x_vals = [1.0, 2.0, 3.0, 4.0, 5.0]
y_vals = [6.0, 7.0, 8.0, 9.0, 10.0]
z_vals = [1.5, 3.5, 5.5, 7.5, 9.5]
test_data = np.array(
[(x, y, z) for x, y, z in zip(x_vals, y_vals, z_vals)],
dtype=[("X", float), ("Y", float), ("Z", float)],
)
pipeline = """
{
"pipeline": [
{
"type":"filters.range",
"limits":"X[2.5:4.5]"
}
]
}
"""
p = pdal.Pipeline(pipeline, arrays=[test_data])
count = p.execute()
assert count == 2
refcount = sys.getrefcount(p.arrays[0])
assert refcount == 1
def openLAZ(path, extra_dims={}):
""" Open a single laz file with pdal pipeline.
Then extract X,Y,Z values and classification.
Input:
String path, filepath
Dictionary extra_dims, names and datatypes of additional fields
Return:
Numpy Array, with dimension (num_points, num_dims)
"""
if extra_dims:
pipeline = {
"pipeline": [{
"type": "readers.las",
"filename": path,
"extra_dims": createDimStr(extra_dims)
}]
}
else:
pipeline = {"pipeline": [{"type": "readers.las", "filename": path}]}
r = pdal.Pipeline(json.dumps(pipeline))
r.validate()
r.execute()
# first stack xyz together
points = np.dstack((r.arrays[0]['X'], r.arrays[0]['Y'], r.arrays[0]['Z'],
r.arrays[0]['Classification']))
# and now each predefined scalar field
for field in extra_dims:
points = np.dstack((points, r.arrays[0][field]))
return points.squeeze() # cut of axis with length one
def set_json_pipeline(self, json_pipe):
infile = None
outfile = None
try:
if "pipeline" not in json_pipe.keys():
json_pipe = {"pipeline": json_pipe}
except AttributeError:
raise AttributeError(
"json_pipe argument is not a json formatted dictionary.")
except Exception as err:
raise err
if type(json_pipe["pipeline"][0]) is str:
infile = json_pipe["pipeline"][0]
if type(json_pipe["pipeline"][-1]) is str:
outfile = json_pipe["pipeline"][-1]
if infile is None:
json_pipe["pipeline"].insert(0, self.infile)
if outfile is None and self.outfile_set:
json_pipe["pipeline"].append(self.outfile)
# Ensure valid pipeline before actually making changes:
try:
pdal.Pipeline(json.dumps(json_pipe)).validate()
self.json_pipeline = json_pipe
if outfile is not None:
self.outfile = outfile
if infile is not None:
self.infile = infile
except Exception as err:
raise err
def write_to_laz(point_cloud, path):
'''
writes a structured array to a .laz file
in:
point_cloud [structured np array]:
The output pointcloud; needs attributes x, y and z.
When createing a pointcloud from scratch, pay attention to
the data types of the specific attributes, this is a pain in the ass.
Easier to add one new collumn to an existing (filtered) pointcloud.
path [string]:
Path to a laz file.
out:
None
'''
WRITE_PIPELINE = """
{{
"pipeline": [
{{
"type": "writers.las",
"filename": "{path}",
"extra_dims": "all"
}}
]
}}
"""
pipeline = pdal.Pipeline(
WRITE_PIPELINE.format(path=path),
arrays=[point_cloud]
)
pipeline.validate()
pipeline.execute()
def pipeline_realization(pip_json, print_result):
try:
# ===============================================
# Pipeline execution
pipeline = pdal.Pipeline(pip_json)
pipeline.validate() # check if our JSON and options were good
pipeline.execute()
if print_result:
arrays = pipeline.arrays
metadata = pipeline.metadata
log = pipeline.log
print("\n================")
print("Arrays:")
print(arrays)
print("\n================")
print("Metadata:")
print(metadata)
print("\n================")
print("Log:")
print(log)
print("pdal pipeline finished")
return True
except:
print(" Error !!")
return False
def remove_noise(lasfile, outputlas):
creating_json = {
"pipeline": [
{
"type": "readers.las",
"filename": f"{lasfile}"
},
{
# Creates a window to find outliers. If they are found they are classified as noise (7).
"type": "filters.outlier",
"method": "statistical",
"multiplier": 3,
"mean_k": 8
},
{
"type": "writers.las",
"compression": "laszip",
"filename": f"{outputlas}"
}
]
}
consulta = json.dumps(creating_json, indent=4)
print(consulta)
pipeline = pdal.Pipeline(consulta)
pipeline.validate() # Check if json options are good
pipeline.loglevel = 8
count = pipeline.execute()
print(count)
def run_pipe(from_table, to_table):
db_params = {
"host": "localhost",
"database": "pointclouds",
"port": 5432,
"user": username,
"password": password
}
conn = dbutils.connect(db_params)
sys.stdout.write("Connected to database 'pointclouds'.\n")
this_pipe = deepcopy(test_pipe)
reader = {
"type": "readers.pgpointcloud",
"connection": f"host=127.0.0.1 dbname='pointclouds' user={username} password={password}",
"table": from_table,
"column": "pa"
}
this_pipe["pipeline"].insert(0, reader)
pipe = pdal.Pipeline(json.dumps(this_pipe))
sys.stdout.write("Executing pdal pipeline.\n")
pipe.execute()
sys.stdout.write("pdal pipeline completed successfully.\n")
dbutils.execute_mogrify(conn, pd.DataFrame(pipe.arrays[0]), to_table)
sys.stdout.write(f"Added {from_table} to {to_table}.\n")
def DEMonizator(lasfile, outputfile, resolution=1000):
creating_json = {
"pipeline": [{
"type": "readers.las",
"filename": f"{lasfile}",
"spatialreference": "EPSG:25830"
}, {
"type": "filters.range",
"limits": "Classification[2:2]"
}, {
"type": "writers.gdal",
"gdaldriver": "GTiff",
"nodata": "-9999",
"output_type": "idw",
"resolution": f"{resolution}",
"filename": f"{outputfile}"
}]
}
consulta = json.dumps(creating_json, indent=4)
print(consulta)
pipeline = pdal.Pipeline(consulta)
pipeline.validate() # Check if json options are good
pipeline.loglevel = 8
count = pipeline.execute()
print(count)
def getMetadata(file_path):
"""
用来获取点云的 max and min xyz value
:param file_path: las点云文件的路径
:return: 返回一个 dict {'maxx'：value , 'maxy':value, 'maxz':value, 'minx':value, 'miny':value, 'minz':value}
"""
'''获取最低点、最高点的字典'''
data_json = """
{
"pipeline": [
\"""" + file_path + """\",
{
"type": "filters.stats"
}
]
}"""
# print(data_json)
pipeline = pdal.Pipeline(data_json)
pipeline.validate() # 检查我们的JSON和选项是否良好
pipeline.loglevel = 8 # really noisy 输入日志相关 未知
count = pipeline.execute() # 执行通道
# arrays = pipeline.arrays
# print("arrays:", arrays)
metadata = pipeline.metadata # 获取通道的 所有详情信息
metadata_dict = eval(str(json.loads(metadata))) # 把字符串 信息转成 字典
bbox_dict = metadata_dict["metadata"]["filters.stats"][1]["bbox"][
"native"]["bbox"] # 获取到 详情信息包含 最小点最大点的
# print(metadata_dict)
subkey = ['maxx', 'maxy', 'maxz', 'minx', 'miny', 'minz']
box_dict = dict([(key, bbox_dict[key]) for key in subkey]) # 获取到最大点、最小点的字典
return box_dict
def test_merged_arrays(self):
"""Can we load data from a list of arrays to PDAL"""
if Version(pdal.info.version) < Version('1.8'):
return True
data = np.load(os.path.join(DATADIRECTORY, 'test3d.npy'))
arrays = [data, data, data]
json = self.fetch_json('chip.json')
chip = u"""{
"pipeline":[
{
"type":"filters.range",
"limits":"Intensity[100:300)"
}
]
}"""
p = pdal.Pipeline(chip, arrays)
p.loglevel = 8
count = p.execute()
arrays = p.arrays
self.assertEqual(len(arrays), 3)
for data in arrays:
self.assertEqual(len(data), 12)
self.assertEqual(data['Intensity'].sum(), 1926)
def remove_overlay_points(lasfile,
outputlas
):
creating_json = {
"pipeline" : [
{
"type": "readers.las",
"filename": f"{lasfile}"
},
{
# Filter assigning points of class overlay (12) to class noise (7)
"type": "filters.assign",
"assignment": "Classification[12:12]=7"
},
{
"type": "writers.las",
"compression": "laszip",
"filename": f"{outputlas}"
}
]
}
consulta = json.dumps(creating_json, indent=4)
print(consulta)
pipeline = pdal.Pipeline(consulta)
pipeline.validate() # Check if json options are good
pipeline.loglevel = 8
count = pipeline.execute()
print(count)
def test_execution(self):
"""Can we execute a PDAL pipeline"""
x = self.fetch_json('sort.json')
r = pdal.Pipeline(x)
r.validate()
r.execute()
self.assertGreater(len(r.pipeline), 200)
def test_merged_arrays(self):
"""Can we load data from a a list of arrays to PDAL"""
data = np.load(os.path.join(DATADIRECTORY, 'perlin.npy'))
arrays = [data, data, data]
arrays = [data]
json = self.fetch_json('chip.json')
chip = u"""{
"pipeline":[
{
"type":"filters.range",
"limits":"Intensity[0:0.10]"
}
]
}"""
p = pdal.Pipeline(chip, arrays)
p.loglevel = 8
count = p.execute()
arrays = p.arrays
self.assertEqual(len(arrays), 1)
data = arrays[0]
self.assertEqual(len(data), 1836)
