def start_log_to_file(self, directory='', append=False):
"""
Add a file handler to the log. STDOUT and STDERR are also redirected to
the log file. If a stream handler was already present, remove it.
:param directory: Directory where to write the logfile (ignore it if
filename already includes path)
:param append: If True, append logs to file (if existing).
"""
self.remove_handlers(file=True)
if not self.filename.parent.name:
file_path = pathlib.Path(directory).joinpath(self.filename.name)
else:
file_path = self.filename
check_dir_exists(file_path.parent, should_exist=True)
fh = logging.FileHandler(file_path,
mode='w' if not append else 'a',
delay=True)
fh.setFormatter(self.formatter)
fh.setLevel(self.level)
self.logger.addHandler(fh)
logger.debug('Start stream to file: {}'.format(file_path.as_posix()))
self._redirect_std_streams(True)
def create_subregion_geotiffs(self, output_handle, EPSG=28992):
"""
Export geotiff per sub-region, loop in band dimension
:param output_handle: Handle of output file. The output will be named
as <output_handle>_TILE_<tile ID>_BAND_<band name>
:param EPSG: (Optional) EPSG code of the spatial reference system of
the input data. Default 28992.
"""
utils.check_dir_exists(self.output_folder, should_exist=True)
outfilestem = os.path.join(self.output_folder.as_posix(),
output_handle)
for subTiffNumber in range(len(self.subtilelists)):
infiles = self.subtilelists[subTiffNumber]
logger.info('Processing sub-region GeoTiff no. {} '
'...'.format(subTiffNumber))
logger.info('... number of constituent tiles: '
'{}'.format(len(infiles)))
if infiles:
outfile = '{}_TILE_{:03d}'.format(outfilestem, subTiffNumber)
_make_geotiff_per_band(infiles,
outfile,
self.bands,
self.input_path.as_posix(),
self.LengthDataRecord,
self.xResolution,
self.yResolution,
EPSG)
else:
logger.warning(
'No data in sub-region no. ' + str(subTiffNumber))
logger.info('... processing of sub-region completed.')
return self
def split_and_redistribute(self):
"""
Split the input file using PDAL and organize the tiles in subfolders
using the location on the input grid as naming scheme.
"""
self._check_input()
logger.info('Splitting file {} with PDAL ...'.format(self.input_path))
_run_PDAL_splitter(self.input_path, self.output_folder,
self.grid.grid_mins, self.grid.grid_maxs,
self.grid.n_tiles_side)
logger.info('... splitting completed.')
tiles = [
f for f in self.output_folder.iterdir()
if (f.is_file() and f.suffix.lower() == self.input_path.suffix.
lower() and f.stem.startswith(self.input_path.stem)
and f.name != self.input_path.name)
]
logger.info('Redistributing files to tiles ...')
for tile in tiles:
(_, tile_mins, tile_maxs, _, _) = _get_details_pc_file(str(tile))
# Get central point to identify associated tile
cpX = tile_mins[0] + ((tile_maxs[0] - tile_mins[0]) / 2.)
cpY = tile_mins[1] + ((tile_maxs[1] - tile_mins[1]) / 2.)
tile_id = _get_tile_name(*self.grid.get_tile_index(cpX, cpY))
retiled_folder = self.output_folder.joinpath(tile_id)
check_dir_exists(retiled_folder, should_exist=True, mkdir=True)
logger.info('... file {} to {}'.format(tile.name, tile_id))
tile.rename(retiled_folder.joinpath(tile.name))
logger.info('... redistributing completed.')
return self
def setup_local_fs(self,
input_folder=None,
output_folder=None,
tmp_folder='.'):
"""
IO setup for the local file system.
:param input_folder: path to input folder on local filesystem.
:param output_folder: path to output folder on local filesystem. This
folder is considered root for all output paths
specified.
:param tmp_folder: path of the temporary folder, used to set default
input and output folders if not specified.
"""
tmp_path = pathlib.Path(tmp_folder)
if input_folder is None:
input_folder = tmp_path / '_'.join([self.label, 'input'])
check_dir_exists(input_folder, should_exist=True, mkdir=True)
self.input_folder = input_folder
logger.info('Input dir set to {}'.format(self.input_folder))
if output_folder is None:
output_folder = tmp_path / '_'.join([self.label, 'output'])
check_dir_exists(output_folder, should_exist=True, mkdir=True)
self.output_folder = output_folder
logger.info('Output dir set to {}'.format(self.output_folder))
if self.logger is not None:
self.logger.start_log_to_file(
directory=self.output_folder.as_posix())
return self
def parse_point_cloud(self):
"""
Parse input point cloud and get the following information:
- Tile list
- Length of a single band
- x and y resolution
"""
utils.check_dir_exists(self.input_path, should_exist=True)
# Get list of input tiles
self.InputTiles = [TileFile
for TileFile in os.listdir(self.input_path)
if TileFile.lower().endswith('.ply')]
if not self.InputTiles:
raise IOError('No PLY file in dir: {}'.format(self.input_path))
else:
logger.info('{} PLY files found'.format(len(self.InputTiles)))
# Read one tile and get the template
file = os.path.join(self.input_path, self.InputTiles[0])
template = plyfile.PlyData.read(file)
if not template.elements[0].name == 'vertex':
raise ValueError('Tile PLY file should '
'have vertex as first object')
# Get length of data record (Nr. of elements in each band)
self.LengthDataRecord = len(template.elements[0].data)
logger.info('No. of points per file: {}'.format(self.LengthDataRecord))
# Get resolution, assume a square tile
delta_x = (template.elements[0].data['x'].max()
- template.elements[0].data['x'].min())
delta_y = (template.elements[0].data['y'].max()
- template.elements[0].data['y'].min())
if numpy.isclose(delta_x, 0.) or numpy.isclose(delta_y, 0.):
raise ValueError('Tile should have finite extend in X and Y!')
self.xResolution = (delta_x / (numpy.sqrt(template.elements[0].data['x'].size) - 1))
self.yResolution = (delta_y / (numpy.sqrt(template.elements[0].data['y'].size) - 1))
if not (numpy.isclose(self.xResolution, self.yResolution) and
numpy.isclose(delta_x, delta_y)):
raise ValueError('Tile read is not square!')
logger.info('Resolution: ({}m x {}m)'.format(self.xResolution,
self.yResolution))
return self
def _get_output_file_dict(path,
file_handle='point_cloud',
features=[],
multi_band_files=True,
format='.ply',
overwrite=False,
**kwargs):
p = pathlib.Path(path)
if not p.suffix:
# expected dir
check_dir_exists(p, should_exist=True)
if features and not multi_band_files:
files = {}
for feature in features:
sub_path = p / feature
check_dir_exists(sub_path, should_exist=True, mkdir=True)
file_path = (sub_path / file_handle).with_suffix(format)
files.update({file_path.as_posix(): [feature]})
else:
file_path = (p / file_handle).with_suffix(format).as_posix()
if features:
files = {file_path: features}
else:
files = {file_path: 'all'}
else:
# expected file - check parent dir
check_dir_exists(p.parent, should_exist=True)
if features:
files = {p.as_posix(): features}
else:
files = {p.as_posix(): 'all'}
if not overwrite:
for file in files.keys():
check_file_exists(file, should_exist=False)
return files
def test_checkDirExists(self):
check_dir_exists(self._test_dir, True)
check_dir_exists(self._test_file_non_existent, False)
check_dir_exists(self._test_dir_non_existent, True, mkdir=True)
self.assertTrue(os.path.isdir(self._test_dir_non_existent))
with self.assertRaises(FileExistsError):
check_dir_exists(self._test_dir, False)
with self.assertRaises(NotADirectoryError):
check_dir_exists(self._test_file_path, True)
with self.assertRaises(FileExistsError):
check_dir_exists(self._test_file_path, False)
with self.assertRaises(FileNotFoundError):
check_dir_exists(self._test_file_non_existent, True)
def _check_input(self):
if not self.grid.is_set:
raise ValueError('The grid has not been set!')
check_file_exists(self.input_path, should_exist=True)
check_dir_exists(self.output_folder, should_exist=True)
def _get_export_path(self, filename=''):
check_dir_exists(self.output_folder, should_exist=True)
if pathlib.Path(filename).parent.name:
raise IOError('filename should not include path!')
return self.output_folder.joinpath(filename).as_posix()