def yield_xyz(self, entry):
src_data = 'gmrt_tmp.tif'
if f_utils.Fetch(entry[0],
callback=self.callback,
verbose=self.verbose).fetch_file(src_data) == 0:
gmrt_ds = datasets.RasterFile(
fn=src_data,
data_format=200,
src_srs='epsg:4326',
dst_srs=self.dst_srs,
weight=.25,
#name=src_data,
src_region=self.region,
verbose=self.verbose)
if self.bathy_only:
for xyz in gmrt_ds.yield_xyz():
if xyz.z < 0:
yield (xyz)
else:
for xyz in gmrt_ds.yield_xyz():
yield (xyz)
else:
utils.echo_error_msg(
'failed to fetch remote file, {}...'.format(src_data))
utils.remove_glob('{}*'.format(src_data))
def yield_xyz(self, entry):
src_ncei = os.path.basename(entry[1])
#try:
# src_ds = gdal.Open(entry[0])
# src_dc = entry[0]
#except Exception as e:
f_utils.Fetch(entry[0], callback=self.callback,
verbose=self.verbose).fetch_file(src_ncei)
try:
src_ds = gdal.Open(src_ncei)
except Exception as e:
utils.echo_error_msg(
'could not read ncei raster file: {}, {}'.format(entry[0], e))
src_ds = None
#except Exception as e:
# utils.echo_error_msg('could not read ncei raster file: {}, {}'.format(entry[0], e))
# src_ds = None
if src_ds is not None:
_ds = datasets.RasterFile(
fn=src_ncei,
data_format=200,
src_srs='epsg:4326',
dst_srs=self.dst_srs,
#name=src_ncei,
src_region=self.region,
verbose=self.verbose)
_ds.src_ds = src_ds
_ds.ds_open_p = True
for xyz in _ds.yield_xyz():
yield (xyz)
src_ds = None
utils.remove_glob(src_ncei)
def yield_xyz(self, entry):
src_dc = os.path.basename(entry[1])
src_ext = src_dc.split('.')[-1].lower()
if src_ext == 'laz' or src_ext == 'las': dt = 'lidar'
elif src_ext == 'tif' or src_ext == 'img': dt = 'raster'
else: dt = None
if dt == 'lidar':
if f_utils.Fetch(entry[0],
callback=self.callback,
verbose=self.verbose).fetch_file(src_dc) == 0:
_ds = datasets.LASFile(
fn=src_dc,
data_format=400,
dst_srs=self.dst_srs,
#name=src_dc,
src_region=self.region,
verbose=self.verbose,
remote=True)
if self.inc is not None:
b_region = regions.regions_reduce(
self.region,
regions.Region().from_list(_ds.infos['minmax']))
xyz_func = lambda p: _ds.dump_xyz(dst_port=p, encode=True)
for xyz in utils.yield_cmd(
'gmt blockmedian -I{:.10f} {} -r -V'.format(
self.inc, b_region.format('gmt')),
verbose=self.verbose,
data_fun=xyz_func):
yield (xyzfun.XYZPoint().from_list(
[float(x) for x in xyz.split()]))
else:
for xyz in _ds.yield_xyz():
yield (xyz)
utils.remove_glob('{}*'.format(src_dc))
elif dt == 'raster':
if f_utils.Fetch(entry[0],
callback=self.callback,
verbose=self.verbose).fetch_file(src_dc) == 0:
_ds = datasets.RasterFile(
fn=src_dc,
data_format=200,
dst_srs=self.dst_srs,
#src_srs=None,
name=src_dc,
src_region=self.region,
verbose=self.verbose)
for xyz in _ds.block_xyz(
inc=self.inc, want_gmt=True
) if self.inc is not None else _ds.yield_xyz():
yield (xyz)
utils.remove_glob('{}.*'.format(src_dc))
def yield_xyz(self, entry):
src_nos = os.path.basename(entry[1])
dt = None
if f_utils.Fetch(entry[0], callback=self.callback, verbose=self.verbose).fetch_file(src_nos) == 0:
dt = self._data_type(src_nos)
if dt == 'geodas_xyz':
nos_fns = utils.p_unzip(src_nos, ['xyz', 'dat'])
for nos_f_r in nos_fns:
_ds = datasets.XYZFile(
fn=nos_f_r,
data_format=168,
skip=1,
xpos=2,
ypos=1,
zpos=3,
z_scale=-1,
src_srs='epsg:4326',
dst_srs=self.dst_srs,
name=nos_f_r,
src_region=self.region,
verbose=self.verbose,
remote=True
)
for xyz in _ds.yield_xyz():
yield(xyz)
utils.remove_glob(*nos_fns, *[x+'.inf' for x in nos_fns])
elif dt == 'grid_bag':
src_bags = utils.p_unzip(src_nos, exts=['bag'])
for src_bag in src_bags:
_ds = datasets.RasterFile(
fn=src_bag,
data_format=200,
dst_srs=self.dst_srs,
name=src_bag,
src_region=self.region,
verbose=self.verbose
)
for xyz in _ds.yield_xyz():
yield(xyz)
utils.remove_glob(*src_bags)
utils.remove_glob(src_nos)
def yield_xyz(self, entry):
"""yield the xyz data from the nasadem fetch module"""
if f_utils.Fetch(entry[0], callback=self.callback, verbose=self.verbose, headers=self.headers).fetch_file(entry[1]) == 0:
_ds = datasets.RasterFile(
fn=entry[1],
data_format=200,
src_srs='epsg:4326',
dst_srs=self.dst_srs,
#name=entry[1],
src_region=self.region,
verbose=self.verbose
)
for xyz in _ds.yield_xyz():
if xyz.z != 0:
yield(xyz)
utils.remove_glob(entry[1])
def yield_xyz(self, entry):
src_emodnet = 'emodnet_tmp.tif'
if f_utils.Fetch(entry[0],
callback=self.callback,
verbose=self.verbose).fetch_file(src_emodnet) == 0:
_ds = datasets.RasterFile(fn=src_emodnet,
data_format=200,
src_srs='epsg:4326',
dst_srs=self.dst_srs,
name=src_emodnet,
src_region=self.region,
verbose=self.verbose)
for xyz in _ds.yield_xyz():
yield (xyz)
else:
utils.echo_error_msg(
'failed to fetch remote file, {}...'.format(src_emodnet))
utils.remove_glob(src_emodnet)
def yield_xyz(self, entry):
"""yield the xyz data from the copernicus fetch module"""
if f_utils.Fetch(entry[0],
callback=self.callback,
verbose=self.verbose,
headers=self.headers).fetch_file(entry[1]) == 0:
src_cop_dems = utils.p_unzip(entry[1], ['tif'])
for src_cop_dem in src_cop_dems:
_ds = datasets.RasterFile(
fn=src_cop_dem,
data_format=200,
src_srs='epsg:4326',
dst_srs=self.dst_srs,
#name=src_cop_dem,
src_region=self.region,
verbose=self.verbose)
for xyz in _ds.yield_xyz():
if xyz.z != 0:
yield (xyz)
utils.remove_glob(src_cop_dem, src_cop_dem + '.inf')
utils.remove_glob(entry[1])
def yield_xyz(self, entry):
src_zip = entry[1]
src_gtx = entry[2]
src_tif = '{}.tif'.format(utils.fn_basename(src_zip, 'zip'))
if f_utils.Fetch(entry[0],
callback=self.callback,
verbose=self.verbose).fetch_file(src_zip) == 0:
v_gtxs = utils.p_f_unzip(src_zip, [src_gtx])
utils.run_cmd('gdalwarp {} {} --config CENTER_LONG 0'.format(
src_gtx, src_tif))
_ds = datasets.RasterFile(
fn=src_tif,
data_format=200,
dst_srs=self.dst_srs,
src_srs=None,
#name=src_tif,
src_region=self.region,
verbose=self.verbose)
for xyz in _ds.yield_xyz():
yield (xyz)
utils.remove_glob(*v_gtxs, src_tif, src_zip)
def yield_xyz(self, entry):
"""yield the xyz data from the tnm fetch module"""
if f_utils.Fetch(entry[0],
callback=self.callback,
verbose=self.verbose).fetch_file(entry[1]) == 0:
datatype = entry[-1]
if datatype == 'ned':
src_tnm, src_zips = utils.procs_unzip(
entry[1], ['tif', 'img', 'gdal', 'asc', 'bag'])
_ds = datasets.RasterFile(fn=src_tnm,
data_format=200,
src_srs='epsg:4326',
dst_srs=self.dst_srs,
name=src_tnm,
src_region=self.region,
verbose=self.verbose)
for xyz in _ds.yield_xyz():
if xyz.z != 0:
yield (xyz)
utils.remove_glob(src_tnm, *src_zips)
utils.remove_glob(entry[1])
def yield_xyz(self, entry):
src_dc = os.path.basename(entry[1])
src_ext = src_dc.split('.')[-1].lower()
if src_ext == 'laz' or src_ext == 'las': dt = 'lidar'
elif src_ext == 'tif' or src_ext == 'img': dt = 'raster'
else: dt = None
if dt == 'lidar':
if f_utils.Fetch(entry[0],
callback=self.callback,
verbose=self.verbose).fetch_file(src_dc) == 0:
# xyz_dat = utils.yield_cmd('las2txt -stdout -parse xyz -keep_xy {} -keep_class {} -i {}\
# '.format(self.region.format('te'), '2 29', src_dc), verbose = False)
# _ds = datasets.XYZFile(fn=xyz_dat, data_format=168, dst_srs=self.dst_Srs,
# name=xyz_dat, src_region=self.region, verbose=self.verbose, remote=True)
#xyz_dat = utils.yield_cmd('las2txt -stdout -parse xyz -keep_xy {} -keep_class {} -i {}\
#'.format(self.region.format('te'), '2 29', src_dc), verbose = False)
_ds = datasets.LASFile(fn=src_dc,
data_format=400,
dst_srs=self.dst_srs,
name=src_dc,
src_region=self.region,
verbose=self.verbose,
remote=True)
if self.inc is not None:
b_region = regions.regions_reduce(
self.region,
regions.Region().from_list(_ds.infos['minmax']))
xyz_func = lambda p: _ds.dump_xyz(dst_port=p, encode=True)
for xyz in utils.yield_cmd(
'gmt blockmedian -I{:.10f} {} -r -V'.format(
self.inc, b_region.format('gmt')),
verbose=self.verbose,
data_fun=xyz_func):
yield (xyzfun.XYZPoint().from_list(
[float(x) for x in xyz.split()]))
else:
for xyz in _ds.yield_xyz():
yield (xyz)
#for xyz in _ds.block_xyz(inc=self.inc, want_gmt=True) if self.inc is not None else _ds.yield_xyz():
# yield(xyz)
#y = _ds.block_xyz if self.inc is not None else _ds.yield_xyz
#for xyz in y():
# yield(xyz)
elif dt == 'raster':
#try:
# src_ds = gdal.Open(entry[0])
# src_dc = entry[0]
#except Exception as e:
if f_utils.Fetch(entry[0],
callback=self.callback,
verbose=self.verbose).fetch_file(src_dc) == 0:
#try:
# src_ds = gdal.Open(src_dc)
#except Exception as e:
# utils.echo_error_msg('could not read dc raster file: {}, {}'.format(entry[0], e))
# src_ds = None
#except Exception as e:
# utils.echo_error_msg('could not read dc raster file: {}, {}'.format(entry[0], e))
# src_ds = None
#if src_ds is not None:
#src_ds = None
_ds = datasets.RasterFile(fn=src_dc,
data_format=200,
dst_srs=self.dst_srs,
src_srs=None,
name=src_dc,
src_region=self.region,
verbose=self.verbose)
#_ds.src_ds = src_ds
#_ds.ds_open_p = True
for xyz in _ds.block_xyz(
inc=self.inc, want_gmt=True
) if self.inc is not None else _ds.yield_xyz():
yield (xyz)
#src_ds = None
utils.remove_glob(src_dc)