def generate_inf(self, callback=lambda: False):
"""return the region of the datalist and generate
an associated `.inf` file if `inf_file` is True.
"""
_region = self.region
out_region = None
out_regions = []
self.region = None
self.infos['name'] = self.fn
self.infos['numpts'] = 0
self.infos['hash'] = self.hash() #dl_hash(self.fn)
self._init_datalist_vector()
for entry in self.parse():
if self.verbose:
callback()
if entry.src_srs is not None and self.dst_srs is not None:
e_region = regions.Region().from_list(entry.infos['minmax'])
e_region.src_srs = entry.src_srs
e_region.warp(self.dst_srs)
entry_region = e_region.export_as_list(include_z=True)
else:
entry_region = entry.infos['minmax']
if regions.Region().from_list(entry_region).valid_p():
self._create_entry_feature(
entry,
regions.Region().from_list(entry_region))
#out_regions.append(entry.infos['minmax'])
out_regions.append(entry_region)
if 'numpts' in self.infos.keys():
self.infos['numpts'] += entry.infos['numpts']
self.ds = self.layer = None
count = 0
for this_region in out_regions:
tmp_region = regions.Region().from_list(this_region)
if tmp_region.valid_p():
if count == 0:
out_region = tmp_region
count += 1
else:
out_region = regions.regions_merge(out_region, tmp_region)
if out_region is not None:
self.infos['minmax'] = out_region.export_as_list(include_z=True)
self.infos['wkt'] = out_region.export_as_wkt()
else:
self.infos['minmax'] = None
self.region = _region
if 'src_srs' not in self.infos.keys() or self.infos['src_srs'] is None:
self.infos['src_srs'] = self.src_srs
else:
self.src_srs = self.infos['src_srs']
return (self.infos)
def parse_(self):
import zipfile
exts = [
DatasetFactory().data_types[x]['fmts']
for x in DatasetFactory().data_types.keys()
]
exts = [x for y in exts for x in y]
datalist = []
if self.fn.split('.')[-1].lower() == 'zip':
with zipfile.ZipFile(self.fn) as z:
zfs = z.namelist()
for ext in exts:
for zf in zfs:
if ext == zf.split('.')[-1]:
datalist.append(os.path.basename(zf))
for this_data in datalist:
this_line = utils.p_f_unzip(self.fn, [this_data])[0]
data_set = DatasetFactory(this_line,
weight=self.weight,
parent=self,
src_region=self.region,
metadata=copy.deepcopy(self.metadata),
src_srs=self.src_srs,
dst_srs=self.dst_srs,
verbose=self.verbose).acquire()
if data_set is not None and data_set.valid_p(
fmts=DatasetFactory.data_types[
data_set.data_format]['fmts']):
if self.region is not None and self.region.valid_p(
check_xy=True):
try:
inf_region = regions.Region().from_string(
data_set.infos['wkt'])
except:
try:
inf_region = regions.Region().from_list(
data_set.infos['minmax'])
except:
inf_region = self.region.copy()
inf_region.wmin = data_set.weight
inf_region.wmax = data_set.weight
if regions.regions_intersect_p(inf_region, self.region):
for ds in data_set.parse():
self.data_entries.append(ds)
yield (ds)
else:
for ds in data_set.parse():
self.data_entries.append(ds)
yield (ds)
utils.remove_glob('{}*'.format(this_data))
def update(self):
self.FRED._open_ds()
v_zip = os.path.basename(self._hrdem_footprints_url)
status = f_utils.Fetch(self._hrdem_footprints_url,
verbose=self.verbose).fetch_ftp_file(v_zip)
v_shps = utils.p_unzip(v_zip, ['shp', 'shx', 'dbf', 'prj'])
v_shp = None
for v in v_shps:
if '.shp' in v: v_shp = v
shp_regions = regions.gdal_ogr_regions(v_shp)
shp_region = regions.Region()
for this_region in shp_regions:
if shp_region.valid_p(check_xy=True):
shp_region = regions.regions_merge(shp_region, this_region)
else:
shp_region = this_region
geom = shp_region.export_as_geom()
self.FRED._attribute_filter(["ID = '{}'".format('HRDEM-1')])
if self.FRED.layer is None or len(self.FRED.layer) == 0:
self.FRED._add_survey(Name='High-Resolution DEM (Canada)',
ID='HRDEM-1',
Agency='NRCAN',
Date=utils.this_year(),
MetadataLink=self._hrdem_info_url,
MetadataDate=utils.this_year(),
DataLink=self._hrdem_footprints_url,
IndexLink=self._hrdem_footprints_url,
DataType='raster',
DataSource='hrdem',
Info='Canada Only',
geom=geom)
utils.remove_glob(v_zip, *v_shps)
self.FRED._close_ds()
def bounds(self, geom=True):
wl = self.xml_doc.find('.//gmd:westBoundLongitude/gco:Decimal',
namespaces=self.namespaces)
el = self.xml_doc.find('.//gmd:eastBoundLongitude/gco:Decimal',
namespaces=self.namespaces)
sl = self.xml_doc.find('.//gmd:southBoundLatitude/gco:Decimal',
namespaces=self.namespaces)
nl = self.xml_doc.find('.//gmd:northBoundLatitude/gco:Decimal',
namespaces=self.namespaces)
if wl is not None and el is not None and sl is not None and nl is not None:
region = [
float(wl.text),
float(el.text),
float(sl.text),
float(nl.text)
]
if geom:
return (regions.Region().from_list([
float(wl.text),
float(el.text),
float(sl.text),
float(nl.text)
]).export_as_geom())
else:
return (region)
else:
return (None)
def update(self):
self.FRED._open_ds()
v_zip = os.path.basename(self._arctic_dem_index_url)
status = f_utils.Fetch(self._arctic_dem_index_url, verbose=self.verbose).fetch_file(v_zip)
v_shps = utils.p_unzip(v_zip, ['shp', 'shx', 'dbf', 'prj'])
v_shp = None
for v in v_shps:
if '.shp' in v: v_shp = v
utils.run_cmd('ogr2ogr arctic_tmp.shp {} -t_srs epsg:4326'.format(v_shp), verbose=True)
utils.remove_glob(v_zip, *v_shps)
v_shp = 'arctic_tmp.shp'
v_shps = ['arctic_tmp.shp','arctic_tmp.dbf','arctic_tmp.shx','arctic_tmp.prj']
shp_regions = regions.gdal_ogr_regions(v_shp)
shp_region = regions.Region()
for this_region in shp_regions:
#this_region.src_srs = 'epsg:3413'
#this_region.warp('epsg:4326')
if shp_region.valid_p(check_xy=True):
shp_region = regions.regions_merge(shp_region, this_region)
else: shp_region = this_region
geom = shp_region.export_as_geom()
self.FRED._attribute_filter(["ID = '{}'".format('ARCTICDEM-1')])
if self.FRED.layer is None or len(self.FRED.layer) == 0:
self.FRED._add_survey(Name = 'ArcticDEM', ID = 'ARCTICDEM-1', Agency = 'UMN', Date = utils.this_year(),
MetadataLink = 'https://data.pgc.umn.edu/elev/dem/setsm/ArcticDEM/', MetadataDate = utils.this_year(),
DataLink = self._arctic_dem_index_url, IndexLink = self._arctic_dem_index_url,
DataType = 'raster', DataSource = 'arcticdem', Info = 'Arctic Only', geom = geom)
utils.remove_glob(*v_shps)
self.FRED._close_ds()
def dist_region(self):
dr = regions.Region().from_region(self.region)
return(
dr.buffer(
x_bv=(self.xinc*self.extend),
y_bv=(self.yinc*self.extend)
)
)
def _get_coverage_region(self, cov_desc):
uc = [
float(x) for x in cov_desc["boundedBy"]["Envelope"]["upperCorner"]
[0].split()
]
lc = [
float(x) for x in cov_desc["boundedBy"]["Envelope"]["lowerCorner"]
[0].split()
]
return (regions.Region().from_list([lc[1], uc[1], lc[0], uc[0]]))
def yield_xyz(self, entry):
src_dc = os.path.basename(entry[1])
src_ext = src_dc.split('.')[-1].lower()
if src_ext == 'laz' or src_ext == 'las': dt = 'lidar'
elif src_ext == 'tif' or src_ext == 'img': dt = 'raster'
else: dt = None
if dt == 'lidar':
if f_utils.Fetch(entry[0],
callback=self.callback,
verbose=self.verbose).fetch_file(src_dc) == 0:
_ds = datasets.LASFile(
fn=src_dc,
data_format=400,
dst_srs=self.dst_srs,
#name=src_dc,
src_region=self.region,
verbose=self.verbose,
remote=True)
if self.inc is not None:
b_region = regions.regions_reduce(
self.region,
regions.Region().from_list(_ds.infos['minmax']))
xyz_func = lambda p: _ds.dump_xyz(dst_port=p, encode=True)
for xyz in utils.yield_cmd(
'gmt blockmedian -I{:.10f} {} -r -V'.format(
self.inc, b_region.format('gmt')),
verbose=self.verbose,
data_fun=xyz_func):
yield (xyzfun.XYZPoint().from_list(
[float(x) for x in xyz.split()]))
else:
for xyz in _ds.yield_xyz():
yield (xyz)
utils.remove_glob('{}*'.format(src_dc))
elif dt == 'raster':
if f_utils.Fetch(entry[0],
callback=self.callback,
verbose=self.verbose).fetch_file(src_dc) == 0:
_ds = datasets.RasterFile(
fn=src_dc,
data_format=200,
dst_srs=self.dst_srs,
#src_srs=None,
name=src_dc,
src_region=self.region,
verbose=self.verbose)
for xyz in _ds.block_xyz(
inc=self.inc, want_gmt=True
) if self.inc is not None else _ds.yield_xyz():
yield (xyz)
utils.remove_glob('{}.*'.format(src_dc))
def generate_inf(self, callback=lambda: False):
"""generate a infos dictionary from the Fetches dataset"""
self.infos['name'] = self.fn
self.infos['hash'] = None
self.infos['numpts'] = 0
if self.region is None:
#self.region = self.fetch_module.region
self.region = regions.Region().from_list([-180, 180, -90, 90])
self.infos['minmax'] = self.region.export_as_list()
self.infos['wkt'] = self.region.export_as_wkt()
return (self.infos)
def run(self):
'''Run the GMRT fetching module'''
if self.region is None:
return ([])
self.data = {
'north': self.region.ymax,
'west': self.region.xmin,
'south': self.region.ymin,
'east': self.region.xmax,
'mformat': 'json',
'resolution': self.res,
'format': self.fmt,
}
req = f_utils.Fetch(self._gmrt_grid_urls_url).fetch_req(
params=self.data, tries=10, timeout=2)
if req is not None:
try:
gmrt_urls = req.json()
except Exception as e:
utils.echo_error_msg(e)
gmrt_urls = []
for url in gmrt_urls:
if self.layer == 'topo-mask':
url = url.replace('topo', 'topo-mask')
opts = {}
for url_opt in url.split('?')[1].split('&'):
opt_kp = url_opt.split('=')
opts[opt_kp[0]] = opt_kp[1]
url_region = regions.Region().from_list([
float(opts['west']),
float(opts['east']),
float(opts['south']),
float(opts['north'])
])
outf = 'gmrt_{}_{}.tif'.format(opts['layer'],
url_region.format('fn'))
self.results.append(
[url, os.path.join(self._outdir, outf), 'gmrt'])
return (self)
def run(self):
"""Run the EMODNET fetching module"""
if self.region is None: return ([])
_data = {
'request': 'DescribeCoverage',
'version': '2.0.1',
'CoverageID': 'emodnet:mean',
'service': 'WCS',
}
_req = f_utils.Fetch(self._emodnet_grid_url).fetch_req(params=_data)
_results = lxml.etree.fromstring(_req.text.encode('utf-8'))
g_env = _results.findall(
'.//{http://www.opengis.net/gml/3.2}GridEnvelope',
namespaces=f_utils.namespaces)[0]
hl = [
float(x) for x in g_env.find(
'{http://www.opengis.net/gml/3.2}high').text.split()
]
g_bbox = _results.findall(
'.//{http://www.opengis.net/gml/3.2}Envelope')[0]
lc = [
float(x) for x in g_bbox.find(
'{http://www.opengis.net/gml/3.2}lowerCorner').text.split()
]
uc = [
float(x) for x in g_bbox.find(
'{http://www.opengis.net/gml/3.2}upperCorner').text.split()
]
ds_region = regions.Region().from_list([lc[1], uc[1], lc[0], uc[0]])
resx = (uc[1] - lc[1]) / hl[0]
resy = (uc[0] - lc[0]) / hl[1]
if regions.regions_intersect_ogr_p(self.region, ds_region):
emodnet_wcs = '{}service=WCS&request=GetCoverage&version=1.0.0&Identifier=emodnet:mean&coverage=emodnet:mean&format=GeoTIFF&bbox={}&resx={}&resy={}&crs=EPSG:4326'\
.format(self._emodnet_grid_url, self.region.format('bbox'), resx, resy)
outf = 'emodnet_{}.tif'.format(self.region.format('fn'))
self.results.append(
[emodnet_wcs,
os.path.join(self._outdir, outf), 'emodnet'])
return (self)
def update(self):
"""Crawl the COP30 database and update/generate the NASADEM reference vector."""
self.FRED._open_ds(1)
surveys = []
f = f_utils.Fetch(self.nasadem_vrt_url, headers=self.headers, verbose=True)
page = f.fetch_xml()
fns = page.findall('.//SourceFilename')
if self.verbose:
_prog = utils.CliProgress('scanning {} tiles in {}...'.format(len(fns), self.nasadem_url))
for i, fn in enumerate(fns):
sid = fn.text.split('/')[-1].split('.')[0]
if self.verbose:
_prog.update_perc((i, len(fns)))
self.FRED._attribute_filter(["ID = '{}'".format(sid)])
if self.FRED.layer is None or len(self.FRED.layer) == 0:
spat = fn.text.split('_HGT_')[-1].split('.')[0]
xsplit = 'e' if 'e' in spat else 'w'
ysplit = 's' if 's' in spat else 'n'
x = int(spat.split(xsplit)[-1])
y = int(spat.split(xsplit)[0].split(ysplit)[-1])
if xsplit == 'w':
x = x * -1
if ysplit == 's':
y = y * -1
this_region = regions.Region().from_list([x, x + 1, y, y + 1])
geom = this_region.export_as_geom()
if geom is not None:
surveys.append({'Name': fn.text.split('.')[0].split('/')[-1], 'ID': sid, 'Agency': 'NASA', 'Date': utils.this_date(),
'MetadataLink': '', 'MetadataDate': utils.this_date(), 'DataLink': self.nasadem_url + fn.text.split('/')[-1] + '?token=',
'DataType': '1', 'DataSource': 'nasadem', 'HorizontalDatum': 4326, 'Etcetra': self.nasadem_rurl,
'VerticalDatum': 'msl', 'Info': '', 'geom': geom})
if self.verbose:
_prog.end(0, 'scanned {} tiles in {}.'.format(len(fns), self.nasadem_url))
utils.echo_msg('added {} NASADEM DEM tiles'.format(len(surveys)))
self.FRED._add_surveys(surveys)
self.FRED._close_ds()
def generate_inf(self, callback=lambda: False):
"""return the region of the datalist and generate
an associated `.inf` file if `inf_file` is True.
"""
_region = self.region
out_region = None
out_regions = []
self.region = None
self.infos['name'] = self.fn
self.infos['numpts'] = 0
self.infos['hash'] = self.hash() #dl_hash(self.fn)
for entry in self.parse_():
if self.verbose:
callback()
out_regions.append(entry.infos['minmax'])
if 'numpts' in self.infos.keys():
self.infos['numpts'] += entry.infos['numpts']
count = 0
for this_region in out_regions:
tmp_region = regions.Region().from_list(this_region)
if tmp_region.valid_p():
if count == 0:
out_region = tmp_region
count += 1
else:
out_region = regions.regions_merge(out_region, tmp_region)
if out_region is not None:
self.infos['minmax'] = out_region.export_as_list(include_z=True)
self.infos['wkt'] = out_region.export_as_wkt()
else:
self.infos['minmax'] = None
self.region = _region
return (self.infos)
def _cdn_transform(self, epsg=None, name=None, invert=False):
"""create a cdn transofrmation grid"""
if epsg is not None:
cdn_results = cudem.fetches.vdatum.search_proj_cdn(self.src_region, epsg=epsg, cache_dir=self.cache_dir)
else: cdn_results = cudem.fetches.vdatum.search_proj_cdn(self.src_region, cache_dir=self.cache_dir)
for _result in cdn_results:
for g in _geoids:
if g in _result['name']:
cdn_results = [_result]
if len(cdn_results) > 0:
for _result in cdn_results:
src_code = int(_result['source_crs_code'].split(':')[-1])
dst_code = int(_result['target_crs_code'].split(':')[-1])
#if epsg == dst_code or epsg == src_code or np.any([g in _result['name'] for g in self._geoids]):
if epsg == dst_code or np.any([g in _result['name'] for g in _geoids]):
if src_code in _htdp_reference_frames.keys():
_trans_grid = os.path.join(self.cache_dir, _result['name'])
if cudem.fetches.utils.Fetch(_result['url'], verbose=self.verbose).fetch_file(_trans_grid) == 0:
tmp_infos = demfun.infos(_trans_grid)
tmp_region = regions.Region().from_geo_transform(tmp_infos['geoT'], tmp_infos['nx'], tmp_infos['ny'])
if os.path.exists('_{}'.format(os.path.basename(_trans_grid))):
utils.remove_glob('_{}'.format(os.path.basename(_trans_grid)))
utils.run_cmd('gdalwarp {} {} -s_srs epsg:4326 -te {} -ts {} {} --config CENTER_LONG 0'.format(
_trans_grid, '_{}'.format(os.path.basename(_trans_grid)), self.src_region.format('te'), self.xcount, self.ycount
), verbose=True)
_tmp_array, _tmp_infos = demfun.get_array('_{}'.format(os.path.basename(_trans_grid)))
utils.remove_glob('_{}'.format(os.path.basename(_trans_grid)))
if invert:
_tmp_array = _tmp_array * -1
return(_tmp_array, src_code)
utils.echo_error_msg('failed to locate transformation for {}'.format(epsg))
return(np.zeros( (self.ycount, self.xcount) ), epsg)
def update(self):
self.FRED._open_ds()
v_shp = None
v_zip = os.path.basename(self._hydrolakes_poly_zip)
status = f_utils.Fetch(self._hydrolakes_poly_zip,
verbose=self.verbose).fetch_file(v_zip)
v_shps = utils.p_unzip(v_zip, ['shp', 'shx', 'dbf', 'prj'])
for v in v_shps:
if '.shp' in v:
v_shp = v
shp_regions = regions.gdal_ogr_regions(v_shp)
shp_region = regions.Region()
for this_region in shp_regions:
if shp_region.valid_p(check_xy=True):
shp_region = regions.regions_merge(shp_region, this_region)
else:
shp_region = this_region
geom = shp_region.export_as_geom()
self.FRED._attribute_filter(["ID = '{}'".format('HYDROLAKES')])
if self.FRED.layer is None or len(self.FRED.layer) == 0:
self.FRED._add_survey(Name='HydorLakes',
ID='HYDROLAKES',
Agency='HydroSheds',
Date=utils.this_year(),
MetadataLink=self._hydrolakes_prods,
MetadataDate=utils.this_year(),
DataLink=self._hydrolakes_poly_zip,
IndexLink=self._hydrolakes_poly_zip,
DataType='vector',
DataSource='hydrolakes',
Info='World-wide lakes',
geom=geom)
utils.remove_glob(v_zip, *v_shps)
self.FRED._close_ds()
def _get_coverage_url(self, coverage, region=None):
dl_coverage = self.fix_coverage_id(coverage)
cov_desc = self._describe_coverage(coverage)
fmt = cov_desc["ServiceParameters"]["nativeFormat"][0]
hl = [
float(x) for x in cov_desc["domainSet"]["RectifiedGrid"]["limits"]
["GridEnvelope"]['high'][0].split()
]
uc = [
float(x) for x in cov_desc["boundedBy"]["Envelope"]["upperCorner"]
[0].split()
]
lc = [
float(x) for x in cov_desc["boundedBy"]["Envelope"]["lowerCorner"]
[0].split()
]
ds_region = regions.Region().from_list([lc[1], uc[1], lc[0], uc[0]])
resx = (uc[1] - lc[1]) / hl[0]
resy = (uc[0] - lc[0]) / hl[1]
data = {
'request': 'GetCoverage',
'version': '1.0.0',
'service': 'WCS',
'resx': resx,
'resy': resy,
'crs': 'EPSG:4326',
'format': fmt,
'coverage': coverage,
'Identifier': coverage
}
if region is not None: data['bbox'] = region.format('bbox')
enc_data = urlencode(data)
#try:
# enc_data = urllib.urlencode(data)
#except: enc_data = urllib.parse.urlencode(data)
return ('{}{}'.format(self.url, enc_data))
def main():
src_grid = None
dst_grid = None
vdatum_in = 5703
vdatum_out = 7662
verbose = False
keep_cache = False
cache_dir = utils.cudem_cache
i = 1
argv = sys.argv
while i < len(argv):
arg = argv[i]
if arg == '-i' or arg == '--vdatum_in':
vdatum_in = argv[i + 1]
i = i + 1
elif arg == '-o' or arg == '--vdatum_out':
vdatum_out = argv[i + 1]
i = i + 1
elif arg == '--cache-dir' or arg == '-D' or arg == '-cache-dir':
cache_dir = os.path.join(utils.str_or(argv[i + 1], os.path.expanduser('~')), '.cudem_cache')
i = i + 1
elif arg[:2] == '-D': cache_dir = os.path.join(utils.str_or(argv[i + 1], os.path.expanduser('~')), '.cudem_cache')
elif arg == '--list-epsg':
#print(_epsg_desc(htdpfun.HTDP()._reference_frames))
print(_epsg_desc('htdp epsg', vdatums._htdp_reference_frames))
print(_epsg_desc('cdn espg', vdatums._cdn_reference_frames))
print(_epsg_desc('tidal epsg', vdatums._tidal_frames))
#list_proj_cdn_epsg()
sys.exit(1)
elif arg == '-k' or arg == '--keep-cache': keep_cache = True
elif arg == '--verbose': verbose = True
elif arg == '-help' or arg == '--help' or arg == '-h':
print(_usage)
sys.exit(1)
elif arg == '-version' or arg == '--version':
print('vertical_datum_convert.py, version {}'.format(_version))
sys.exit(1)
elif src_grid is None:
src_grid = arg
elif dst_grid is None:
dst_grid = arg
else:
sys.stderr.write(_usage)
sys.exit(1)
i = i + 1
if src_grid is None:
sys.stderr.write(_usage)
sys.exit(1)
if dst_grid is None:
dst_grid = '.'.join(src_grid.split('.')[:-1]) + '_' + str(vdatum_out.replace('(', '_').replace(')', '_')) + '.' + src_grid.split('.')[-1]
if not os.path.exists(src_grid):
utils.echo_error_msg('Error: {} is not a valid file'.format(src_grid))
else:
src_infos = demfun.infos(src_grid)
src_region = regions.Region().from_geo_transform(src_infos['geoT'], src_infos['nx'], src_infos['ny'])
src_region.src_srs = demfun.get_srs(src_grid)
src_region.warp()
x_inc, y_inc = src_region.increments(src_infos['nx'], src_infos['ny'])
tmp_x_inc, tmp_y_inc = src_region.increments(src_infos['nx']/10, src_infos['ny']/10)
vt = vdatums.VerticalTransform(src_region, tmp_x_inc, tmp_y_inc, vdatum_in, vdatum_out, cache_dir=cache_dir)
_trans_grid = vt.run()
if _trans_grid is not None:
utils.run_cmd('gdalwarp {} {} -ts {} {} -s_srs epsg:4326 -t_srs {}'.format(_trans_grid, '_{}'.format(_trans_grid), src_infos['nx'], src_infos['ny'], demfun.get_srs(src_grid)), verbose=True)
utils.run_cmd('gdal_calc.py -A {} -B {} --calc "A+B" --outfile {}'.format(src_grid.replace(' ', '\ '), '_{}'.format(_trans_grid).replace(' ', '\ '), dst_grid.replace(' ', '\ ')), verbose=True)
utils.remove_glob(_trans_grid, '_{}'.format(_trans_grid))
else:
utils.echo_error_msg('could not parse input/output vertical datums: {} -> {}; check spelling, etc'.format(vdatum_in, vdatum_out))
if not keep_cache:
utils.remove_glob(cache_dir)
def yield_xyz(self, entry):
src_dc = os.path.basename(entry[1])
src_ext = src_dc.split('.')[-1].lower()
if src_ext == 'laz' or src_ext == 'las': dt = 'lidar'
elif src_ext == 'tif' or src_ext == 'img': dt = 'raster'
else: dt = None
if dt == 'lidar':
if f_utils.Fetch(entry[0],
callback=self.callback,
verbose=self.verbose).fetch_file(src_dc) == 0:
# xyz_dat = utils.yield_cmd('las2txt -stdout -parse xyz -keep_xy {} -keep_class {} -i {}\
# '.format(self.region.format('te'), '2 29', src_dc), verbose = False)
# _ds = datasets.XYZFile(fn=xyz_dat, data_format=168, dst_srs=self.dst_Srs,
# name=xyz_dat, src_region=self.region, verbose=self.verbose, remote=True)
#xyz_dat = utils.yield_cmd('las2txt -stdout -parse xyz -keep_xy {} -keep_class {} -i {}\
#'.format(self.region.format('te'), '2 29', src_dc), verbose = False)
_ds = datasets.LASFile(fn=src_dc,
data_format=400,
dst_srs=self.dst_srs,
name=src_dc,
src_region=self.region,
verbose=self.verbose,
remote=True)
if self.inc is not None:
b_region = regions.regions_reduce(
self.region,
regions.Region().from_list(_ds.infos['minmax']))
xyz_func = lambda p: _ds.dump_xyz(dst_port=p, encode=True)
for xyz in utils.yield_cmd(
'gmt blockmedian -I{:.10f} {} -r -V'.format(
self.inc, b_region.format('gmt')),
verbose=self.verbose,
data_fun=xyz_func):
yield (xyzfun.XYZPoint().from_list(
[float(x) for x in xyz.split()]))
else:
for xyz in _ds.yield_xyz():
yield (xyz)
#for xyz in _ds.block_xyz(inc=self.inc, want_gmt=True) if self.inc is not None else _ds.yield_xyz():
# yield(xyz)
#y = _ds.block_xyz if self.inc is not None else _ds.yield_xyz
#for xyz in y():
# yield(xyz)
elif dt == 'raster':
#try:
# src_ds = gdal.Open(entry[0])
# src_dc = entry[0]
#except Exception as e:
if f_utils.Fetch(entry[0],
callback=self.callback,
verbose=self.verbose).fetch_file(src_dc) == 0:
#try:
# src_ds = gdal.Open(src_dc)
#except Exception as e:
# utils.echo_error_msg('could not read dc raster file: {}, {}'.format(entry[0], e))
# src_ds = None
#except Exception as e:
# utils.echo_error_msg('could not read dc raster file: {}, {}'.format(entry[0], e))
# src_ds = None
#if src_ds is not None:
#src_ds = None
_ds = datasets.RasterFile(fn=src_dc,
data_format=200,
dst_srs=self.dst_srs,
src_srs=None,
name=src_dc,
src_region=self.region,
verbose=self.verbose)
#_ds.src_ds = src_ds
#_ds.ds_open_p = True
for xyz in _ds.block_xyz(
inc=self.inc, want_gmt=True
) if self.inc is not None else _ds.yield_xyz():
yield (xyz)
#src_ds = None
utils.remove_glob(src_dc)
def update(self):
"""Update or create the reference vector file"""
self.FRED._open_ds(1)
for vd in self._vdatums:
surveys = []
if vd == 'TIDAL' or vd == 'IGLD85':
## ==============================================
## All tidal inf data is in each one, so we only
## have to download one of the tidal zips to process
## them all; lets use the smallest one
## Keep this link up-to-date!
## ==============================================
if vd == 'TIDAL':
vd_ = 'DEVAemb12_8301'
else:
vd_ = vd
vd_zip_url = '{}{}.zip'.format(self._vdatum_data_url, vd_)
v_inf = 'tidal_area.inf'
elif vd == 'VERTCON':
vd_zip_url = '{}vdatum_{}.zip'.format(self._vdatum_data_url,
vd)
v_inf = 'vcn.inf'
else:
vd_zip_url = '{}vdatum_{}.zip'.format(self._vdatum_data_url,
vd)
v_inf = '{}.inf'.format(vd.lower())
if f_utils.Fetch(vd_zip_url, verbose=True).fetch_file(
'{}.zip'.format(vd)) == 0:
v_infs = utils.p_unzip('{}.zip'.format(vd), ['inf'])
v_dict = proc_vdatum_inf(
v_inf, name=vd if vd != 'TIDAL' else
None) #, loff=-360 if vd =='TIDAL' else -180)
v_dict = proc_vdatum_inf(
v_inf, name=vd if vd != 'TIDAL' else None) #, loff=-360)
for key in v_dict.keys():
v_dict[key]['vdatum'] = vd
v_dict[key]['remote'] = vd_zip_url
## tidal datums:
if vd == 'TIDAL':
v_dict_ = {}
for tidal_key in v_dict.keys():
for t in self._tidal_datums:
key_ = '{}_{}'.format(t, tidal_key)
v_dict_[key_] = {}
v_dict_[key_]['region'] = v_dict[tidal_key][
'region']
v_dict_[key_]['vdatum'] = t
v_dict_[key_]['grid'] = '{}.gtx'.format(t)
v_dict_[key_]['remote'] = '{}{}.zip'.format(
self._vdatum_data_url, tidal_key)
v_dict = v_dict_
print(v_dict)
for key in v_dict.keys():
self.FRED._attribute_filter(["ID = '{}'".format(key)])
if self.FRED.layer is None or len(self.FRED.layer) == 0:
geom = regions.Region().from_list(
v_dict[key]['region']).export_as_geom()
if geom is not None:
surveys.append({
'Name':
v_dict[key]['grid'],
'ID':
key,
'Agency':
'NOAA',
'Date':
utils.this_date(),
'MetadataLink':
"",
'MetadataDate':
utils.this_date(),
'DataLink':
v_dict[key]['remote'],
'Link':
self._vdatum_data_url,
'DataType':
v_dict[key]['vdatum'],
'DataSource':
'vdatum',
'HorizontalDatum':
4326,
'VerticalDatum':
v_dict[key]['vdatum'],
'Info':
"",
'geom':
geom
})
self.FRED._add_surveys(surveys)
utils.remove_glob(*v_infs, '{}.zip'.format(vd))
self.FRED._close_ds()
def dist_region(self):
dr = regions.Region().from_region(self.region)
return (dr.buffer((self.inc * self.extend)))
elif arg == '-r' or arg == '--grid-node':
node = 'grid'
elif arg == '-p' or arg == '--prefix':
want_prefix = True
elif arg == '-help' or arg == '--help' or arg == '-h':
sys.stderr.write(_usage)
sys.exit(1)
elif arg == '-version' or arg == '--version':
sys.stderr.write('{}\n'.format(_version))
sys.exit(1)
else:
dls.append(arg)
i = i + 1
for i_region in i_regions:
tmp_region = regions.Region().from_string(i_region)
if tmp_region.valid_p():
these_regions.append(tmp_region)
else:
tmp_region = regions.ogr_wkts(i_region)
for i in tmp_region:
if i.valid_p():
these_regions.append(i)
if len(these_regions) == 0:
these_regions = [None]
else:
if want_verbose:
utils.echo_msg('parsed {} region(s)'.format(len(these_regions)))
for rn, this_region in enumerate(these_regions):
def update(self):
"""Crawl the COP30 database and update/generate the COPERNICUS reference vector."""
self.FRED._open_ds(1)
surveys = []
page = f_utils.Fetch(self.cop_10_url, verbose=True).fetch_html()
rows = page.xpath('//a[contains(@href, ".zip")]/@href')
if self.verbose:
_prog = utils.CliProgress('scanning {} tiles in {}...'.format(
len(rows), self.cop_10_url))
for i, row in enumerate(rows):
sid = row.split('.')[0]
if self.verbose:
_prog.update_perc((i, len(rows)))
self.FRED._attribute_filter(["ID = '{}'".format(sid)])
if self.FRED.layer is None or len(self.FRED.layer) == 0:
spat = row.split('.')[0].split('_')[-1]
x = int(spat.split('x')[-1])
y = int(spat.split('x')[0].split('y')[-1])
this_region = regions.Region().from_list(
[x, x + 10, y, y + 10])
geom = this_region.export_as_geom()
if geom is not None:
surveys.append({
'Name': row.split('.')[0],
'ID': sid,
'Agency': 'EU',
'Date': utils.this_date(),
'MetadataLink': self.cop_10_aux_url,
'MetadataDate': utils.this_date(),
'DataLink': self.cop_10_url + row,
'DataType': '3',
'DataSource': 'copernicus',
'HorizontalDatum': 'epsg:4326',
'VerticalDatum': 'msl',
'Info': '',
'geom': geom
})
if self.verbose:
_prog.end(
0, 'scanned {} tiles in {}.'.format(len(rows),
self.cop_10_url))
f = f_utils.Fetch(self.cop30_vrt_url,
headers=self.headers,
verbose=True)
page = f.fetch_xml()
fns = page.findall('.//SourceFilename')
if self.verbose:
_prog = utils.CliProgress('scanning {} tiles in {}...'.format(
len(fns), self.cop30_url))
for i, fn in enumerate(fns):
sid = fn.text.split('/')[-1].split('.')[0]
if self.verbose:
_prog.update_perc((i, len(fns)))
self.FRED._attribute_filter(["ID = '{}'".format(sid)])
if self.FRED.layer is None or len(self.FRED.layer) == 0:
spat = fn.text.split('_10_')[-1].split('_DEM')[0]
xsplit = '_E' if 'E' in spat else '_W'
ysplit = 'S' if 'S' in spat else 'N'
x = int(spat.split(xsplit)[-1].split('_')[0])
y = int(spat.split(xsplit)[0].split(ysplit)[-1].split('_')[0])
if xsplit == '_W':
x = x * -1
if ysplit == 'S':
y = y * -1
this_region = regions.Region().from_list([x, x + 1, y, y + 1])
geom = this_region.export_as_geom()
if geom is not None:
surveys.append({
'Name':
fn.text.split('.')[0].split('/')[-1],
'ID':
sid,
'Agency':
'EU',
'Date':
utils.this_date(),
'MetadataLink':
'',
'MetadataDate':
utils.this_date(),
'DataLink':
self.cop30_url + fn.text.split('/')[-1] + '?token=',
'DataType':
'1',
'DataSource':
'copernicus',
'HorizontalDatum':
'epsg:4326',
'Etcetra':
self.cop30_rurl,
'VerticalDatum':
'msl',
'Info':
'',
'geom':
geom
})
if self.verbose:
_prog.end(
0, 'scanned {} tiles in {}.'.format(len(fns), self.cop30_url))
utils.echo_msg('added {} COPERNICUS DEM tiles'.format(
len(surveys)))
self.FRED._add_surveys(surveys)
self.FRED._close_ds()
verbose = True
elif arg[0] == '-':
sys.stderr.write(_usage)
sys.exit(1)
elif output is None:
output = arg
else:
sys.stderr.write(_usage)
sys.exit(1)
i = i + 1
if output is None:
sys.stderr.write(_usage)
utils.echo_error_msg('you must enter an output file name')
sys.exit(0)
if extent == None:
extent = '1'
else:
this_region = regions.Region().from_list(extent)
#Run the program given the user input
if cpgrd is not None:
createNullCopy(cpgrd, output, nodata, d_format, verbose, overwrite)
else:
createGrid(output, this_region, cellsize, nodata, d_format, verbose,
overwrite)
### END
def parse(self):
"""import a datalist entry from a string"""
if self.verbose:
_prog = utils.CliProgress('parsing datalist {}{}'.format(
self.fn,
' @{}'.format(self.weight) if self.weight is not None else ''))
if os.path.exists(self.fn):
with open(self.fn, 'r') as f:
count = sum(1 for _ in f)
with open(self.fn, 'r') as op:
for l, this_line in enumerate(op):
if self.verbose:
_prog.update_perc((l, count))
if this_line[0] != '#' and this_line[
0] != '\n' and this_line[0].rstrip() != '':
data_set = DatasetFactory(
this_line,
weight=self.weight,
parent=self,
src_region=self.region,
metadata=copy.deepcopy(self.metadata),
src_srs=self.src_srs,
dst_srs=self.dst_srs,
x_inc=self.x_inc,
y_inc=self.y_inc,
sample_alg=self.sample_alg,
verbose=self.verbose).acquire()
if data_set is not None and data_set.valid_p(
fmts=DatasetFactory.data_types[
data_set.data_format]['fmts']):
if self.region is not None and self.region.valid_p(
check_xy=True):
# try:
# inf_region = regions.Region().from_string(
# data_set.infos['wkt']
# )
# except:
try:
inf_region = regions.Region().from_list(
data_set.infos['minmax'])
except:
inf_region = self.region.copy()
inf_region.wmin = data_set.weight
inf_region.wmax = data_set.weight
if regions.regions_intersect_p(
inf_region, self.region
if data_set.dst_trans is None else
data_set.trans_region):
for ds in data_set.parse():
self.data_entries.append(ds)
yield (ds)
else:
for ds in data_set.parse():
self.data_entries.append(ds)
yield (ds)
elif len(self.data_entries) > 0:
for data_set in self.data_entries:
for ds in data_set.parse():
yield (ds)
else:
if self.verbose:
utils.echo_warning_msg(
'could not open datalist/entry {}'.format(self.fn))
if self.verbose:
_prog.end(
0, 'parsed datalist {}{}'.format(
self.fn, ' @{}'.format(self.weight)
if self.weight is not None else ''))
def spat_meta_cli(argv = sys.argv):
i = 1
dls = []
i_regions = []
these_regions = []
src_srs = 'epsg:4326'
xinc = utils.str2inc('1s')
yinc = utils.str2inc('1s')
node = 'pixel'
name = 'waffles_spat'
ogr_format = 'ESRI Shapefile'
extend = 0
want_verbose = True
want_prefix = False
want_recursive = False
prefix_args = {}
argv = sys.argv
while i < len(argv):
arg = sys.argv[i]
if arg == '--region' or arg == '-R':
i_regions.append(str(argv[i + 1]))
i = i + 1
elif arg[:2] == '-R':
i_regions.append(str(arg[2:]))
elif arg == '--outname' or arg == '-O':
name = argv[i + 1]
i += 1
elif arg[:2] == '-O': name = arg[2:]
elif arg == '-s_srs' or arg == '--s_srs' or arg == '-P':
src_srs = argv[i + 1]
i = i + 1
elif arg == '--increment' or arg == '-E':
incs = argv[i + 1].split(':')
xy_inc = incs[0].split('/')
xinc = utils.str2inc(xy_inc[0])
if len(xy_inc) > 1:
yinc = utils.str2inc(xy_inc[1])
else:
yinc = utils.str2inc(xy_inc[0])
i = i + 1
elif arg[:2] == '-E':
incs = arg[2:].split(':')
xy_inc = incs[0].split('/')
xinc = utils.str2inc(xy_inc[0])
if len(xy_inc) > 1:
yinc = utils.str2inc(xy_inc[1])
else:
yinc = utils.str2inc(xy_inc[0])
elif arg == '--extend' or arg == '-X':
exts = argv[i + 1].split(':')
extend = utils.int_or(exts[0], 0)
i += 1
elif arg[:2] == '-X':
exts = arg[2:].split(':')
extend = utils.int_or(exts[0], 0)
elif arg == '--format' or arg == '-F':
ogr_format = argv[i + 1]
i += 1
elif arg[:2] == '-F':
ogr_format = argv[2:]
elif arg == '-p' or arg == '--prefix':
want_prefix = True
prefix_opts = argv[i + 1].split(':')
prefix_args = utils.args2dict(prefix_opts, prefix_args)
if len(prefix_args) > 0:
i += 1
elif arg == '-r' or arg == '--grid-node': node = 'grid'
elif arg == '-c' or arg == '--recursive': want_recursive = True
elif arg == '--quiet' or arg == '-q': want_verbose = False
elif arg == '-help' or arg == '--help' or arg == '-h':
sys.stderr.write(_usage)
sys.exit(1)
elif arg == '-version' or arg == '--version':
sys.stdout.write('{}\n'.format(__version__))
sys.exit(1)
else: dls.append(arg)
i = i + 1
for i_region in i_regions:
tmp_region = regions.Region().from_string(i_region)
if tmp_region.valid_p(check_xy=True):
these_regions.append(tmp_region)
else:
i_region_s = i_region.split(':')
tmp_region = regions.ogr_wkts(i_region_s[0])
for i in tmp_region:
if i.valid_p():
if len(i_region_s) > 1:
these_regions.append(
regions.Region().from_string(
'/'.join([i.format('str'), i_region_s[1]])
)
)
else:
these_regions.append(i)
if len(these_regions) == 0:
these_regions = [None]
utils.echo_error_msg('Could not parse region {}'.format(these_regions))
sys.stderr.write('{}\n'.format(_usage))
sys.exit(1)
else:
if want_verbose:
utils.echo_msg(
'parsed {} region(s)'.format(len(these_regions))
)
name_ = name
for rn, this_region in enumerate(these_regions):
utils.echo_msg('using region {}'.format(this_region.format('gmt')))
if len(dls) == 0:
sys.stderr.write(_usage)
utils.echo_error_msg('you must specify some type of data')
else:
if want_prefix or len(these_regions) > 1:
name_ = utils.append_fn(name, this_region, xinc, **prefix_args)
if os.path.exists('{}_sm.{}'.format(name_, utils.ogr_fext(ogr_format))):
utils.echo_msg(
'SPATIAL METADATA {} already exists, skipping...'.format('{}_sm.{}'.format(name_, utils.ogr_fext(ogr_format)))
)
else:
SpatialMetadata(
data=dls,
src_region=this_region,
xinc=xinc,
yinc=yinc,
extend=extend,
src_srs=src_srs,
node=node,
name=name_,
verbose=want_verbose,
recursive=want_recursive,
ogr_format=ogr_format
).run()
def datalists_cli(argv=sys.argv):
"""run datalists from command-line
See `datalists_cli_usage` for full cli options.
"""
dls = []
src_srs = None
dst_srs = None
i_regions = []
these_regions = []
xy_inc = [None, None]
want_weights = False
want_inf = False
want_list = False
want_glob = False
want_archive = False
want_verbose = True
want_region = False
want_csv = False
want_json = False
want_datalists = False
want_separate = False
## ==============================================
## parse command line arguments.
## ==============================================
i = 1
while i < len(argv):
arg = argv[i]
if arg == '--region' or arg == '-R':
i_regions.append(str(argv[i + 1]))
i = i + 1
elif arg[:2] == '-R':
i_regions.append(str(arg[2:]))
elif arg == '--increment' or arg == '-E':
xy_inc = argv[i + 1].split('/')
i = i + 1
elif arg[:2] == '-E':
xy_inc = arg[2:].split('/')
elif arg == '-s_srs' or arg == '--s_srs' or arg == '-P':
src_srs = argv[i + 1]
i = i + 1
elif arg == '-t_srs' or arg == '--t_srs' or arg == '-W':
dst_srs = argv[i + 1]
i = i + 1
elif arg == '--separate' or arg == '-s':
want_separate = True
elif arg == '--archive' or arg == '-a':
want_archive = True
elif arg == '--weights' or arg == '-w':
want_weights = True
elif arg == '--info' or arg == '-i':
want_inf = True
elif arg == '--region_inf' or arg == '-r':
want_region = True
elif arg == '--list' or arg == '-l':
want_list = True
elif arg == '--glob' or arg == '-g':
want_glob = True
elif arg == '--datalists' or arg == '-d':
want_datalists = True
elif arg == '--csv' or arg == '-c':
want_csv = True
elif arg == '--json' or arg == '-j':
want_json = True
elif arg == '--quiet' or arg == '-q':
want_verbose = False
elif arg == '--help' or arg == '-h':
print(datalists_usage)
sys.exit(1)
elif arg == '--version' or arg == '-v':
print('{}, version {}'.format(os.path.basename(sys.argv[0]),
cudem.__version__))
sys.exit(1)
elif arg[0] == '-':
print(datalists_usage)
sys.exit(0)
else:
dls.append(arg)
i = i + 1
if len(xy_inc) < 2:
xy_inc.append(xy_inc[0])
elif len(xy_inc) == 0:
xy_inc = [None, None]
if want_glob:
import glob
for key in DatasetFactory().data_types.keys():
if key != -1:
for f in DatasetFactory().data_types[key]['fmts']:
globs = glob.glob('*.{}'.format(f))
[
sys.stdout.write('{}\n'.format(' '.join(
[x, str(key), '1']))) for x in globs
]
sys.exit(0)
for i_region in i_regions:
tmp_region = regions.Region().from_string(i_region)
if tmp_region.valid_p(check_xy=True):
these_regions.append(tmp_region)
else:
i_region_s = i_region.split(':')
tmp_region = regions.ogr_wkts(i_region_s[0])
for i in tmp_region:
if i.valid_p():
if len(i_region_s) > 1:
these_regions.append(regions.Region().from_string(
'/'.join([i.format('str'), i_region_s[1]])))
else:
these_regions.append(i)
if len(these_regions) == 0:
these_regions = [None]
else:
if want_verbose:
utils.echo_msg('parsed {} region(s)'.format(len(these_regions)))
for rn, this_region in enumerate(these_regions):
if len(dls) == 0:
sys.stderr.write(datalists_usage)
utils.echo_error_msg('you must specify some type of data')
else:
this_datalist = init_data(dls, this_region, src_srs, dst_srs,
xy_inc, 'bilinear', want_verbose)
if this_datalist is not None and this_datalist.valid_p(
fmts=DatasetFactory.data_types[
this_datalist.data_format]['fmts']):
if not want_weights:
this_datalist.weight = None
if want_inf:
print(this_datalist.inf())
elif want_list:
this_datalist.echo()
elif want_archive:
[x for x in this_datalist.archive_xyz()]
elif want_region:
print(regions.Region().from_list(
this_datalist.inf()['minmax']).format('gmt'))
elif want_csv:
this_datalist.parse_data_lists()
for x in this_datalist.data_lists.keys():
this_datalist.data_entries = this_datalist.data_lists[
x]['data']
p = this_datalist.data_lists[x]['parent']
print('|'.join([
'"{}"'.format(str(y)) for y in [
x, p.metadata['title'] if p.metadata['title']
is not None else x, p.metadata['source'],
p.metadata['date'], p.metadata['data_type'],
p.metadata['resolution'], p.metadata['hdatum'],
p.metadata['vdatum'], p.metadata['url']
]
]))
elif want_datalists:
#import json
#j = open('{}.json'.format(this_datalist.metadata['name']), 'w')
this_datalist.parse_data_lists()
# for x in this_datalist.data_lists.keys():
# p = this_datalist.data_lists[x]['parent']
# out_json = {
# "Name": x,
# "Title": p.title if p.title is not None else x,
# "Source": p.source,
# "Date": p.date,
# "DataType": p.data_type,
# "Resolution": p.resolution,
# "HDatum": p.hdatum,
# "VDatum": p.vdatum,
# "URL": p.url
# }
# j.write(json.dumps(out_json))
# j.write('\n')
# j.close()
for x in this_datalist.data_lists.keys():
p = this_datalist.data_lists[x]['parent']
print('{} ({})|{}'.format(p.metadata['title'],
p.metadata['name'],
p.weight))
#print(xdl.data_lists[x]['parent'].echo_())
#print('{}'.format(this_datalist.data_lists[x]['parent'].fn))
#print(this_datalist.data_lists[x])
#print('{} ({})|{}'.format(this_datalist.data_lists[x]['parent'].metadata['title'], this_datalist.data_lists[x]['parent'].metadata['name'], this_datalist.data_lists[x]['parent'].weight))
else:
if want_separate:
try:
for this_entry in this_datalist.parse():
this_entry.dump_xyz()
except KeyboardInterrupt:
utils.echo_error_msg('Killed by user')
break
except BrokenPipeError:
utils.echo_error_msg('Pipe Broken')
break
else:
try:
this_datalist.dump_xyz()
except KeyboardInterrupt:
utils.echo_error_msg('Killed by user')
break
except BrokenPipeError:
utils.echo_error_msg('Pipe Broken')
break
def fetches_cli(argv=sys.argv):
"""run fetches from command-line
See `fetches_cli_usage` for full cli options.
"""
i_regions = []
these_regions = []
mods = []
mod_opts = {}
want_list = False
want_proc = False
want_verbose = True
stop_threads = False
## ==============================================
## parse command line arguments.
## ==============================================
i = 1
while i < len(argv):
arg = argv[i]
if arg == '--region' or arg == '-R':
i_regions.append(str(argv[i + 1]))
i = i + 1
elif arg[:2] == '-R':
i_regions.append(str(arg[2:]))
elif arg == '--list' or arg == '-l':
want_list = True
elif arg == '--process' or arg == '-p':
want_proc = True
elif arg == '--quiet' or arg == '-q':
want_verbose = False
elif arg == '--help' or arg == '-h':
sys.stderr.write(fetches_usage)
sys.exit(1)
elif arg == '--version' or arg == '-v':
print('{}, version {}'.format(os.path.basename(sys.argv[0]),
fetches.__version__))
sys.exit(1)
elif arg == '--modules' or arg == '-m':
try:
if argv[i + 1] in FetchesFactory.mods.keys():
sys.stderr.write(
_fetches_module_long_desc({
k: FetchesFactory.mods[k]
for k in (argv[i + 1], )
}))
else:
sys.stderr.write(
_fetches_module_long_desc(FetchesFactory.mods))
except:
sys.stderr.write(_fetches_module_long_desc(
FetchesFactory.mods))
sys.exit(0)
elif arg[0] == '-':
sys.stderr.write(fetches_usage)
sys.exit(0)
else:
mods.append(arg)
i = i + 1
if len(mods) == 0:
sys.stderr.write(fetches_usage)
utils.echo_error_msg('you must select at least one fetch module')
sys.exit(-1)
for i_region in i_regions:
tmp_region = regions.Region().from_string(i_region)
if tmp_region.valid_p(check_xy=True):
these_regions.append(tmp_region)
else:
i_region_s = i_region.split(':')
tmp_region = regions.ogr_wkts(i_region_s[0])
for i in tmp_region:
if i.valid_p():
if len(i_region_s) > 1:
these_regions.append(regions.Region().from_string(
'/'.join([i.format('str'), i_region_s[1]])))
else:
these_regions.append(i)
if not these_regions:
these_regions = [regions.Region().from_string('-R-180/180/-90/90')]
if want_verbose:
utils.echo_msg('parsed {} region(s)'.format(len(these_regions)))
for rn, this_region in enumerate(these_regions):
if stop_threads:
return
x_fs = [
FetchesFactory(mod=mod,
src_region=this_region,
verbose=want_verbose).acquire(dst_srs='epsg:4326')
for mod in mods
]
for x_f in x_fs:
if x_f is None:
continue
if want_verbose:
utils.echo_msg(
'running fetch module {} on region {}...'.format(
x_f.name, this_region.format('str')))
x_f.run()
if want_verbose:
utils.echo_msg('found {} data files.'.format(len(x_f.results)))
if len(x_f.results) == 0:
break
if want_list:
for result in x_f.results:
print(result[0])
else:
fr = f_utils.fetch_results(x_f, want_proc=want_proc)
fr.daemon = True
_p = utils.CliProgress('fetching {} remote data files'.format(
len(x_f.results)))
try:
fr.start()
while True:
time.sleep(2)
sys.stderr.write('\x1b[2K\r')
perc = float(
(len(x_f.results) - fr.fetch_q.qsize())) / len(
x_f.results) * 100 if len(
x_f.results) > 0 else 1
if want_verbose:
_p.update_perc(
(len(x_f.results) - fr.fetch_q.qsize(),
len(x_f.results)))
sys.stderr.flush()
if not fr.is_alive():
break
except (KeyboardInterrupt, SystemExit):
utils.echo_error_msg(
'user breakage...please wait for while fetches exits.')
x_f.status = -1
stop_threads = True
while not fr.fetch_q.empty():
try:
fr.fetch_q.get(False)
except Empty:
continue
fr.fetch_q.task_done()
fr.join()
_p.end(x_f.status,
'fetched {} remote data files'.format(len(x_f.results)))
if want_verbose:
utils.echo_msg('ran fetch module {} on region {}...\
'.format(x_f.name, this_region.format('str')))
def _update_prods(self):
"""updated FRED with each product file available from TNM"""
for dsTag in self._elev_ds:
offset = 0
utils.echo_msg('processing TNM dataset {}...'.format(dsTag))
_req = f_utils.Fetch(
self._tnm_product_url).fetch_req(params={
'max': 1,
'datasets': dsTag
})
try:
_dsTag_results = _req.json()
except ValueError:
utils.echo_error_msg('tnm server error, try again')
except Exception as e:
utils.echo_error_msg('error, {}'.format(e))
total = _dsTag_results['total']
if self.verbose:
_prog = utils.CliProgress(
'gathering {} products from {}...'.format(total, dsTag))
ds = self._datasets(dataset=dsTag)
#this_xml = f_utils.iso_xml('{}{}?format=iso'.format(self._tnm_meta_base, ds['id']))
this_xml = f_utils.iso_xml('{}?format=iso'.format(ds['infoUrl']))
h_epsg, v_epsg = this_xml.reference_system()
while True:
_data = {'max': 100, 'datasets': dsTag, 'offset': offset}
_req = f_utils.Fetch(
self._tnm_product_url).fetch_req(params=_data)
try:
_dsTag_results = _req.json()
except ValueError:
utils.echo_error_msg('tnm server error, try again')
except Exception as e:
utils.echo_error_msg('error, {}'.format(e))
if self.verbose:
_prog.update_perc(
(offset, total),
msg='gathering {} products from {}...'.format(
total, dsTag))
for i, item in enumerate(_dsTag_results['items']):
if self.verbose:
_prog.update_perc(
(i + offset, total),
msg='gathering {} products from {}...'.format(
total, dsTag))
try:
self.FRED.layer.SetAttributeFilter("ID = '{}'".format(
item['sourceId']))
except:
pass
if self.FRED.layer is None or len(self.FRED.layer) == 0:
bbox = item['boundingBox']
geom = regions.Region().from_list([
bbox['minX'], bbox['maxX'], bbox['minY'],
bbox['maxY']
]).export_as_geom()
if item['format'] == 'IMG' or item[
'format'] == 'GeoTIFF':
tnm_ds = 'raster'
elif item['format'] == 'LAZ' or item['format'] == 'LAS':
tnm_ds = 'lidar'
else:
tnm_ds = 'tnm'
if geom is not None:
self.FRED._add_survey(
Name=item['title'],
ID=item['sourceId'],
Agency='USGS',
Date=item['publicationDate'],
MetadataLink=item['metaUrl'],
MetadataDate=item['dateCreated'],
DataLink=item['downloadURL'],
Link=item['sourceOriginId'],
Resolution=item['extent'],
DataType=tnm_ds,
DataSource='tnm',
HorizontalDatum=h_epsg,
VerticalDatum=v_epsg,
Etcetra=dsTag,
Info=item['moreInfo'],
geom=geom)
offset += 100
if total - offset <= 0: break
if self.verbose:
_prog.end(0,
'gathered {} products from {}'.format(total, dsTag))
