def update(self):
self.FRED._open_ds()
v_zip = os.path.basename(self._hrdem_footprints_url)
status = f_utils.Fetch(self._hrdem_footprints_url,
verbose=self.verbose).fetch_ftp_file(v_zip)
v_shps = utils.p_unzip(v_zip, ['shp', 'shx', 'dbf', 'prj'])
v_shp = None
for v in v_shps:
if '.shp' in v: v_shp = v
shp_regions = regions.gdal_ogr_regions(v_shp)
shp_region = regions.Region()
for this_region in shp_regions:
if shp_region.valid_p(check_xy=True):
shp_region = regions.regions_merge(shp_region, this_region)
else:
shp_region = this_region
geom = shp_region.export_as_geom()
self.FRED._attribute_filter(["ID = '{}'".format('HRDEM-1')])
if self.FRED.layer is None or len(self.FRED.layer) == 0:
self.FRED._add_survey(Name='High-Resolution DEM (Canada)',
ID='HRDEM-1',
Agency='NRCAN',
Date=utils.this_year(),
MetadataLink=self._hrdem_info_url,
MetadataDate=utils.this_year(),
DataLink=self._hrdem_footprints_url,
IndexLink=self._hrdem_footprints_url,
DataType='raster',
DataSource='hrdem',
Info='Canada Only',
geom=geom)
utils.remove_glob(v_zip, *v_shps)
self.FRED._close_ds()
def _update_all(self):
self.FRED._open_ds(1)
v_zip = os.path.basename(self._hrdem_footprints_url)
status = f_utils.Fetch(self._hrdem_footprints_url,
verbose=self.verbose).fetch_ftp_file(v_zip)
v_shps = utils.p_unzip(v_zip, ['shp', 'shx', 'dbf', 'prj'])
v_shp = None
for v in v_shps:
if '.shp' in v: v_shp = v
try:
v_ds = ogr.Open(v_shp)
except:
v_ds = None
status = -1
if v_ds is not None:
layer = v_ds.GetLayer()
fcount = layer.GetFeatureCount()
if self.verbose:
_prog = utils.CliProgress(
'scanning {} datasets...'.format(fcount))
for f in range(0, fcount):
feature = layer[f]
name = feature.GetField('Tile_name')
if self.verbose:
_prog.update_perc((f, fcount))
try:
self.FRED.layer.SetAttributeFilter(
"Name = '{}'".format(name))
except:
pass
if self.FRED.layer is None or len(self.FRED.layer) == 0:
data_link = feature.GetField('Ftp_dtm')
if data_link is not None:
geom = feature.GetGeometryRef()
self.FRED._add_survey(
Name=name,
ID=feature.GetField('Project'),
Agency='NRCAN',
Date=utils.this_year(),
MetadataLink=feature.GetField('Meta_dtm'),
MetadataDate=utils.this_year(),
DataLink=data_link.replace('http', 'ftp'),
IndexLink=self._hrdem_footprints_url,
DataType='raster',
DataSource='hrdem',
HorizontalDatum=feature.GetField(
'Coord_Sys').split(':')[-1],
Info=feature.GetField('Provider'),
geom=geom)
if self.verbose:
_prog.end('scanned {} datasets.'.format(fcount))
utils.remove_glob(v_zip, *v_shps)
self.FRED._close_ds()
def write_datalist(data_list, outname=None):
if outname is None:
outname = '{}_{}'.format(self.metadata['name'], utils.this_year())
if os.path.exists('{}.datalist'.format(outname)):
utils.remove_glob('{}.datalist*'.format(outname))
with open('{}.datalist'.format(outname), 'w') as tmp_dl:
[tmp_dl.write('{}\n'.format(x.format_entry())) for x in data_list]
return ('{}.datalist'.format(outname))
def update(self):
self.FRED._open_ds()
v_shp = None
v_zip = os.path.basename(self._hydrolakes_poly_zip)
status = f_utils.Fetch(self._hydrolakes_poly_zip,
verbose=self.verbose).fetch_file(v_zip)
v_shps = utils.p_unzip(v_zip, ['shp', 'shx', 'dbf', 'prj'])
for v in v_shps:
if '.shp' in v:
v_shp = v
shp_regions = regions.gdal_ogr_regions(v_shp)
shp_region = regions.Region()
for this_region in shp_regions:
if shp_region.valid_p(check_xy=True):
shp_region = regions.regions_merge(shp_region, this_region)
else:
shp_region = this_region
geom = shp_region.export_as_geom()
self.FRED._attribute_filter(["ID = '{}'".format('HYDROLAKES')])
if self.FRED.layer is None or len(self.FRED.layer) == 0:
self.FRED._add_survey(Name='HydorLakes',
ID='HYDROLAKES',
Agency='HydroSheds',
Date=utils.this_year(),
MetadataLink=self._hydrolakes_prods,
MetadataDate=utils.this_year(),
DataLink=self._hydrolakes_poly_zip,
IndexLink=self._hydrolakes_poly_zip,
DataType='vector',
DataSource='hydrolakes',
Info='World-wide lakes',
geom=geom)
utils.remove_glob(v_zip, *v_shps)
self.FRED._close_ds()
def _update_dataset(self, ds, fmt, geom, h_epsg, v_epsg):
self.FRED._attribute_filter(["ID = '{}'".format(ds['id'])])
if self.FRED.layer is None or len(self.FRED.layer) == 0:
if 'IMG' in fmt or 'TIFF' in fmt:
datatype = 'raster'
elif 'LAS' in fmt or 'LAZ' in fmt:
datatype = 'lidar'
else:
datatype = 'tnm'
url_enc = f_utils.urlencode({'datasets': ds['sbDatasetTag']})
try:
pubDate = ds['lastPublishedDate']
except:
pubDate = utils.this_year()
try:
metadataDate = ds['lastUpdatedDate']
except:
metadataDate = utils.this_year()
if geom is not None:
self.FRED._add_survey(Name=ds['sbDatasetTag'],
ID=ds['id'],
Agency='USGS',
Date=pubDate[-4:],
MetadataLink=ds['infoUrl'],
MetadataDate=metadataDate,
DataLink='{}{}'.format(
self._tnm_product_url, url_enc),
Link=ds['dataGovUrl'],
Resolution=','.join(ds['extents']),
DataType=datatype,
DataSource='tnm',
HorizontalDatum=h_epsg,
VerticalDatum=v_epsg,
Etcetra=fmt,
Info=ds['refreshCycle'],
geom=geom)
def yield_xyz(self, entry):
src_data = os.path.basename(entry[1])
src_mb = src_data[:-4]
try:
survey, src_data, mb_fmt, mb_perc, mb_date = self.parse_entry_inf(entry)
except TypeError:
return
this_inf = self.parse_entry_inf(entry)
if f_utils.Fetch(entry[0], callback=self.callback, verbose=self.verbose).fetch_file(src_data) == 0:
src_xyz = os.path.basename(src_data) + '.xyz'
if not self.process:
this_weight = self.weight
out, status = utils.run_cmd('mblist -OXYZ -I{} -M{} > {}'.format(src_data, 'X{}'.format(self.exclude) if self.exclude is not None else 'A', src_xyz), verbose=True)
else:
#this_weight = (float(mb_perc) * (1 + (2*((int(mb_date)-2015)/100))))/100.
this_year = int(utils.this_year()) if self.min_year is None else self.min_year
this_weight = float(mb_perc) * ((int(mb_date)-2000)/(this_year-2000))/100.
if this_weight <= 0.: this_weight = 0.0000001
#this_weight = (float(mb_perc) * ((int(mb_date)-2000)/int(this_year) - 2000))/100.
out, status = utils.run_cmd('mblist -OXYZ -I{} -MX{} > {}'.format(src_data, str(100-float(mb_perc)), src_xyz), verbose=True)
if status != 0:
if f_utils.Fetch('{}.inf'.format(entry[0]), callback=self.callback, verbose=self.verbose).fetch_file('{}.inf'.format(src_mb)) == 0:
mb_fmt = self.mb_inf_data_format('{}.inf'.format(src_mb))
mb_date = self.mb_inf_data_date('{}.inf'.format(src_mb))
out, status = utils.run_cmd('mblist -F{} -OXYZ -I{} -MX{} > {}'.format(mb_fmt, src_data, str(100-float(mb_perc)), src_xyz), verbose=True)
if status == 0:
_ds = datasets.XYZFile(
fn=src_xyz,
delim='\t',
data_format=168,
src_srs='epsg:4326',
dst_srs=self.dst_srs,
#name=os.path.basename(entry[1]),
src_region=self.region,
verbose=self.verbose,
weight=this_weight,
remote=True
)
if self.inc is not None:
xyz_func = lambda p: _ds.dump_xyz(dst_port=p, encode=True)
for xyz in utils.yield_cmd(
'gmt blockmedian -I{:.10f} {} -r -V'.format(
self.inc, self.region.format('gmt')
),
verbose=self.verbose,
data_fun=xyz_func
):
yield(xyzfun.XYZPoint().from_list([float(x) for x in xyz.split()]))
else:
for xyz in _ds.yield_xyz():
yield(xyz)
utils.remove_glob(src_data, '{}*'.format(src_xyz), '{}*.inf'.format(src_mb))
else:
utils.echo_error_msg('failed to process local file, {} [{}]...'.format(src_data, entry[0]))
with open(
'{}'.format(os.path.join(self._outdir, 'fetch_{}_{}.err'.format(self.name, self.region.format('fn')))),
'a'
) as mb_err:
mb_err.write('{}\n'.format(','.join([src_mb, entry[0]])))
os.rename(src_data, os.path.join(self._outdir, src_data))
utils.remove_glob(src_xyz)
else:
utils.echo_error_msg(
'failed to fetch remote file, {}...'.format(src_data)
)
def update(self):
self.FRED._open_ds()
v_zip = os.path.basename(self._arctic_dem_index_url)
status = f_utils.Fetch(self._arctic_dem_index_url, verbose=self.verbose).fetch_file(v_zip)
v_shps = utils.p_unzip(v_zip, ['shp', 'shx', 'dbf', 'prj'])
v_shp = None
for v in v_shps:
if '.shp' in v: v_shp = v
utils.run_cmd('ogr2ogr arctic_tmp.shp {} -t_srs epsg:4326'.format(v_shp), verbose=True)
utils.remove_glob(v_zip, *v_shps)
v_shp = 'arctic_tmp.shp'
v_shps = ['arctic_tmp.shp','arctic_tmp.dbf','arctic_tmp.shx','arctic_tmp.prj']
shp_regions = regions.gdal_ogr_regions(v_shp)
shp_region = regions.Region()
for this_region in shp_regions:
#this_region.src_srs = 'epsg:3413'
#this_region.warp('epsg:4326')
if shp_region.valid_p(check_xy=True):
shp_region = regions.regions_merge(shp_region, this_region)
else: shp_region = this_region
geom = shp_region.export_as_geom()
self.FRED._attribute_filter(["ID = '{}'".format('ARCTICDEM-1')])
if self.FRED.layer is None or len(self.FRED.layer) == 0:
self.FRED._add_survey(Name = 'ArcticDEM', ID = 'ARCTICDEM-1', Agency = 'UMN', Date = utils.this_year(),
MetadataLink = 'https://data.pgc.umn.edu/elev/dem/setsm/ArcticDEM/', MetadataDate = utils.this_year(),
DataLink = self._arctic_dem_index_url, IndexLink = self._arctic_dem_index_url,
DataType = 'raster', DataSource = 'arcticdem', Info = 'Arctic Only', geom = geom)
utils.remove_glob(*v_shps)
self.FRED._close_ds()