def load_metrics(self):
fn = os.path.join(
self.save_path, self._subdir,
get_fn_from_coords(self.coords, 'edge_metrics' + self.post_fn))
if os.path.exists(fn + '.npy'):
self.n_done, self.n_coulddo, self.percent_done = \
np.load(fn + '.npy')
else:
self.n_done, self.n_coulddo, self.percent_done = [None] * 3
def mk_test_multifile(testnum, NN, testdir, nx_grid=3, ny_grid=4, nx_overlap=16,
ny_overlap=32, lat=[46, 45], lon=[-73, -72]):
"""
Written to make test case for multi-file edge resolution.
"""
path = os.path.split(make_file_names(testnum, NN, os.path.join(
testdir, 'chunks'))['elev'])[0]
try:
os.makedirs(path)
except:
pass
def _get_chunk_edges(NN, chunk_size, chunk_overlap):
left_edge = np.arange(0, NN - chunk_overlap, chunk_size)
left_edge[1:] -= chunk_overlap
right_edge = np.arange(0, NN - chunk_overlap, chunk_size)
right_edge[:-1] = right_edge[1:] + chunk_overlap
right_edge[-1] = NN
right_edge = np.minimum(right_edge, NN)
return left_edge, right_edge
elev_data, ang_data, fel_data = get_test_data(testnum, NN)
try:
raster = fel_data.raster_data
except:
raster = elev_data.raster_data
ni, nj = raster.shape
top_edge, bottom_edge = _get_chunk_edges(ni, ni // ny_grid, ny_overlap)
left_edge, right_edge = _get_chunk_edges(nj, nj // nx_grid, nx_overlap)
# gc = elev_data.grid_coordinates
# lat = gc.y_axis
# lon = gc.x_axis
lat = np.linspace(lat[0], lat[1], ni)
lon = np.linspace(lon[0], lon[1], nj)
count = 0
for te, be in zip(top_edge, bottom_edge):
for le, re in zip(left_edge, right_edge):
count += 1
fn = os.path.join(path,
get_fn_from_coords((lat[be-1], lon[le], lat[te],
lon[re-1]), 'elev'))
print count, ": [%d:%d, %d:%d]" % (te, be, le, re), \
'(lat, lon) = (%g to %g, %g to %g)' % (lat[te], lat[be-1],
lon[le], lon[re-1]), \
'min,max = (%g to %g, %g to %g)' % (lat.min(), lat.max(),
lon.min(), lon.max())
mk_geotiff_obj(raster[te:be, le:re], fn,
bands=1, gdal_data_type=gdal.GDT_Float32,
lat=[lat[te], lat[be-1]], lon=[lon[le], lon[re-1]])
def mk_test_multifile(testnum, NN, testdir, nx_grid=3, ny_grid=4, nx_overlap=16,
ny_overlap=32, lat=[46, 45], lon=[-73, -72]):
"""
Written to make test case for multi-file edge resolution.
"""
path = os.path.split(make_file_names(testnum, NN, os.path.join(
testdir, 'chunks'))['elev'])[0]
try:
os.makedirs(path)
except:
pass
def _get_chunk_edges(NN, chunk_size, chunk_overlap):
left_edge = np.arange(0, NN - chunk_overlap, chunk_size)
left_edge[1:] -= chunk_overlap
right_edge = np.arange(0, NN - chunk_overlap, chunk_size)
right_edge[:-1] = right_edge[1:] + chunk_overlap
right_edge[-1] = NN
right_edge = np.minimum(right_edge, NN)
return left_edge, right_edge
elev_data, ang_data, fel_data = get_test_data(testnum, NN)
try:
raster = fel_data.raster_data
except:
raster = elev_data.raster_data
ni, nj = raster.shape
top_edge, bottom_edge = _get_chunk_edges(ni, ni // ny_grid, ny_overlap)
left_edge, right_edge = _get_chunk_edges(nj, nj // nx_grid, nx_overlap)
# gc = elev_data.grid_coordinates
# lat = gc.y_axis
# lon = gc.x_axis
lat = np.linspace(lat[0], lat[1], ni)
lon = np.linspace(lon[0], lon[1], nj)
count = 0
for te, be in zip(top_edge, bottom_edge):
for le, re in zip(left_edge, right_edge):
count += 1
fn = os.path.join(path,
get_fn_from_coords((lat[be-1], lon[le], lat[te],
lon[re-1]), 'elev'))
print count, ": [%d:%d, %d:%d]" % (te, be, le, re), \
'(lat, lon) = (%g to %g, %g to %g)' % (lat[te], lat[be-1],
lon[le], lon[re-1]), \
'min,max = (%g to %g, %g to %g)' % (lat.min(), lat.max(),
lon.min(), lon.max())
mk_geotiff_obj(raster[te:be, le:re], fn,
bands=1, gdal_data_type=gdal.GDT_Float32,
lat=[lat[te], lat[be-1]], lon=[lon[le], lon[re-1]])
def update_metrics(self):
todo = self.get('todo')
done = self.get('done')
data = self.get('data')
coulddo = self.coulddo(todo, data, done)
self.n_done = self.calc_n_done(coulddo, done)
self.n_coulddo = self.calc_n_coulddo(coulddo)
self.percent_done = self.calc_percent_done(coulddo, done)
fn = os.path.join(
self.save_path, self._subdir,
get_fn_from_coords(self.coords, 'edge_metrics' + self.post_fn))
np.save(fn, np.array([self.n_done, self.n_coulddo, self.percent_done]))
# clean up
del todo
del done
del data
del coulddo
def process_command(self, command, save_name='custom', index=None):
"""
Processes the hillshading
Parameters
-----------
index : int/slice (optional)
Default: None - process all tiles in source directory. Otherwise,
will only process the index/indices of the files as listed in
self.elev_source_files
"""
if index is not None:
elev_source_files = [self.elev_source_files[index]]
else:
elev_source_files = self.elev_source_files
save_root = os.path.join(self.save_path, save_name)
if not os.path.exists(save_root):
os.makedirs(save_root)
for i, esfile in enumerate(elev_source_files):
try:
status = 'Success' # optimism
# Check if file is locked
lckfn = _get_lockfile_name(esfile)
coords = parse_fn(esfile)
fn = get_fn_from_coords(coords, save_name)
fn = os.path.join(save_root, fn)
if os.path.exists(lckfn): # another process is working on it
print fn, 'is locked'
status = 'locked'
elif os.path.exists(fn):
print fn, 'already exists'
status = 'cached'
else: # lock this tile
print fn, '... calculating ', save_name
fid = file(lckfn, 'w')
fid.close()
# Calculate the custom process for this tile
status = command(esfile, fn)
os.remove(lckfn)
if index is None:
self.custom_status[i] = status
else:
self.custom_status[index] = status
except:
lckfn = _get_lockfile_name(esfile)
try:
os.remove(lckfn)
except:
pass
traceback.print_exc()
print traceback.format_exc()
if index is None:
self.custom_status[i] = "Error " + traceback.format_exc()
else:
self.custom_status[
index] = "Error " + traceback.format_exc()
def calculate_twi(self,
esfile,
save_path,
use_cache=True,
do_edges=False,
skip_uca_twi=False):
"""
Calculates twi for supplied elevation file
Parameters
-----------
esfile : str
Path to elevation file to be processed
save_path: str
Root path to location where TWI will be saved. TWI will be saved in
a subdirectory 'twi'.
use_cache : bool (optional)
Default True. If a temporary file exists (from a previous run),
the cached file will be used. Otherwise, if False, existing files
will be recomputed
do_edges : bool (optional)
See :py:func:`process_twi` for details on this argument.
skip_uca_twi : bool (optional)
Skips the calculation of the UCA and TWI (only calculates the
magnitude and direction)
"""
if os.path.exists(os.path.join(save_path, 'tile_edge.pkl')) and \
self.tile_edge is None:
with open(os.path.join(save_path, 'tile_edge.pkl'), 'r') as fid:
self.tile_edge = cPickle.load(fid)
elif self.tile_edge is None:
self.tile_edge = TileEdgeFile(self.elev_source_files, save_path)
with open(os.path.join(save_path, 'tile_edge.pkl'), 'wb') as fid:
cPickle.dump(self.tile_edge, fid)
status = 'Success' # optimism
# Check if file is locked
lckfn = _get_lockfile_name(esfile)
coords = parse_fn(esfile)
fn = get_fn_from_coords(coords, 'twi')
print '*' * 79
if skip_uca_twi:
print '*' * 10, fn, 'Slope Calculation starting...:', '*' * 10
else:
print '*' * 10, fn, 'TWI Calculation starting...:', '*' * 10
print '*' * 79
if os.path.exists(lckfn): # another process is working on it
print fn, 'is locked'
return fn, "Locked"
else: # lock this tile
fid = file(lckfn, 'w')
fid.close()
dem_proc = DEMProcessor(esfile)
# check if the slope already exists for the file. If yes, we should
# move on to the next tile without doing anything else
if skip_uca_twi \
and os.path.exists(dem_proc.get_full_fn('mag', save_path)
+ '.npz') \
and os.path.exists(dem_proc.get_full_fn('ang', save_path)
+ '.npz'):
print dem_proc.get_full_fn('mag',
save_path) + '.npz', 'already exists'
print dem_proc.get_full_fn('ang',
save_path) + '.npz', 'already exists'
# remove lock file
os.remove(lckfn)
return fn, 'Cached: Slope'
# check if the twi already exists for the file. If not in the edge
# resolution round, we should move on to the next tile
if os.path.exists(dem_proc.get_full_fn('twi', save_path)) \
and (do_edges is False):
print dem_proc.get_full_fn('twi', save_path), 'already exists'
# remove lock file
os.remove(lckfn)
return fn, 'Cached'
# only calculate the slopes and direction if they do not exist in cache
fn_ang = dem_proc.get_full_fn('ang', save_path)
fn_mag = dem_proc.get_full_fn('mag', save_path)
if os.path.exists(fn_ang + '.npz') and os.path.exists(fn_mag + '.npz')\
and not self.overwrite_cache:
dem_proc.load_direction(fn_ang)
dem_proc.load_slope(fn_mag)
dem_proc.find_flats()
else:
if os.path.exists(fn_ang + '.npz') and os.path_exists(fn_mag + '.npz')\
and self.overwrite_cache:
os.remove(fn_ang)
os.remove(fn_mag)
dem_proc.calc_slopes_directions()
dem_proc.save_slope(save_path, raw=True)
dem_proc.save_direction(save_path, raw=True)
if self._DEBUG:
dem_proc.save_slope(save_path, as_int=False)
dem_proc.save_direction(save_path, as_int=False)
if skip_uca_twi:
# remove lock file
os.remove(lckfn)
return fn, status + ":mag-dir-only"
fn_uca = dem_proc.get_full_fn('uca', save_path)
fn_uca_ec = dem_proc.get_full_fn('uca_edge_corrected', save_path)
fn_twi = dem_proc.get_full_fn('twi', save_path)
# check if edge structure exists for this tile and initialize
edge_init_data, edge_init_done, edge_init_todo = \
self.tile_edge.get_edge_init_data(esfile, save_path)
# Check if uca data exists (if yes, we are in the
# edge-resolution round)
uca_init = None
if os.path.exists(fn_uca + '.npz'):
if os.path.exists(fn_uca_ec + '.npz'):
dem_proc.load_uca(fn_uca_ec)
else:
dem_proc.load_uca(fn_uca)
uca_init = dem_proc.uca
if do_edges or uca_init is None:
dem_proc.calc_uca(uca_init=uca_init,
edge_init_data=[
edge_init_data, edge_init_done,
edge_init_todo
])
if uca_init is None:
dem_proc.save_uca(save_path, raw=True)
if self._DEBUG:
# Also save a geotiff for debugging
dem_proc.save_uca(save_path, as_int=False)
else:
if os.path.exists(fn_uca_ec):
os.remove(fn_uca_ec)
dem_proc.save_array(dem_proc.uca,
None,
'uca_edge_corrected',
save_path,
raw=True)
if self._DEBUG:
dem_proc.save_array(dem_proc.uca,
None,
'uca_edge_corrected',
save_path,
as_int=False)
# Saving Edge Data, and updating edges
self.tile_edge.update_edges(esfile, dem_proc)
dem_proc.calc_twi()
if os.path.exists(fn_twi):
os.remove(fn_twi)
dem_proc.save_twi(save_path, raw=False)
# clean up for in case
gc.collect()
# remove lock file
os.remove(lckfn)
# Save last-used dem_proc for debugging purposes
if self._DEBUG:
self.dem_proc = dem_proc
return fn, status
def get_fn(self, name):
fn = os.path.join(
self.save_path, self._subdir,
get_fn_from_coords(self.coords, 'edge_' + name + self.post_fn))
return fn + '.npy'
