class DinfFlowDirOp(Operation):
""" Compute Slope and Aspect from a DEM """
title = 'D-infinity Flow Direction'
name = 'dinf-flow-direction'
output_types = [OutputType('slope', 'tif'), OutputType('aspect', 'tif')]
def run(self, elevation_ds):
subprocess.call([
'dinfflowdir', '-fel', elevation_ds.loc.path, '-slp',
self.locs['slope'].path, '-ang', self.locs['aspect'].path
])
class DownloadOpenDASOp(Operation):
title = 'Download OpenDAS data using ncks'
name = 'download-opendas'
output_types = [OutputType('chunks', 'nc'), OutputType('data', 'nc')]
def run(self, ds, vars, bbox, start, end, chunk=None):
# Split large data sets into chunks
if chunk:
dates = pd.date_range(start, end, freq=chunk)
else:
dates = [start, end]
print(self.locs['chunks'].filename)
self.locs['chunks'] = DatetimeLoc(datetimes=dates,
template=self.locs['chunks'])
self.locs['chunks'].configure(self.cfg)
# Download
nco = Nco()
for i, (start_date, end_date) in enumerate(zip(dates[:-1], dates[1:])):
info = {'year': start_date.year}
logging.debug(ds.loc.url.format(**info))
logging.debug(self.locs['chunks'].locs[i].filename)
nco.ncks(
input=ds.loc.url.format(**info),
output=self.locs['chunks'].locs[i].path,
options=[
'--mk_rec_dmn time', '-v ' + ','.join(vars),
'-d ' + ','.join(
['time',
start_date.isoformat(),
end_date.isoformat()]),
'-d ' + ','.join(
['lon', str(bbox.min.lon),
str(bbox.max.lon)]),
'-d ' + ','.join(
['lat', str(bbox.min.lat),
str(bbox.max.lat)])
])
# Merge chunks
cat_args = [
'SKIP_SAME_TIME=1', 'cdo', 'mergetime',
os.path.join(self.locs['chunks'].dirname, '*'),
self.locs['data'].path
]
logging.info('Calling process %s', ' '.join(cat_args))
cat_process = subprocess.Popen(cat_args,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
cat_output, _ = cat_process.communicate()
class DownloadThreddsOp(Operation):
title = 'Download Thredds data'
name = 'download-thredds'
output_types = [OutputType('chunks', 'nc'), OutputType('data', 'nc')]
def run(self, ds, vars, bbox, start, end, chunk=None):
# Split large data sets into chunks
if chunk:
dates = pd.date_range(start, end, freq=chunk)
else:
dates = [start, end]
self.locs['chunks'] = DatetimeLoc(datetimes=dates,
template=self.locs['chunks'])
self.locs['chunks'].configure(self.cfg)
# Download
for i, (start_date, end_date) in enumerate(zip(dates[:-1], dates[1:])):
info = {'year': start_date.year}
params = {
'var': vars,
'north': bbox.urc.lat,
'west': bbox.llc.lon,
'east': bbox.urc.lon,
'south': bbox.llc.lat,
'horizStride': 1,
'time_start': start_date,
'time_end': end_date,
'timeStride': 1,
'accept': 'netcdf'
}
r = requests.get(ds.loc.url.format(**info),
params=params,
stream=True)
with open(self.locs['chunks'].locs[i].path, 'wb') as file:
for chunk in r.iter_content(chunk_size=128):
file.write(chunk)
# Merge chunks
cat_args = [
'SKIP_SAME_TIME=1', 'cdo', 'mergetime',
os.path.join(self.locs['chunks'].dirname, '*'),
self.locs['data'].path
]
logging.info('Calling process %s', ' '.join(cat_args))
cat_process = subprocess.Popen(cat_args,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
cat_output, _ = cat_process.communicate()
class FlowDirectionOp(Operation):
title = 'Flow Direction'
name = 'flow-direction'
output_types = [
OutputType('flow-direction', 'tif'),
OutputType('slope', 'tif')
]
def run(self, dem_ds):
subprocess.call([
'd8flowdir', '-fel', dem_ds.loc.path, '-p',
self.locs['flow-direction'].path, '-sd8', self.locs['slope'].path
])
class PeukerDouglasStreamDefinitionOp(Operation):
""" Run the TauDEM Peuker Douglas Stream Definition Command """
title = 'Peuker Douglas Stream Definition'
name = 'stream-def-pd'
output_types = [
OutputType('ssa', 'tif'),
OutputType('drop-analysis', 'txt'),
OutputType('stream-definition', 'tif')
]
def run(self, no_sinks_ds, flow_dir_ds, source_area_ds, outlet_ds):
# The threshold range should be selected based on the raster size
# Something like 10th to 99th percentile of flow accumulation?
## This is a three-step process - first compute the D8 source area
subprocess.call([
'aread8', '-p', flow_dir_ds.loc.path, '-o', outlet_ds.loc.path,
'-ad8', self.locs['ssa'].path
])
## Next perform the drop analysis
# This selects a sensible flow accumulation threshold value
thresh_min = np.percentile(source_area_ds.array, 95)
thresh_max = np.max(source_area_ds.array)
subprocess.call([
'dropanalysis', '-p', flow_dir_ds.loc.path, '-fel',
no_sinks_ds.loc.path, '-ad8', source_area_ds.loc.path, '-o',
outlet_ds.loc.path, '-ssa', self.locs['ssa'].path, '-drp',
self.locs['drop-analysis'].path, '-par',
str(thresh_min),
str(thresh_max), '40', '0'
])
## Finally define the stream
# Extract the threshold from the first row with drop statistic t < 2
with open(self.locs['drop-analysis'].path, 'r') as drop_file:
# Get optimum threshold value from last line
for line in drop_file:
pass
last = line
thresh_re = re.compile(r'([\.\d]*)$')
threshold = thresh_re.search(last).group(1)
subprocess.call([
'threshold', '-ssa', self.locs['ssa'].path, '-thresh', threshold,
'-src', self.locs['stream-definition'].path
])
class LabelGagesOp(Operation):
""" Add a sequential id field to each outlet in a shapefile """
title = 'Labeled Gages'
name = 'label-outlet'
output_types = [OutputType('labelled-outlet', 'shp')]
def run(self, outlet_ds):
## Fix this - modify dataset at the new location, not the old
outlet_ds.chmod(True)
outlet_path = outlet_ds.loc
layer = outlet_ds.dataset.GetLayer()
id_field = ogr.FieldDefn('id', ogr.OFTInteger)
layer.CreateField(id_field)
feature = layer.GetNextFeature()
gage_id = 1
while feature:
feature.SetField("id", gage_id)
layer.SetFeature(feature)
feature = layer.GetNextFeature()
gage_id += 1
# Clean up and copy to correct location
outlet_ds.dataset.Destroy()
for a_file in glob.glob(r'{}.*'.format(outlet_path.no_ext)):
shutil.copyfile(
a_file, self.locs['labelled-outlet'].ext(
os.path.splitext(a_file)[1][1:]))
class ReprojectVectorOp(Operation):
title = 'Reproject Vector Dataset'
name = 'reproject-vector'
output_types = [OutputType('reprojected', 'shp')]
def run(self, input_ds, srs):
reproj_ds = BoundaryDataset(self.locs['reprojected'],
update=True).new()
out_srs = osr.SpatialReference()
if not srs.startswith('EPSG:'):
raise ValueError('SRS definition must start with "EPSG:"')
out_srs.ImportFromEPSG(int(srs[5:]))
for layer in input_ds.layers:
# SRS transform
in_srs = layer.GetSpatialRef()
transform = osr.CoordinateTransformation(in_srs, out_srs)
# create the output layer
reproj_layer = reproj_ds.dataset.CreateLayer(
layer.GetName(), srs=out_srs, geom_type=layer.GetGeomType())
# add fields
layer_defn = layer.GetLayerDefn()
for i in range(0, layer_defn.GetFieldCount()):
field_defn = layer_defn.GetFieldDefn(i)
reproj_layer.CreateField(field_defn)
# loop through the input features
reproj_layer_defn = reproj_layer.GetLayerDefn()
feature = layer.GetNextFeature()
while feature:
# reproject the input geometry
geom = feature.GetGeometryRef()
geom.Transform(transform)
# create a new feature with same geometry and attributes
reproj_feature = ogr.Feature(reproj_layer_defn)
reproj_feature.SetGeometry(geom)
for j in range(0, reproj_layer_defn.GetFieldCount()):
reproj_feature.SetField(
reproj_layer_defn.GetFieldDefn(j).GetNameRef(),
feature.GetField(j))
# add the feature to the shapefile
reproj_layer.CreateFeature(reproj_feature)
# dereference the features and get the next input feature
reproj_feature = None
feature = layer.GetNextFeature()
# Save and close the shapefiles
input_ds.close()
reproj_ds.close()
class StreamNetworkOp(Operation):
""" Run the TauDEM Stream Reach and Watershed Command """
title = 'Stream Network'
name = 'stream-network'
output_types = [
OutputType('order', 'tif'),
OutputType('tree', 'tsv'),
OutputType('coord', 'tsv'),
OutputType('network', 'shp'),
OutputType('watershed', 'tif')
]
tree_colnames = [
'link_no', 'start_coord', 'end_coord', 'next_link', 'previous_link',
'order', 'monitoring_point_id', 'network_magnitude'
]
coord_colnames = [
'x', 'y', 'distance_to_terminus', 'elevation', 'contributing_area'
]
def run(self, no_sinks_ds, flow_dir_ds, source_area_ds, pd_stream_def_ds,
outlet_ds):
subprocess.call([
'streamnet', '-p', flow_dir_ds.loc.path, '-fel',
no_sinks_ds.loc.path, '-ad8', source_area_ds.loc.path, '-src',
pd_stream_def_ds.loc.path, '-o', outlet_ds.loc.path, '-ord',
self.locs['order'].path, '-tree', self.locs['tree'].path, '-coord',
self.locs['coord'].path, '-net', self.locs['network'].path, '-w',
self.locs['watershed'].path
])
self.locs['tree'].csvargs.update({
'delimiter': '\t',
'header': None,
'names': self.tree_colnames
})
self.locs['coord'].csvargs.update({
'delimiter': '\t',
'header': None,
'names': self.coord_colnames
})
class SoilType(Operation):
""" Soil texture category from percent clay/sand/slit """
title = "Soil Type"
name = 'soil-type'
output_types = [OutputType('type', 'gtif')]
textures = {
'Sand': 1,
'Loamy sand': 2,
'Sandy loam': 3,
'Silty loam': 4,
'Silt': 5,
'Loam': 6,
'Sandy clay loam': 7,
'Silty clay loam': 8,
'Clay loam': 9,
'Sandy clay': 10,
'Silty clay': 11,
'Clay': 12,
'Loamy sand': 13,
}
def run(self, texture_ds):
types = {ds.meta['type']: ds for ds in texture_ds}
clay = types['clay'].array
sand = types['sand'].array
silt = types['silt'].array
t = 0 * np.ones(clay.shape)
t = np.where((clay > 40) & (silt > 40), self.textures['Silty clay'], t)
t = np.where((t == 0) & (clay > 40) & (sand < 56),
self.textures['Clay'], t)
t = np.where((t == 0) & (clay > 28) & (sand < 20),
self.textures['Silty clay loam'], t)
t = np.where((t == 0) & (clay > 28) & (sand < 44),
self.textures['Silty clay loam'], t)
t = np.where((t == 0) & (clay > 36), self.textures['Sandy clay'], t)
t = np.where((t == 0) & (clay > 20) & (silt < 28),
self.textures['Sandy clay'], t)
t = np.where((t == 0) & (clay < 12) & (silt > 80),
self.textures['Silt'], t)
t = np.where((t == 0) & (silt > 50), self.textures['Silty loam'], t)
t = np.where((t == 0) & (clay > 8) & (sand < 52),
self.textures['Loam'], t)
t = np.where((t == 0) & (sand - clay < 70),
self.textures['Sandy loam'], t)
t = np.where((t == 0) & (sand - clay / 4. < 87.5),
self.textures['Loamy sand'], t)
t = np.where(t == 0, self.textures['Sand'], t)
texture_ds = GDALDataset(self.locs['type'], template=types['clay'])
texture_ds.array = t
class DownloadPolarisOp(Operation):
""" Download POLARIS data """
title = 'Download POLARIS data'
name = 'download-polaris'
output_types = [OutputType('polaris', 'tif')]
def run(self, input_ds):
if not input_ds.loc.exists:
req = requests.get(input_ds.loc.url, stream=True)
with open(input_ds.loc.path, 'wb') as file:
shutil.copyfileobj(req.raw, file)
class StreamDefinitionByThresholdOp(Operation):
""" Run the TauDEM Stream Definition By Threshold Command """
title = 'Stream Definition By Threshold'
name = 'stream-definition-threshold'
output_types = [OutputType('stream-raster', 'tif')]
def run(self, source_area_ds):
threshold = np.percentile(source_area_ds.array, 98)
subprocess.call([
'threshold', '-ssa', source_area_ds.loc.path, '-thresh',
'{:.1f}'.format(threshold), '-src', self.locs['stream-raster'].path
])
class SourceAreaOp(Operation):
title = 'Source Area/Flow Accumulation'
name = 'source-area'
output_types = [OutputType('source-area', 'tif')]
def run(self, flow_dir_ds):
subprocess.call([
'aread8', '-p', flow_dir_ds.loc.path, '-ad8',
self.locs['source-area'].path, '-nc'
])
source_area_ds = GDALDataset(self.locs['source-area'])
source_area_ds.nodata = 0
class ClipOp(Operation):
title = 'Raster clipped to boundary'
name = 'clip'
output_types = [OutputType('clipped_raster', 'tif')]
def run(self, input_ds, boundary_ds, algorithm='bilinear'):
clip_warp_options = gdal.WarpOptions(format='GTiff',
cutlineDSName=boundary_ds.loc.shp,
cutlineBlend=.5,
dstNodata=input_ds.nodata,
resampleAlg=algorithm)
gdal.Warp(path, input_ds.dataset, options=clip_warp_options)
class RemoveSinksOp(Operation):
title = 'Sinks Removed'
name = 'remove-sinks'
output_types = [OutputType('no-sinks', 'tif')]
def run(self, input_ds):
# Remove Pits / Fill sinks
proc = subprocess.call([
'pitremove', '-z', input_ds.loc.path, '-fel',
self.locs['no-sinks'].path
],
stdout=sys.stdout)
class MoveOutletsToStreamOp(Operation):
""" Run the TauDEM Move Outlets to Streams Command """
title = 'Move Outlets to Streams'
name = 'snap-outlet'
output_types = [
OutputType('outlet-on-stream-nosrs', 'shp'),
OutputType('outlet-on-stream', 'shp')
]
def run(self, flow_dir_ds, stream_ds, outlet_ds):
subprocess.call([
'moveoutletstostrm', '-p', flow_dir_ds.loc.path, '-src',
stream_ds.loc.path, '-o', outlet_ds.loc.path, '-om',
self.locs['outlet-on-stream-nosrs'].path, '-omlyr', 'outlet'
])
# Copy spatial reference from original outlet
in_ds = BoundaryDataset(self.locs['outlet-on-stream-nosrs'])
out_ds = BoundaryDataset(self.locs['outlet-on-stream'],
update=True).new()
# Create layer with outlet srs
in_lyr = in_ds.dataset.GetLayer()
outlet_lyr = outlet_ds.dataset.GetLayer()
srs = outlet_lyr.GetSpatialRef()
geom_type = in_lyr.GetLayerDefn().GetGeomType()
out_lyr = out_ds.dataset.CreateLayer('outlet', srs, geom_type)
# Copy outlet in correct srs
outlet = in_lyr.GetFeature(0)
out_feature = ogr.Feature(out_lyr.GetLayerDefn())
out_feature.SetGeometry(outlet.GetGeometryRef().Clone())
out_lyr.CreateFeature(out_feature)
# Clean up
out_ds.dataset.Destroy()
in_ds.dataset.Destroy()
class SkyviewOp(Operation):
""" Compute skyview files from a DEM """
title = 'Skyview'
name = 'skyview'
output_types = [OutputType('skyview', 'nc')]
def run(self,
elevation_ds,
elevation_epsg,
time_zone,
dt=1,
n_directions=8,
year=2000):
standard_meridian_srs = osr.SpatialReference()
standard_meridian_srs.ImportFromEPSG(4326)
elevation_srs = osr.SpatialReference()
elevation_srs.ImportFromEPSG(int(elevation_epsg[5:]))
dem_to_latlon = osr.CoordinateTransformation(elevation_srs,
standard_meridian_srs)
center = CoordProperty(*dem_to_latlon.TransformPoint(
elevation_ds.center.lon, elevation_ds.center.lat)[:-1])
# Skyview
n_rows = str(elevation_ds.size.y)
n_cols = str(elevation_ds.size.x)
cell_size = str(int(elevation_ds.res.x))
longitude = str(center.lon)
latitude = str(center.lat)
standard_meridian_lon = str(time_zone * 15.0)
x_origin = str(elevation_ds.ul.x)
y_origin = str(elevation_ds.ul.y)
n_directions = str(n_directions)
year = str(year)
day = '15'
steps_per_day = str(int(24 / dt))
dt = str(dt)
skyview_args = [
self.scripts['skyview'], elevation_ds.loc.path,
self.locs['skyview'].path, n_directions, n_rows, n_cols, cell_size,
x_origin, y_origin
]
logging.info('Calling process %s', ' '.join(skyview_args))
skyview_process = subprocess.Popen(skyview_args,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
skyview_output, _ = skyview_process.communicate()
logging.info(skyview_output)
class ConvertOp(Operation):
output_types = [OutputType('flow-direction', 'gtif')]
def run(self, flow_dir_ds):
converted_ds = GDALDataset(self.locs['flow-direction'],
template=flow_dir_ds)
convert_array = np.vectorize(self.convert)
input_array = np.ma.masked_where(
flow_dir_ds.array == flow_dir_ds.nodata, flow_dir_ds.array)
input_array.set_fill_value(flow_dir_ds.nodata)
converted = convert_array(input_array)
converted_ds.array = converted.filled()
class MatchRasterOp(Operation):
title = 'Warp dataset to match a template raster'
name = 'match-raster'
output_types = [OutputType('matched', 'tif')]
def run(self, input_ds, template_ds=None, algorithm='bilinear'):
agg_warp_options = gdal.WarpOptions(
outputBounds=template_ds.rev_warp_output_bounds,
width=template_ds.size.x,
height=template_ds.size.y,
resampleAlg=algorithm,
)
gdal.Warp(self.locs['matched'].path,
input_ds.dataset,
options=agg_warp_options)
class GageWatershedOp(Operation):
"""
Run the TauDEM Gage Watershed Command
Raster labeling each point by which gage it drains to directly
"""
title = 'Gage Watershed'
name = 'gage-watershed'
output_types = [OutputType('gage-watershed', 'tif')]
def run(self, flow_dir_ds, outlet_ds):
subprocess.call([
'gagewatershed', '-p', flow_dir_ds.loc.path, '-o',
outlet_ds.loc.path, '-gw', self.locs['gage-watershed'].path
])
class SoilDepthOp(Operation):
""" Compute soil depth from slope, elevation, and source area"""
title = 'Soil Depth'
name = 'soil-depth'
output_types = [OutputType('soil-depth', 'gtif')]
def run(self,
slope_ds,
source_ds,
elev_ds,
min_depth,
max_depth,
wt_slope=0.7,
wt_source=0.0,
wt_elev=0.3,
max_slope=30.0,
max_source=100000.0,
max_elev=1500.0,
pow_slope=0.25,
pow_source=1.0,
pow_elev=0.75):
wt_total = float(wt_slope + wt_source + wt_elev)
if not wt_total == 1.0:
logging.warning(
'Soil depth weights do not add up to 1.0 - scaling')
wt_slope = wt_slope / wt_total
wt_source = wt_source / wt_total
wt_elev = wt_elev / wt_total
# Scale sources: [min, max] -> [min/max, 1]
slope_arr = np.clip(slope_ds.array / max_slope, None, 1)
source_arr = np.clip(source_ds.array / max_source, None, 1)
elev_arr = np.clip(elev_ds.array / max_elev, None, 1)
# Calculate soil depth
soil_depth_arr = min_depth + \
(max_depth - min_depth) * (
wt_slope * (1.0 - np.power(slope_arr, pow_slope)) +
wt_source * np.power(source_arr, pow_source) +
wt_elev * (1.0 - np.power(elev_arr, pow_elev))
)
# Save in a dataset matching the DEM
soil_depth_ds = GDALDataset(self.locs['soil-depth'], template=elev_ds)
soil_depth_ds.array = soil_depth_arr
class ConvertFileType(Operation):
title = 'Convert Filetype'
name = 'convert-filetype'
output_types = [OutputType('converted', '')]
def run(self, input_ds, filetype, compress=None):
self.locs['converted'].default_ext = filetype
coptions = []
if compress:
coptions.append('COMPRESS=' + compress)
convert_translate_options = gdal.TranslateOptions(
format=input_ds.filetypes[filetype], creationOptions=coptions)
gdal.Translate(self.locs['converted'].path,
input_ds.dataset,
options=convert_translate_options,
format=input_ds.filetypes[filetype])
class CropOp(Operation):
title = 'Crop Raster Dataset'
name = 'crop'
output_types = [OutputType('cropped', 'tif')]
def run(self,
input_ds,
template_ds=None,
bbox=None,
padding=CoordProperty(x=0, y=0),
algorithm='bilinear',
ignore_srs=False):
if template_ds:
bbox = copy.copy(template_ds.bbox)
# SRS needs to match for output bounds and input dataset
if not ignore_srs:
if not template_ds.srs == input_ds.srs:
transform = osr.CoordinateTransformation(
template_ds.srs, input_ds.srs)
logging.debug('Bounding Box: %s', bbox)
logging.debug('Coordinate transformation: %s', transform)
logging.debug(template_ds.srs)
logging.debug(input_ds.srs)
bbox = BBox(llc=CoordProperty(*transform.TransformPoint(
bbox.llc.x, bbox.llc.y)[:-1]),
urc=CoordProperty(*transform.TransformPoint(
bbox.urc.x, bbox.urc.y)[:-1]))
grid_box = [
bbox.llc.x - padding.x, bbox.llc.y - padding.y,
bbox.urc.x + padding.x, bbox.urc.y + padding.y
]
agg_warp_options = gdal.WarpOptions(
outputBounds=grid_box,
resampleAlg=algorithm,
)
gdal.UseExceptions()
out = gdal.Warp(self.locs['cropped'].path,
input_ds.dataset,
options=agg_warp_options)
logging.debug('GDAL output: %s', out)
class MosaicOp(Operation):
title = 'Mosaic Rasters with gdal_merge'
name = 'mosaic'
output_types = [OutputType('merged', 'tif')]
def run(self, input_ds):
if not hasattr(input_ds, '__iter__'):
input_ds = [input_ds]
merge_args = [
'gdal_merge.py', '-o', self.locs['merged'].path,
*[ds.loc.path for ds in input_ds]
]
logging.info('Calling process %s', ' '.join(merge_args))
merge_process = subprocess.Popen(merge_args,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
merge_output, _ = merge_process.communicate()
logging.info(merge_output)
class AverageLayers(Operation):
""" Weighted average of layers by depth """
title = "Average Layers"
name = 'average-layers'
output_types = [OutputType('average', 'gtif')]
def run(self, layered_ds):
diff = []
for layer in layered_ds:
diff.append(layer.meta['layer_max'] - layer.meta['layer_min'])
weights = [d / sum(diff) for d in diff]
average = 0 * np.ones(layered_ds[0].array.shape)
for layer, weight in zip(layered_ds, weights):
average += layer.array * weight
average_ds = GDALDataset(self.locs['average'], template=layered_ds[0])
average_ds.array = average
class MergeOp(Operation):
title = 'Merge rasters into a virtual raster'
name = 'merge'
output_types = [OutputType('merged', 'vrt')]
def run(self, input_ds):
if not hasattr(input_ds, '__iter__'):
input_ds = [input_ds]
vrt_args = [
'gdalbuildvrt', self.locs['merged'].path,
*[ds.loc.path for ds in input_ds]
]
logging.info('Calling process %s', ' '.join(vrt_args))
vrt_process = subprocess.Popen(vrt_args,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
vrt_output, _ = vrt_process.communicate()
logging.info(vrt_output)
class FlowDistanceOp(Operation):
output_types = [OutputType('flow-distance', 'gtif')]
def dir2dist(self, lat, lon, direction, res, nodata):
distance = direction.astype(np.float64)
np.copyto(distance,
self.distance(lon - res.lon / 2, lat, lon + res.lon / 2,
lat),
where=np.isin(direction, [1, 5]))
np.copyto(distance,
self.distance(lon, lat - res.lat / 2, lon,
lat + res.lat / 2),
where=np.isin(direction, [3, 7]))
np.copyto(distance,
self.distance(lon - res.lon / 2, lat - res.lat / 2,
lon + res.lon / 2, lat + res.lat / 2),
where=np.isin(direction, [2, 4, 6, 8]))
return distance
def run(self, flow_dir_ds):
direction = flow_dir_ds.array
lon, lat = np.meshgrid(flow_dir_ds.cgrid.lon, flow_dir_ds.cgrid.lat)
res = flow_dir_ds.resolution
direction = np.ma.masked_where(direction == flow_dir_ds.nodata,
direction)
lat = np.ma.masked_where(direction == flow_dir_ds.nodata, lat)
lon = np.ma.masked_where(direction == flow_dir_ds.nodata, lon)
distance = self.dir2dist(lat, lon, direction, res, flow_dir_ds.nodata)
flow_dist_ds = GDALDataset(self.locs['flow-distance'],
template=flow_dir_ds)
# Fix masking side-effects - not sure why this needs to be done
flow_dist_ds.nodata = -9999
distance = distance.filled(flow_dist_ds.nodata)
distance[distance == None] = flow_dist_ds.nodata
distance = distance.astype(np.float32)
flow_dist_ds.array = distance
class ReprojectRasterOp(Operation):
title = 'Reproject raster'
name = 'reproject-raster'
output_types = [OutputType('reprojected', '')]
def run(self,
input_ds,
template_ds=None,
srs=None,
algorithm='bilinear',
extent=None,
format='tif'):
if template_ds:
srs = template_ds.srs
self.locs['reprojected'].default_ext = format
agg_warp_options = gdal.WarpOptions(dstSRS=srs,
resampleAlg=algorithm,
outputBounds=extent)
gdal.Warp(self.locs['reprojected'].path,
input_ds.dataset,
options=agg_warp_options)
class GridAlignOp(Operation):
title = 'Align Dataset'
name = 'grid-align'
output_types = [OutputType('aligned', 'ti')]
def run(self,
input_ds,
template_ds=None,
bbox=None,
resolution=CoordProperty(x=1 / 240., y=1 / 240.),
grid_res=None,
padding=CoordProperty(x=0, y=0),
algorithm='bilinear'):
if not grid_res:
grid_res = resolution
if template_ds:
resolution = template_ds.resolution
bbox = copy.copy(template_ds.bbox)
if bbox:
grid_box = [
bbox.llc.x - padding.x, bbox.llc.y - padding.y,
bbox.urc.x + padding.x, bbox.urc.y + padding.y
]
else:
grid_box = input_ds.gridcorners(grid_res,
padding=padding).warp_output_bounds
agg_warp_options = gdal.WarpOptions(
xRes=resolution.x,
yRes=resolution.y,
outputBounds=grid_box,
targetAlignedPixels=True,
resampleAlg=algorithm,
)
gdal.Warp(self.locs['aligned'].path,
input_ds.dataset,
options=agg_warp_options)
class NLCDtoDHSVM(Operation):
"""
Convert vegetation classes from the National Land Cover
Database to corresponding DHSVM values
"""
title = 'Remap NLCD to DHSVM'
name = 'nlcd-to-dhsvm'
output_types = [OutputType('veg-type', 'gtif')]
remap_table = {
3: 12,
11: 14,
12: 20,
21: 10,
22: 13,
23: 13,
24: 13,
31: 12,
41: 4,
42: 1,
43: 5,
51: 9,
52: 9,
71: 10,
72: 10,
81: 10,
82: 11,
90: 17,
95: 9,
}
def run(self, nlcd_ds, nodata=-99):
veg_type_ds = GDALDataset(self.locs['veg-type'], template=nlcd_ds)
array = copy.copy(nlcd_ds.array)
for nlcd, dhsvm in self.remap_table.items():
array[nlcd_ds.array == nlcd] = dhsvm
veg_type_ds.array = array
class CropDataFrameOp(Operation):
title = 'Merged Raster'
name = 'merge'
output_types = [OutputType('merged', 'tif')]
def run(self, input_ds):
out_file = 'processed/glc_swe_alldates.csv'
if not os.path.exists(os.path.dirname(out_file)):
raise FileNotFoundError('Directory does not exist: {}'.format(out_file))
slide = pd.read_csv('../data/SLIDE_NASA_GLC/GLC20180821.csv',
index_col='OBJECTID')
origen_x = -125
origen_y = 32
res = 0.05
# Filter Landslides to study area and duration
slide = slide[slide.latitude > 32]
slide = slide[slide.latitude < 43]
slide = slide[slide.longitude > -125]
slide = slide[slide.longitude < -114]
slide['event_date'] = pd.to_datetime(
slide['event_date'], format='%Y/%m/%d %H:%M')
slide['event_date'] = slide['event_date'].dt.normalize()
slide = slide[slide.event_date >= '2004-01-01']
slide = slide[slide.event_date < '2016-01-01']
# Open swe files
files = glob.glob('data/daymet/daymet_v3_swe_*_na.nc4')
files.sort()
arrays = [xr.open_dataset(file) for file in files]
coords = arrays[0].isel(time=0).drop(['time'])
# Filter coordinates
coords = coords.where(
(coords.lat < 43) & (coords.lat > 32) &
(coords.lon > -125) & (coords.lon < -114), drop=True)
# Build KD-Tree
locs = list(zip(coords.lon.values.flatten(), coords.lat.values.flatten()))
kdt = cKDTree(locs)
xa, ya = np.meshgrid(coords.x.values, coords.y.values)
xv = xa.flatten()
yv = ya.flatten()
# Add swe to dataframe
event_dfs = []
count = 0
for index, row in slide.iterrows():
print(count)
count += 1
# Pull out swe values for location
loc_i = kdt.query((row.longitude, row.latitude))[1]
event_swe = pd.concat([
arr.sel(x=xv[loc_i], y=yv[loc_i])[['swe']].to_dataframe()
for arr in arrays])
# Add location identifier to the index
event_swe['OBJECTID'] = index
event_swe = event_swe.set_index(['OBJECTID'], append=True)
event_dfs.append(event_swe)
swe_df = pd.concat(event_dfs)
swe_df.to_csv(out_file)
print(swe_df.sort_index().head())
