def _add_gage_ids_natur_flow_to_network(self):
"""
This adds gage and natural flow information
to the network from the file
"""
print("Adding Gage Station and Natur Flow info from: {0}".format(
self.gage_ids_natur_flow_file))
gage_id_natur_flow_table = csv_to_list(self.gage_ids_natur_flow_file)
for stream_info in gage_id_natur_flow_table[1:]:
if stream_info[0] != "":
stream_index = self._find_stream_segment_index(
int(float(stream_info[0])))
if stream_index != None:
#add natural flow
self.stream_segments[stream_index].natural_flow = int(
float(stream_info[1]))
#add station id
try:
station_id = str(int(float(stream_info[2])))
except Exception:
continue
pass
if station_id != "":
self.stream_undex_with_usgs_station.append(
stream_index)
self.stream_segments[
stream_index].station = USGSStreamGage(station_id)
def append_streamflow_from_stream_shapefile(self, stream_id_field, streamflow_field):
"""
Appends streamflow from values in shapefile
"""
stream_shapefile = ogr.Open(self.stream_shapefile_path)
stream_shp_layer = stream_shapefile.GetLayer()
self.spatially_filter_streamfile_layer_by_elevation_dem(stream_shp_layer)
print "Writing output to file ..."
stream_info_table = csv_to_list(self.stream_info_file, ", ")[1:]
#Columns: DEM_1D_Index Row Col StreamID StreamDirection
stream_id_list = np.array([row[3] for row in stream_info_table], dtype=np.int32)
temp_stream_info_file = "{0}_temp.txt".format(os.path.splitext(self.stream_info_file)[0])
with open(temp_stream_info_file, 'wb') as outfile:
writer = csv.writer(outfile, delimiter=" ")
writer.writerow(["DEM_1D_Index", "Row", "Col", "StreamID", "StreamDirection", "Slope", "Flow"])
for feature in stream_shp_layer:
#find all raster indices associates with the comid
raster_index_list = np.where(stream_id_list==int(float(feature.GetField(stream_id_field))))[0]
#add streamflow associated with comid
streamflow = feature.GetField(streamflow_field)
for raster_index in raster_index_list:
writer.writerow(stream_info_table[raster_index][:6] + [streamflow])
os.remove(self.stream_info_file)
os.rename(temp_stream_info_file, self.stream_info_file)
def append_streamflow_from_return_period_file(self, return_period_file,
return_period):
"""
Generates return period raster from return period file
"""
print "Extracting Return Period Data ..."
return_period_nc = Dataset(return_period_file, mode="r")
if return_period == "return_period_20":
return_period_data = return_period_nc.variables['return_period_20'][:]
elif return_period == "return_period_10":
return_period_data = return_period_nc.variables['return_period_10'][:]
elif return_period == "return_period_2":
return_period_data = return_period_nc.variables['return_period_2'][:]
elif return_period == "max_flow":
return_period_data = return_period_nc.variables['return_period_2'][:]
else:
raise Exception("Invalid return period definition.")
rivid_var = 'COMID'
if 'rivid' in return_period_nc.variables:
rivid_var = 'rivid'
return_period_comids = return_period_nc.variables[rivid_var][:]
return_period_nc.close()
#get where streamids are in the lookup grid id table
stream_info_table = csv_to_list(self.stream_info_file, ", ")[1:]
streamid_list_full = np.array([row[3] for row in stream_info_table], dtype=np.int32)
streamid_list_unique = np.unique(streamid_list_full)
print "Analyzing data and appending to list ..."
temp_stream_info_file = "{0}_temp.txt".format(os.path.splitext(self.stream_info_file)[0])
with open(temp_stream_info_file, 'wb') as outfile:
writer = csv.writer(outfile, delimiter=" ")
writer.writerow(["DEM_1D_Index", "Row", "Col", "StreamID", "StreamDirection", "Slope", "Flow"])
for streamid in streamid_list_unique:
try:
#get where streamids are in netcdf file
streamid_index = np.where(return_period_comids==streamid)[0][0]
peak_flow = return_period_data[streamid_index]
except IndexError:
print "ReachID", streamid, "not found in netCDF dataset. Setting value to zero ..."
peak_flow = 0
pass
#get where streamids are in the lookup grid id table
raster_index_list = np.where(streamid_list_full==streamid)[0]
for raster_index in raster_index_list:
writer.writerow(stream_info_table[raster_index][:6] + [peak_flow])
os.remove(self.stream_info_file)
os.rename(temp_stream_info_file, self.stream_info_file)
def _generate_network_from_connectivity(self):
"""
Generate river network from connectivity file
"""
print "Generating river network from connectivity file ..."
connectivity_table = csv_to_list(self.connectivity_file)
self.stream_id_array = np.array([row[0] for row in connectivity_table], dtype=np.int)
#add each stream segment to network
for connectivity_info in connectivity_table:
stream_id = int(connectivity_info[0])
downstream_id = int(connectivity_info[1])
#add outlet to list of outlets if downstream id is zero
if downstream_id == 0:
self.outlet_id_list.append(stream_id)
self.stream_segments.append(StreamSegment(stream_id=stream_id,
down_id=downstream_id,
up_id_array=connectivity_info[2:2+int(connectivity_info[2])]))
def _generate_network_from_connectivity(self):
"""
Generate river network from connectivity file
"""
print("Generating river network from connectivity file ...")
connectivity_table = csv_to_list(self.connectivity_file)
self.stream_id_array = np.array([row[0] for row in connectivity_table], dtype=np.int)
#add each stream segment to network
for connectivity_info in connectivity_table:
stream_id = int(connectivity_info[0])
downstream_id = int(connectivity_info[1])
#add outlet to list of outlets if downstream id is zero
if downstream_id == 0:
self.outlet_id_list.append(stream_id)
self.stream_segments.append(StreamSegment(stream_id=stream_id,
down_id=downstream_id,
up_id_array=connectivity_info[2:2+int(connectivity_info[2])]))
def _add_gage_ids_natur_flow_to_network(self):
"""
This adds gage and natural flow information
to the network from the file
"""
print "Adding Gage Station and Natur Flow info from:" , self.gage_ids_natur_flow_file
gage_id_natur_flow_table = csv_to_list(self.gage_ids_natur_flow_file)
for stream_info in gage_id_natur_flow_table[1:]:
if stream_info[0] != "":
stream_index = self._find_stream_segment_index(int(float(stream_info[0])))
if stream_index != None:
#add natural flow
self.stream_segments[stream_index].natural_flow = int(float(stream_info[1]))
#add station id
try:
station_id = str(int(float(stream_info[2])))
except Exception:
continue
pass
if station_id != "":
self.stream_undex_with_usgs_station.append(stream_index)
self.stream_segments[stream_index].station = USGSStreamGage(station_id)
def append_streamflow_from_rapid_output(self, rapid_output_file,
date_peak_search_start=None,
date_peak_search_end=None):
"""
Generate StreamFlow raster
Create AutoRAPID INPUT from single RAPID output
"""
print "Appending streamflow for:", self.stream_info_file
#get information from datasets
#get list of streamids
stream_info_table = csv_to_list(self.stream_info_file, ", ")[1:]
#Columns: DEM_1D_Index Row Col StreamID StreamDirection
streamid_list_full = np.array([row[3] for row in stream_info_table], dtype=np.int32)
streamid_list_unique = np.unique(streamid_list_full)
temp_stream_info_file = "{0}_temp.txt".format(os.path.splitext(self.stream_info_file)[0])
print "Analyzing data and appending to list ..."
with open(temp_stream_info_file, 'wb') as outfile:
writer = csv.writer(outfile, delimiter=" ")
writer.writerow(["DEM_1D_Index", "Row", "Col", "StreamID", "StreamDirection", "Slope", "Flow"])
with RAPIDDataset(rapid_output_file) as data_nc:
time_range = data_nc.get_time_index_range(date_search_start=date_peak_search_start,
date_search_end=date_peak_search_end)
#perform operation in max chunk size of 4,000
max_chunk_size = 8*365*5*4000 #5 years of 3hr data (8/day) with 4000 comids at a time
time_length = 8*365*5 #assume 5 years of 3hr data
if time_range is not None:
time_length = len(time_range)
else:
time_length = data_nc.size_time
streamid_list_length = len(streamid_list_unique)
if streamid_list_length <=0:
raise IndexError("Invalid stream info file {0}." \
" No stream ID's found ...".format(self.stream_info_file))
step_size = min(max_chunk_size/time_length, streamid_list_length)
for list_index_start in xrange(0, streamid_list_length, step_size):
list_index_end = min(list_index_start+step_size, streamid_list_length)
print "River ID subset range {0} to {1} of {2} ...".format(list_index_start,
list_index_end,
streamid_list_length)
print "Extracting data ..."
valid_stream_indices, valid_stream_ids, missing_stream_ids = \
data_nc.get_subset_riverid_index_list(streamid_list_unique[list_index_start:list_index_end])
streamflow_array = data_nc.get_qout_index(valid_stream_indices,
time_index_array=time_range)
print "Calculating peakflow and writing to file ..."
for streamid_index, streamid in enumerate(valid_stream_ids):
#get where streamids are in the lookup grid id table
peak_flow = max(streamflow_array[streamid_index])
raster_index_list = np.where(streamid_list_full==streamid)[0]
for raster_index in raster_index_list:
writer.writerow(stream_info_table[raster_index][:6] + [peak_flow])
for missing_streamid in missing_stream_ids:
#set flow to zero for missing stream ids
raster_index_list = np.where(streamid_list_full==missing_streamid)[0]
for raster_index in raster_index_list:
writer.writerow(stream_info_table[raster_index][:6] + [0])
os.remove(self.stream_info_file)
os.rename(temp_stream_info_file, self.stream_info_file)
print "Appending streamflow complete for:", self.stream_info_file
def append_streamflow_from_ecmwf_rapid_output(self, prediction_folder,
method_x, method_y):
"""
Generate StreamFlow raster
Create AutoRAPID INPUT from ECMWF predicitons
method_x = the first axis - it produces the max, min, mean, mean_plus_std, mean_minus_std hydrograph data for the 52 ensembles
method_y = the second axis - it calculates the max, min, mean, mean_plus_std, mean_minus_std value from method_x
"""
print "Generating Streamflow Raster ..."
#get list of streamidS
stream_info_table = csv_to_list(self.stream_info_file, ", ")[1:]
#Columns: DEM_1D_Index Row Col StreamID StreamDirection
streamid_list_full = np.array([row[3] for row in stream_info_table], dtype=np.int32)
streamid_list_unique = np.unique(streamid_list_full)
if not streamid_list_unique:
raise Exception("ERROR: No stream id values found in stream info file.")
#Get list of prediciton files
prediction_files = sorted([os.path.join(prediction_folder,f) for f in os.listdir(prediction_folder) \
if not os.path.isdir(os.path.join(prediction_folder, f)) and f.lower().endswith('.nc')],
reverse=True)
print "Finding streamid indices ..."
with RAPIDDataset(prediction_files[0]) as data_nc:
reordered_streamid_index_list = data_nc.get_subset_riverid_index_list(streamid_list_unique)[0]
first_half_size = 40
if data_nc.size_time == 41 or data_nc.size_time == 61:
first_half_size = 41
elif data_nc.size_time == 85 or data_nc.size_time == 125:
#run at full resolution for all
first_half_size = 65
print "Extracting Data ..."
reach_prediciton_array_first_half = np.zeros((len(streamid_list_unique),len(prediction_files),first_half_size))
reach_prediciton_array_second_half = np.zeros((len(streamid_list_unique),len(prediction_files),20))
#get information from datasets
for file_index, prediction_file in enumerate(prediction_files):
data_values_2d_array = []
try:
ensemble_index = int(os.path.basename(prediction_file)[:-3].split("_")[-1])
#Get hydrograph data from ECMWF Ensemble
with RAPIDDataset(prediction_file) as data_nc:
data_values_2d_array = data_nc.get_qout_index(reordered_streamid_index_list)
#add data to main arrays and order in order of interim comids
if len(data_values_2d_array) > 0:
for comid_index in range(len(streamid_list_unique)):
if(ensemble_index < 52):
reach_prediciton_array_first_half[comid_index][file_index] = data_values_2d_array[comid_index][:first_half_size]
reach_prediciton_array_second_half[comid_index][file_index] = data_values_2d_array[comid_index][first_half_size:]
if(ensemble_index == 52):
if first_half_size == 65:
#convert to 3hr-6hr
streamflow_1hr = data_values_2d_array[comid_index][:90:3]
# get the time series of 3 hr/6 hr data
streamflow_3hr_6hr = data_values_2d_array[comid_index][90:]
# concatenate all time series
reach_prediciton_array_first_half[comid_index][file_index] = np.concatenate([streamflow_1hr, streamflow_3hr_6hr])
elif data_nc.size_time == 125:
#convert to 6hr
streamflow_1hr = data_values_2d_array[comid_index][:90:6]
# calculate time series of 6 hr data from 3 hr data
streamflow_3hr = data_values_2d_array[comid_index][90:109:2]
# get the time series of 6 hr data
streamflow_6hr = data_values_2d_array[comid_index][109:]
# concatenate all time series
reach_prediciton_array_first_half[comid_index][file_index] = np.concatenate([streamflow_1hr, streamflow_3hr, streamflow_6hr])
else:
reach_prediciton_array_first_half[comid_index][file_index] = data_values_2d_array[comid_index][:]
except Exception as e:
print e
#pass
print "Analyzing data and writing output ..."
temp_stream_info_file = "{0}_temp.txt".format(os.path.splitext(self.stream_info_file)[0])
with open(temp_stream_info_file, 'wb') as outfile:
writer = csv.writer(outfile, delimiter=" ")
writer.writerow(["DEM_1D_Index", "Row", "Col", "StreamID", "StreamDirection", "Slope", "Flow"])
for streamid_index, streamid in enumerate(streamid_list_unique):
#perform analysis on datasets
all_data_first = reach_prediciton_array_first_half[streamid_index]
all_data_second = reach_prediciton_array_second_half[streamid_index]
series = []
if "mean" in method_x:
#get mean
mean_data_first = np.mean(all_data_first, axis=0)
mean_data_second = np.mean(all_data_second, axis=0)
series = np.concatenate([mean_data_first,mean_data_second])
if "std" in method_x:
#get std dev
std_dev_first = np.std(all_data_first, axis=0)
std_dev_second = np.std(all_data_second, axis=0)
std_dev = np.concatenate([std_dev_first,std_dev_second])
if method_x == "mean_plus_std":
#mean plus std
series += std_dev
elif method_x == "mean_minus_std":
#mean minus std
series -= std_dev
elif method_x == "max":
#get max
max_data_first = np.amax(all_data_first, axis=0)
max_data_second = np.amax(all_data_second, axis=0)
series = np.concatenate([max_data_first,max_data_second])
elif method_x == "min":
#get min
min_data_first = np.amin(all_data_first, axis=0)
min_data_second = np.amin(all_data_second, axis=0)
series = np.concatenate([min_data_first,min_data_second])
data_val = 0
if "mean" in method_y:
#get mean
data_val = np.mean(series)
if "std" in method_y:
#get std dev
std_dev = np.std(series)
if method_y == "mean_plus_std":
#mean plus std
data_val += std_dev
elif method_y == "mean_minus_std":
#mean minus std
data_val -= std_dev
elif method_y == "max":
#get max
data_val = np.amax(series)
elif method_y == "min":
#get min
data_val = np.amin(series)
#get where streamids are in the lookup grid id table
raster_index_list = np.where(streamid_list_full==streamid)[0]
for raster_index in raster_index_list:
writer.writerow(stream_info_table[raster_index][:6] + [data_val])
os.remove(self.stream_info_file)
os.rename(temp_stream_info_file, self.stream_info_file)