コード例 #1
0
 def _add_gage_ids_natur_flow_to_network(self):
     """
     This adds gage and natural flow information 
     to the network from the file
     """
     print("Adding Gage Station and Natur Flow info from: {0}".format(
         self.gage_ids_natur_flow_file))
     gage_id_natur_flow_table = csv_to_list(self.gage_ids_natur_flow_file)
     for stream_info in gage_id_natur_flow_table[1:]:
         if stream_info[0] != "":
             stream_index = self._find_stream_segment_index(
                 int(float(stream_info[0])))
             if stream_index != None:
                 #add natural flow
                 self.stream_segments[stream_index].natural_flow = int(
                     float(stream_info[1]))
                 #add station id
                 try:
                     station_id = str(int(float(stream_info[2])))
                 except Exception:
                     continue
                     pass
                 if station_id != "":
                     self.stream_undex_with_usgs_station.append(
                         stream_index)
                     self.stream_segments[
                         stream_index].station = USGSStreamGage(station_id)
コード例 #2
0
ファイル: prepare.py プロジェクト: solomonvimal/AutoRoutePy
    def append_streamflow_from_stream_shapefile(self, stream_id_field, streamflow_field):
        """
        Appends streamflow from values in shapefile 
        """
        stream_shapefile = ogr.Open(self.stream_shapefile_path)
        stream_shp_layer = stream_shapefile.GetLayer()

        self.spatially_filter_streamfile_layer_by_elevation_dem(stream_shp_layer)

        print "Writing output to file ..."
        stream_info_table = csv_to_list(self.stream_info_file, ", ")[1:]
        #Columns: DEM_1D_Index Row Col StreamID StreamDirection
        stream_id_list = np.array([row[3] for row in stream_info_table], dtype=np.int32)
        
        temp_stream_info_file = "{0}_temp.txt".format(os.path.splitext(self.stream_info_file)[0])
        with open(temp_stream_info_file, 'wb') as outfile:
            writer = csv.writer(outfile, delimiter=" ")
            writer.writerow(["DEM_1D_Index", "Row", "Col", "StreamID", "StreamDirection", "Slope", "Flow"])
            for feature in stream_shp_layer:
                #find all raster indices associates with the comid
                raster_index_list = np.where(stream_id_list==int(float(feature.GetField(stream_id_field))))[0]
                #add streamflow associated with comid    
                streamflow = feature.GetField(streamflow_field)
                for raster_index in raster_index_list:
                    writer.writerow(stream_info_table[raster_index][:6] + [streamflow])

        os.remove(self.stream_info_file)
        os.rename(temp_stream_info_file, self.stream_info_file)
コード例 #3
0
ファイル: prepare.py プロジェクト: solomonvimal/AutoRoutePy
 def append_streamflow_from_return_period_file(self, return_period_file, 
                                               return_period):
     """
     Generates return period raster from return period file
     """
     print "Extracting Return Period Data ..."
     return_period_nc = Dataset(return_period_file, mode="r")
     if return_period == "return_period_20": 
         return_period_data = return_period_nc.variables['return_period_20'][:]
     elif return_period == "return_period_10": 
         return_period_data = return_period_nc.variables['return_period_10'][:]
     elif return_period == "return_period_2": 
         return_period_data = return_period_nc.variables['return_period_2'][:]
     elif return_period == "max_flow": 
         return_period_data = return_period_nc.variables['return_period_2'][:]
     else:
         raise Exception("Invalid return period definition.")
     rivid_var = 'COMID'
     if 'rivid' in return_period_nc.variables:
         rivid_var = 'rivid'
     return_period_comids = return_period_nc.variables[rivid_var][:]
     return_period_nc.close()
     
     #get where streamids are in the lookup grid id table
     stream_info_table = csv_to_list(self.stream_info_file, ", ")[1:]
     streamid_list_full = np.array([row[3] for row in stream_info_table], dtype=np.int32)
     streamid_list_unique = np.unique(streamid_list_full)
     print "Analyzing data and appending to list ..."
     
     temp_stream_info_file = "{0}_temp.txt".format(os.path.splitext(self.stream_info_file)[0])
     with open(temp_stream_info_file, 'wb') as outfile:
         writer = csv.writer(outfile, delimiter=" ")
         writer.writerow(["DEM_1D_Index", "Row", "Col", "StreamID", "StreamDirection", "Slope", "Flow"])
         for streamid in streamid_list_unique:
             try:
                 #get where streamids are in netcdf file
                 streamid_index = np.where(return_period_comids==streamid)[0][0]
                 peak_flow = return_period_data[streamid_index]
             except IndexError:
                 print "ReachID", streamid, "not found in netCDF dataset. Setting value to zero ..."
                 peak_flow = 0
                 pass
                 
             #get where streamids are in the lookup grid id table
             raster_index_list = np.where(streamid_list_full==streamid)[0]
             for raster_index in raster_index_list:
                 writer.writerow(stream_info_table[raster_index][:6] + [peak_flow])
                 
     os.remove(self.stream_info_file)
     os.rename(temp_stream_info_file, self.stream_info_file)
コード例 #4
0
 def _generate_network_from_connectivity(self):
     """
     Generate river network from connectivity file
     """
     print "Generating river network from connectivity file ..."
     connectivity_table = csv_to_list(self.connectivity_file)
     self.stream_id_array = np.array([row[0] for row in connectivity_table], dtype=np.int)
     #add each stream segment to network
     for connectivity_info in connectivity_table:
         stream_id = int(connectivity_info[0])
         downstream_id = int(connectivity_info[1])
         #add outlet to list of outlets if downstream id is zero
         if downstream_id == 0:
             self.outlet_id_list.append(stream_id)
             
         self.stream_segments.append(StreamSegment(stream_id=stream_id,
                                                   down_id=downstream_id,
                                                   up_id_array=connectivity_info[2:2+int(connectivity_info[2])]))
コード例 #5
0
 def _generate_network_from_connectivity(self):
     """
     Generate river network from connectivity file
     """
     print("Generating river network from connectivity file ...")
     connectivity_table = csv_to_list(self.connectivity_file)
     self.stream_id_array = np.array([row[0] for row in connectivity_table], dtype=np.int)
     #add each stream segment to network
     for connectivity_info in connectivity_table:
         stream_id = int(connectivity_info[0])
         downstream_id = int(connectivity_info[1])
         #add outlet to list of outlets if downstream id is zero
         if downstream_id == 0:
             self.outlet_id_list.append(stream_id)
             
         self.stream_segments.append(StreamSegment(stream_id=stream_id,
                                                   down_id=downstream_id,
                                                   up_id_array=connectivity_info[2:2+int(connectivity_info[2])]))
コード例 #6
0
 def _add_gage_ids_natur_flow_to_network(self):
     """
     This adds gage and natural flow information 
     to the network from the file
     """
     print "Adding Gage Station and Natur Flow info from:" , self.gage_ids_natur_flow_file
     gage_id_natur_flow_table = csv_to_list(self.gage_ids_natur_flow_file)
     for stream_info in gage_id_natur_flow_table[1:]:
         if stream_info[0] != "":
             stream_index = self._find_stream_segment_index(int(float(stream_info[0])))
             if stream_index != None:
                 #add natural flow
                 self.stream_segments[stream_index].natural_flow = int(float(stream_info[1]))
                 #add station id
                 try:
                     station_id = str(int(float(stream_info[2])))
                 except Exception:
                     continue
                     pass
                 if station_id != "":
                     self.stream_undex_with_usgs_station.append(stream_index)
                     self.stream_segments[stream_index].station = USGSStreamGage(station_id)
コード例 #7
0
ファイル: prepare.py プロジェクト: solomonvimal/AutoRoutePy
    def append_streamflow_from_rapid_output(self, rapid_output_file,
                                            date_peak_search_start=None,
                                            date_peak_search_end=None):
        """
        Generate StreamFlow raster
        Create AutoRAPID INPUT from single RAPID output
        """
        print "Appending streamflow for:", self.stream_info_file
        #get information from datasets
        #get list of streamids
        stream_info_table = csv_to_list(self.stream_info_file, ", ")[1:]
        #Columns: DEM_1D_Index Row Col StreamID StreamDirection
        streamid_list_full = np.array([row[3] for row in stream_info_table], dtype=np.int32)
        streamid_list_unique = np.unique(streamid_list_full)
        
        temp_stream_info_file = "{0}_temp.txt".format(os.path.splitext(self.stream_info_file)[0])
        print "Analyzing data and appending to list ..."
        with open(temp_stream_info_file, 'wb') as outfile:
            writer = csv.writer(outfile, delimiter=" ")
            writer.writerow(["DEM_1D_Index", "Row", "Col", "StreamID", "StreamDirection", "Slope", "Flow"])
            
            with RAPIDDataset(rapid_output_file) as data_nc:
                
                time_range = data_nc.get_time_index_range(date_search_start=date_peak_search_start,
                                                          date_search_end=date_peak_search_end)
                #perform operation in max chunk size of 4,000
                max_chunk_size = 8*365*5*4000 #5 years of 3hr data (8/day) with 4000 comids at a time
                time_length = 8*365*5 #assume 5 years of 3hr data
                if time_range is not None:
                    time_length = len(time_range)
                else:
                    time_length = data_nc.size_time

                streamid_list_length = len(streamid_list_unique)
                if streamid_list_length <=0:
                    raise IndexError("Invalid stream info file {0}." \
                                     " No stream ID's found ...".format(self.stream_info_file))
                
                step_size = min(max_chunk_size/time_length, streamid_list_length)
                for list_index_start in xrange(0, streamid_list_length, step_size):
                    list_index_end = min(list_index_start+step_size, streamid_list_length)
                    print "River ID subset range {0} to {1} of {2} ...".format(list_index_start,
                                                                               list_index_end,
                                                                               streamid_list_length)
                    print "Extracting data ..."
                    valid_stream_indices, valid_stream_ids, missing_stream_ids = \
                        data_nc.get_subset_riverid_index_list(streamid_list_unique[list_index_start:list_index_end])
                        
                    streamflow_array = data_nc.get_qout_index(valid_stream_indices,
                                                              time_index_array=time_range)
                    
                    print "Calculating peakflow and writing to file ..."
                    for streamid_index, streamid in enumerate(valid_stream_ids):
                        #get where streamids are in the lookup grid id table
                        peak_flow = max(streamflow_array[streamid_index])
                        raster_index_list = np.where(streamid_list_full==streamid)[0]
                        for raster_index in raster_index_list:
                            writer.writerow(stream_info_table[raster_index][:6] + [peak_flow])

                    for missing_streamid in missing_stream_ids:
                        #set flow to zero for missing stream ids
                        raster_index_list = np.where(streamid_list_full==missing_streamid)[0]
                        for raster_index in raster_index_list:
                            writer.writerow(stream_info_table[raster_index][:6] + [0])


         
        os.remove(self.stream_info_file)
        os.rename(temp_stream_info_file, self.stream_info_file)

        print "Appending streamflow complete for:", self.stream_info_file
コード例 #8
0
ファイル: prepare.py プロジェクト: solomonvimal/AutoRoutePy
    def append_streamflow_from_ecmwf_rapid_output(self, prediction_folder,
                                                  method_x, method_y):
        """
        Generate StreamFlow raster
        Create AutoRAPID INPUT from ECMWF predicitons
     
        method_x = the first axis - it produces the max, min, mean, mean_plus_std, mean_minus_std hydrograph data for the 52 ensembles
        method_y = the second axis - it calculates the max, min, mean, mean_plus_std, mean_minus_std value from method_x
        """
     
        print "Generating Streamflow Raster ..."
        #get list of streamidS
        stream_info_table = csv_to_list(self.stream_info_file, ", ")[1:]

        #Columns: DEM_1D_Index Row Col StreamID StreamDirection
        streamid_list_full = np.array([row[3] for row in stream_info_table], dtype=np.int32)
        streamid_list_unique = np.unique(streamid_list_full)
        if not streamid_list_unique:
            raise Exception("ERROR: No stream id values found in stream info file.")
        
        #Get list of prediciton files
        prediction_files = sorted([os.path.join(prediction_folder,f) for f in os.listdir(prediction_folder) \
                                  if not os.path.isdir(os.path.join(prediction_folder, f)) and f.lower().endswith('.nc')],
                                  reverse=True)
     
     
        print "Finding streamid indices ..."
        with RAPIDDataset(prediction_files[0]) as data_nc:
            reordered_streamid_index_list = data_nc.get_subset_riverid_index_list(streamid_list_unique)[0]

            first_half_size = 40
            if data_nc.size_time == 41 or data_nc.size_time == 61:
                first_half_size = 41
            elif data_nc.size_time == 85 or data_nc.size_time == 125:
                #run at full resolution for all
                first_half_size = 65
        
        print "Extracting Data ..."
        reach_prediciton_array_first_half = np.zeros((len(streamid_list_unique),len(prediction_files),first_half_size))
        reach_prediciton_array_second_half = np.zeros((len(streamid_list_unique),len(prediction_files),20))
        #get information from datasets
        for file_index, prediction_file in enumerate(prediction_files):
            data_values_2d_array = []
            try:
                ensemble_index = int(os.path.basename(prediction_file)[:-3].split("_")[-1])
                #Get hydrograph data from ECMWF Ensemble
                with RAPIDDataset(prediction_file) as data_nc:
                    data_values_2d_array = data_nc.get_qout_index(reordered_streamid_index_list)
    
                    #add data to main arrays and order in order of interim comids
                    if len(data_values_2d_array) > 0:
                        for comid_index in range(len(streamid_list_unique)):
                            if(ensemble_index < 52):
                                reach_prediciton_array_first_half[comid_index][file_index] = data_values_2d_array[comid_index][:first_half_size]
                                reach_prediciton_array_second_half[comid_index][file_index] = data_values_2d_array[comid_index][first_half_size:]
                            if(ensemble_index == 52):
                                if first_half_size == 65:
                                    #convert to 3hr-6hr
                                    streamflow_1hr = data_values_2d_array[comid_index][:90:3]
                                    # get the time series of 3 hr/6 hr data
                                    streamflow_3hr_6hr = data_values_2d_array[comid_index][90:]
                                    # concatenate all time series
                                    reach_prediciton_array_first_half[comid_index][file_index] = np.concatenate([streamflow_1hr, streamflow_3hr_6hr])
                                elif data_nc.size_time == 125:
                                    #convert to 6hr
                                    streamflow_1hr = data_values_2d_array[comid_index][:90:6]
                                    # calculate time series of 6 hr data from 3 hr data
                                    streamflow_3hr = data_values_2d_array[comid_index][90:109:2]
                                    # get the time series of 6 hr data
                                    streamflow_6hr = data_values_2d_array[comid_index][109:]
                                    # concatenate all time series
                                    reach_prediciton_array_first_half[comid_index][file_index] = np.concatenate([streamflow_1hr, streamflow_3hr, streamflow_6hr])
                                else:
                                    reach_prediciton_array_first_half[comid_index][file_index] = data_values_2d_array[comid_index][:]
                                
            except Exception as e:
                print e
                #pass
     
        print "Analyzing data and writing output ..."
        temp_stream_info_file = "{0}_temp.txt".format(os.path.splitext(self.stream_info_file)[0])
        with open(temp_stream_info_file, 'wb') as outfile:
            writer = csv.writer(outfile, delimiter=" ")
            writer.writerow(["DEM_1D_Index", "Row", "Col", "StreamID", "StreamDirection", "Slope", "Flow"])

            for streamid_index, streamid in enumerate(streamid_list_unique):
                #perform analysis on datasets
                all_data_first = reach_prediciton_array_first_half[streamid_index]
                all_data_second = reach_prediciton_array_second_half[streamid_index]
         
                series = []
         
                if "mean" in method_x:
                    #get mean
                    mean_data_first = np.mean(all_data_first, axis=0)
                    mean_data_second = np.mean(all_data_second, axis=0)
                    series = np.concatenate([mean_data_first,mean_data_second])
                    if "std" in method_x:
                        #get std dev
                        std_dev_first = np.std(all_data_first, axis=0)
                        std_dev_second = np.std(all_data_second, axis=0)
                        std_dev = np.concatenate([std_dev_first,std_dev_second])
                        if method_x == "mean_plus_std":
                            #mean plus std
                            series += std_dev
                        elif method_x == "mean_minus_std":
                            #mean minus std
                            series -= std_dev
         
                elif method_x == "max":
                    #get max
                    max_data_first = np.amax(all_data_first, axis=0)
                    max_data_second = np.amax(all_data_second, axis=0)
                    series = np.concatenate([max_data_first,max_data_second])
                elif method_x == "min":
                    #get min
                    min_data_first = np.amin(all_data_first, axis=0)
                    min_data_second = np.amin(all_data_second, axis=0)
                    series = np.concatenate([min_data_first,min_data_second])
         
                data_val = 0
                if "mean" in method_y:
                    #get mean
                    data_val = np.mean(series)
                    if "std" in method_y:
                        #get std dev
                        std_dev = np.std(series)
                        if method_y == "mean_plus_std":
                            #mean plus std
                            data_val += std_dev
                        elif method_y == "mean_minus_std":
                            #mean minus std
                            data_val -= std_dev
         
                elif method_y == "max":
                    #get max
                    data_val = np.amax(series)
                elif method_y == "min":
                    #get min
                    data_val = np.amin(series)
         
                #get where streamids are in the lookup grid id table
                raster_index_list = np.where(streamid_list_full==streamid)[0]
                for raster_index in raster_index_list:
                    writer.writerow(stream_info_table[raster_index][:6] + [data_val])

        os.remove(self.stream_info_file)
        os.rename(temp_stream_info_file, self.stream_info_file)