def netcdf2PCRobj(ncFile,varName,dateInput): # EHS (04 APR 2013): To convert netCDF (tss) file to PCR file. # The cloneMap is globally defined (outside this method). # Get netCDF file and variable name: f = nc.Dataset(ncFile) varName = str(varName) # date date = dateInput if isinstance(date, str) == True: date = \ datetime.datetime.strptime(str(date),'%Y-%m-%d') date = datetime.datetime(date.year,date.month,date.day) # time index (in the netCDF file) nctime = f.variables['time'] # A netCDF time variable object. idx = nc.date2index(date, nctime, calendar=nctime.calendar, \ select='exact') # convert to PCR object and close f outPCR = pcr.numpy2pcr(pcr.Scalar,(f.variables[varName][idx].data), \ float(f.variables[varName]._FillValue)) f.close(); f = None ; del f # PCRaster object return (outPCR)
def testOrdinalArray2Raster(self): pcraster.setclone("boolean_Expr.map") try: a = numpy.array([ [0, 1, 3], [20, -3, -2], [0, 9, 8] ]) result = pcraster.numpy2pcr(pcraster.Ordinal, a, 20) self.failUnless(self.mapEqualsValidated(result, "ordinal_Result.map"), "test1: %s" % ("Result and validated result are not the same")) except Exception as exception: self.failUnless(False, "test1: %s" % (str(exception)))
def testBooleanArray2Raster(self): pcraster.setclone("boolean_Expr.map") try: a = numpy.array([ [1, 0, 1], [20, 1, 1], [1, 1, 0] ], numpy.uint8) # uint8 is bugzilla #271 result = pcraster.numpy2pcr(pcraster.Boolean, a, 20) self.failUnless(self.mapEqualsValidated(result, "boolean_Result.map"), "test1: %s" % ("Result and validated result are not the same")) except Exception as exception: self.failUnless(False, "test1: %s" % (str(exception)))
def testLddArray2Raster(self): pcraster.setclone("and_Expr1.map") try: a = numpy.array([ [6, 5, 4], [6, 8, 7], [8, 8, 8] ]) result = pcraster.numpy2pcr(pcraster.Ldd, a, 20) self.failUnless(self.mapEqualsValidated(result, "ldd_Result.map"), "test1: %s" % ("Result and validated result are not the same")) except Exception as exception: self.failUnless(False, "test1: %s" % (str(exception)))
def testScalarArray2Raster(self): pcraster.setclone("boolean_Expr.map") try: a = numpy.array([ [0.5, 0.34202, 0.310676], [20, 0, -0.981627], [0.707107, 0.144356, 0.0174524] ]) result = pcraster.numpy2pcr(pcraster.Scalar, a, 20) self.failUnless(self.mapEqualsValidated(result, "sin_Result.map"), "test1: %s" % ("Result and validated result are not the same")) except Exception as exception: self.failUnless(False, "test1: %s" % (str(exception)))
def testDirectionalArray2Raster(self): pcraster.setclone("boolean_Expr.map") pcraster.setglobaloption("degrees") try: a = numpy.array([ [math.radians(350),math.radians(0),math.radians(0.01)],\ [20,math.radians(350),math.radians(21)],\ [math.radians(359),math.radians(40),math.radians(0)] ]) result = pcraster.numpy2pcr(pcraster.Directional, a, 20) self.failUnless(self.mapEqualsValidated(result, "directional_Result2.map"), "test1: %s" % ("Result and validated result are not the same")) except Exception as exception: self.failUnless(False, "test1: %s" % (str(exception)))
def lookupResRegMatr(ReserVoirLocs, values, hq, JDOY): np_res_ids = pcr.pcr2numpy(ReserVoirLocs, 0) npvalues = pcr.pcr2numpy(values, 0) out = np.copy(npvalues) * 0.0 if len(hq) > 0: for key in hq: value = npvalues[np.where(np_res_ids == key)] val = np.interp(value, hq[key][:, 0], hq[key][:, JDOY]) out[np.where(np_res_ids == key)] = val return pcr.numpy2pcr(pcr.Scalar, out, 0)
def returnMapValue(pcrX,x,coord): #-retrieves value from an array and update values in the map if x.ndim == 1: nrRows= 1 tempIDArray= pcr.pcr2numpy(pcrX,MV) #print tempIDArray temporary= tempIDArray nrRows= coord.shape[0] for iCnt in xrange(nrRows): row,col= coord[iCnt,:] if row != MV and col != MV: tempIDArray[row,col]= (x[iCnt]) # print iCnt,row,col,x[iCnt] pcrX= pcr.numpy2pcr(pcr.Scalar,tempIDArray,MV) return pcrX
def test_round_trip_numpy_array_with_nan(self): array = numpy.array([ [-2, -1], [ 0, numpy.nan], [ 1, 2 ] ]) nrRows, nrCols, cellSize = 3, 2, 1.0 west, north = 0.0, 0.0 pcraster.setclone(nrRows, nrCols, cellSize, west, north) raster = pcraster.numpy2pcr(pcraster.Scalar, array, numpy.nan) array2 = pcraster.pcr2numpy(raster, numpy.nan) self.assertEqual(array2[0][0], -2) self.assertEqual(array2[0][1], -1) self.assertEqual(array2[1][0], 0) self.assertTrue(numpy.isnan(array2[1][1])) self.assertEqual(array2[2][0], 1) self.assertEqual(array2[2][1], 2)
def set_value(self, long_var_name, src): """ Set the values(s) in a map using a numpy array as source :var long_var_name: identifier of a variable in the model. :var src: all values to set for the given variable. If only one value is present a uniform map will be set in the wflow model. """ # first part should be the component name self.bmilogger.debug("set_value: " + long_var_name) cname = long_var_name.split(self.comp_sep) if cname[0] in self.bmimodels: self.bmimodels[cname[0]].set_value(cname[1], src) if self.wrtodisk: pcr.report( pcr.numpy2pcr(pcr.Scalar, src, -999), long_var_name + "_set_" + str(self.get_current_time()) + ".map", )
def lookupResFunc(ReserVoirLocs, values, sh, dirLookup): np_res_ids = pcr.pcr2numpy(ReserVoirLocs, 0) npvalues = pcr.pcr2numpy(values, 0) out = np.copy(npvalues) * 0.0 if len(sh) > 0: for key in sh: value = npvalues[np.where(np_res_ids == key)] if dirLookup == "0-1": val = np.interp(value, sh[key][:, 0], sh[key][:, 1]) if dirLookup == "1-0": val = np.interp(value, sh[key][:, 1], sh[key][:, 0]) out[np.where(np_res_ids == key)] = val return pcr.numpy2pcr(pcr.Scalar, out, 0)
def points_to_map(in_map, xcor, ycor, tolerance): """ Returns a map with non zero values at the points defined in X, Y pairs. It's goal is to replace the pcraster col2map program. tolerance should be 0.5 to select single points Performance is not very good and scales linear with the number of points Input: - in_map - map to determine coordinates from - xcor - x coordinate (array or single value) - ycor - y coordinate (array or single value) - tolerance - tolerance in cell units. 0.5 selects a single cell\ 10 would select a 10x10 block of cells Output: - Map with values burned in. 1 for first point, 2 for second and so on """ point = in_map * 0.0 x = pcr.pcr2numpy(pcr.xcoordinate(pcr.defined(in_map)), np.nan) y = pcr.pcr2numpy(pcr.ycoordinate(pcr.defined(in_map)), np.nan) cell_length = float(pcr.celllength()) # simple check to use both floats and numpy arrays try: c = xcor.ndim except: xcor = np.array([xcor]) ycor = np.array([ycor]) # Loop over points and "burn in" map for n in range(0, xcor.size): if Verbose: print(n) diffx = x - xcor[n] diffy = y - ycor[n] col_ = np.absolute(diffx) <= (cell_length * tolerance) # cellsize row_ = np.absolute(diffy) <= (cell_length * tolerance) # cellsize point = point + pcr.numpy2pcr(pcr.Scalar, ((col_ * row_) * (n + 1)), np.nan) return pcr.ordinal(point)
def gettimestep(self, timestep, logging, var="P"): """ Gets a map for a single timestep. reads data in blocks assuming sequential access timestep: framework timestep (1-based) logging: python logging object var: variable to get from the file """ if var in self.dataset.variables: np_step = self.alldat[var][ timestep - 1, self.latidx.min() : self.latidx.max() + 1, self.lonidx.min() : self.lonidx.max() + 1, ] miss = float(self.dataset.variables[var]._FillValue) return pcr.numpy2pcr(pcr.Scalar, np_step, miss), True else: logging.debug("Var (" + var + ") not found returning map with 0.0") return pcr.cover(pcr.scalar(0.0)), False
def get_value_at_indices(self, long_var_name, inds): """ Get a numpy array of the values at the given indices :var long_var_name: identifier of a variable in the model: :var inds: List of list each tuple contains one index for each dimension of the given variable, i.e. each tuple indicates one element in the multi-dimensional variable array: :return: numpy array of values in the data type returned by the function get_var_type. """ cname = long_var_name.split(self.comp_sep) if cname[0] in self.bmimodels: tmp = self.bmimodels[cname[0]].get_value(cname[1]) if self.wrtodisk: pcr.report( pcr.numpy2pcr(pcr.Scalar, tmp, -999), long_var_name + "_get_" + str(self.get_current_time()) + ".map", ) return self.bmimodels[cname[0]].get_value_at_indices(cname[1], inds) # else: return None
def get_value(self, long_var_name): """ Get the value(s) of a variable as a numpy array :var long_var_name: name of the variable :return: a np array of long_var_name """ # first part should be the component name self.bmilogger.debug("get_value: " + long_var_name) cname = long_var_name.split(self.comp_sep) if cname[0] in self.bmimodels: tmp = self.bmimodels[cname[0]].get_value(cname[1]) if self.wrtodisk: pcr.report( pcr.numpy2pcr(pcr.Scalar, tmp, -999), long_var_name + "_get_" + str(self.get_current_time()) + ".map", ) return tmp else: self.bmilogger.error("get_value: " + long_var_name + " returning None!!!!") return None
def idtoid(sourceidmap, targetidmap, valuemap): """ tranfer the values from valuemap at the point id's in sourceidmap to the areas in targetidmap. :param pointmap: :param areamap: :param valuemap: :return: """ _area = pcr.pcr2numpy(targetidmap, 0.0).copy().astype(float) _pt = pcr.pcr2numpy(sourceidmap, 0.0).copy() _val = pcr.pcr2numpy(valuemap, 0.0).copy() for val in np.unique(_pt): if val > 0: # _area[_area == val] = np.mean(_val[_pt == val]) retmap = pcr.numpy2pcr(pcr.Scalar, _area, 0.0) return retmap
def test_pcr_as_numpy(self): array = numpy.array([ [-2.0, -1.0 ], [ 0.0, numpy.nan], [ 1.0, 2.0 ] ]) nrRows, nrCols, cellSize = 3, 2, 1.0 west, north = 0.0, 0.0 pcraster.setclone(nrRows, nrCols, cellSize, west, north) # Create a raster. raster = pcraster.numpy2pcr(pcraster.Scalar, array, 999.0) # Test type checking. with self.assertRaises(Exception) as context_manager: pcraster.pcr_as_numpy(5) self.assertEqual(str(context_manager.exception), "Expecting a PCRaster field") # Create an array referencing the raster. array2 = pcraster.pcr_as_numpy(raster) self.assertEqual(array2[0][0], -2) self.assertEqual(array2[0][1], -1) self.assertEqual(array2[1][0], 0) self.assertTrue(numpy.isnan(array2[1][1])) self.assertEqual(array2[2][0], 1) self.assertEqual(array2[2][1], 2) # Change the array and verify the raster changed too. array2[0][0] = 5.0 self.assertEqual(pcraster.pcr2numpy(raster, 999.0)[0][0], 5.0) # Replace exising raster and verify the array still behaves. raster += 1.0 self.assertEqual(array2[0][0], 5.0) # Delete the raster and verify the array still behaves. del raster self.assertEqual(array2[0][0], 5.0) self.assertEqual(array2[2][1], 2.0)
def gettimestep(self, timestep, logging, var="P", tsdatetime=None): """ Gets a map for a single timestep. reads data in blocks assuming sequential access timestep: framework timestep (1-based) logging: python logging object var: variable to get from the file """ ncindex = timestep - 1 if var in self.dataset.variables: if tsdatetime != None: if tsdatetime.replace(tzinfo=None) != self.datetimelist[ ncindex ].replace(tzinfo=None): logging.warning( "Date/time of state (" + var + " in " + self.fname + ")does not match. Wanted " + str(tsdatetime) + " got " + str(self.datetimelist[ncindex]) ) np_step = self.dataset.variables[var][ ncindex, self.latidx.min() : self.latidx.max() + 1, self.lonidx.min() : self.lonidx.max() + 1, ] miss = float(self.dataset.variables[var]._FillValue) return pcr.numpy2pcr(pcr.Scalar, np_step, miss), True else: # logging.debug("Var (" + var + ") not found returning map with 0.0") return pcr.cover(pcr.scalar(0.0)), False
def gettimestep(self, timestep, logging, tsdatetime=None, var="P", shifttime=False): """ Gets a map for a single timestep. reads data in blocks assuming sequential access :var timestep: framework timestep (1-based) :var logging: python logging object :var var: variable to get from the file :var shifttime: is True start at 1 in the NC file (instead of 0) :var tsdatetime: Assumed date/time of this timestep without ensembles (dims = 3) window = data[dpos,latidx.min():latidx.max()+1,lonidx.min():lonidx.max()+1] with ensembles from Delft-FEWS (dims = 4): window = data[dpos,realization,latidx.min():latidx.max()+1,lonidx.min():lonidx.max()+1] """ if shifttime: ncindex = timestep else: ncindex = timestep - 1 ncindex = ncindex + self.offset if self.datetimelist.size < ncindex + 1: ncindex = self.datetimelist.size - 1 if tsdatetime != None: if tsdatetime.replace( tzinfo=None) != self.datetimelist[ncindex].replace( tzinfo=None): logging.warning("Date/time does not match. Wanted " + str(tsdatetime) + " got " + str(self.datetimelist[ncindex])) import bisect pos = bisect.bisect_left(self.datetimelist, tsdatetime.replace(tzinfo=None)) if pos >= self.datetimelist.size: pos = self.datetimelist.size - 1 logging.warning( "No matching date/time found using last date/time again..." ) self.offset = pos - ncindex logging.warning( "Adjusting to the date/time at index and setting offset: " + str(pos) + ":" + str(self.offset) + ":" + str(self.datetimelist[pos])) ncindex = pos if var in self.alldat: # if ncindex == self.lstep: # Read new block of data in mem # logging.debug("reading new netcdf data block starting at: " + str(ncindex)) # for vars in self.alldat: # self.alldat[vars] = self.dataset.variables[vars][ncindex:ncindex + self.maxsteps] # # self.fstep = ncindex # self.lstep = ncindex + self.maxsteps if len(self.alldat[var].dimensions) == 3: np_step = self.alldat[var][ ncindex - self.fstep, self.latidx.min():self.latidx.max() + 1, self.lonidx.min():self.lonidx.max() + 1, ] if len(self.alldat[var].dimensions) == 4: np_step = self.alldat[var][ ncindex - self.fstep, 0, self.latidx.min():self.latidx.max() + 1, self.lonidx.min():self.lonidx.max() + 1, ] miss = float(self.dataset.variables[var]._FillValue) if self.flip: return pcr.numpy2pcr(pcr.Scalar, np.flipud(np_step).copy(), miss), True else: return pcr.numpy2pcr(pcr.Scalar, np_step, miss), True else: # logging.debug("Var (" + var + ") not found returning 0") return pcr.cover(pcr.scalar(0.0)), False
def main(): ### Read input arguments ##### logfilename = 'wtools_static_maps.log' parser = OptionParser() usage = "usage: %prog [options]" parser = OptionParser(usage=usage) parser.add_option('-q', '--quiet', dest='verbose', default=True, action='store_false', help='do not print status messages to stdout') parser.add_option('-i', '--ini', dest='inifile', default=None, help='ini file with settings for static_maps.exe') parser.add_option('-s', '--source', dest='source', default='wflow', help='Source folder containing clone (default=./wflow)') parser.add_option('-d', '--destination', dest='destination', default='staticmaps', help='Destination folder (default=./staticmaps)') parser.add_option('-r', '--river', dest='rivshp', default=None, help='river network polyline layer (ESRI Shapefile)') parser.add_option('-c', '--catchment', dest='catchshp', default=None, help='catchment polygon layer (ESRI Shapefile)') parser.add_option('-g', '--gauges', dest='gaugeshp', default=None, help='gauge point layer (ESRI Shapefile)') parser.add_option('-D', '--dem', dest='dem_in', default=None, help='digital elevation model (GeoTiff)') parser.add_option('-L', '--landuse', dest='landuse', default=None, help='land use / land cover layer (GeoTiff)') parser.add_option('-S', '--soiltype', dest='soil', default=None, help='soil type layer (GeoTiff)') parser.add_option('-V', '--vegetation', dest='lai', default=None, help='vegetation LAI layer location (containing 12 GeoTiffs <LAI00000.XXX.tif>)') parser.add_option('-O', '--other_maps', dest='other_maps', default=None, help='bracketed [] comma-separated list of paths to other maps that should be reprojected') parser.add_option('-C', '--clean', dest='clean', default=False, action='store_true', help='Clean the .xml files from static maps folder when finished') parser.add_option('-A', '--alltouch', dest='alltouch', default=False, action='store_true', help='option to burn catchments "all touching".\nUseful when catchment-size is small compared to cellsize') (options, args) = parser.parse_args() # parse other maps into an array options.other_maps = options.other_maps.replace(' ', '').replace('[', '').replace(']', '').split(',') options.source = os.path.abspath(options.source) clone_map = os.path.join(options.source, 'mask.map') clone_shp = os.path.join(options.source, 'mask.shp') clone_prj = os.path.join(options.source, 'mask.prj') if None in (options.inifile, options.rivshp, options.catchshp, options.dem_in): msg = """The following files are compulsory: - ini file - DEM (raster) - river (shape) - catchment (shape) """ print(msg) parser.print_help() sys.exit(1) if not os.path.exists(options.inifile): print 'path to ini file cannot be found' sys.exit(1) if not os.path.exists(options.rivshp): print 'path to river shape cannot be found' sys.exit(1) if not os.path.exists(options.catchshp): print 'path to catchment shape cannot be found' sys.exit(1) if not os.path.exists(options.dem_in): print 'path to DEM cannot be found' sys.exit(1) # open a logger, dependent on verbose print to screen or not logger, ch = wtools_lib.setlogger(logfilename, 'WTOOLS', options.verbose) # create directories # TODO: check if workdir is still necessary, try to keep in memory as much as possible # delete old files (when the source and destination folder are different) if np.logical_and(os.path.isdir(options.destination), options.destination is not options.source): shutil.rmtree(options.destination) if options.destination is not options.source: os.makedirs(options.destination) # Read mask if not(os.path.exists(clone_map)): logger.error('Clone file {:s} not found. Please run create_grid first.'.format(clone_map)) sys.exit(1) else: # set clone pcr.setclone(clone_map) # get the extent from clone.tif xax, yax, clone, fill_value = gis.gdal_readmap(clone_map, 'GTiff') trans = wtools_lib.get_geotransform(clone_map) extent = wtools_lib.get_extent(clone_map) xmin, ymin, xmax, ymax = extent zeros = np.zeros(clone.shape) ones = pcr.numpy2pcr(pcr.Scalar, np.ones(clone.shape), -9999) # get the projection from clone.tif srs = wtools_lib.get_projection(clone_map) unit_clone = srs.GetAttrValue('UNIT').lower() ### READ CONFIG FILE # open config-file config=wtools_lib.OpenConf(options.inifile) # read settings snapgaugestoriver = wtools_lib.configget(config, 'settings', 'snapgaugestoriver', True, datatype='boolean') burnalltouching = wtools_lib.configget(config, 'settings', 'burncatchalltouching', True, datatype='boolean') burninorder = wtools_lib.configget(config, 'settings', 'burncatchalltouching', False, datatype='boolean') verticetollerance = wtools_lib.configget(config, 'settings', 'vertice_tollerance', 0.0001, datatype='float') ''' read parameters ''' burn_outlets = wtools_lib.configget(config, 'parameters', 'burn_outlets', 10000, datatype='int') burn_rivers = wtools_lib.configget(config, 'parameters', 'burn_rivers', 200, datatype='int') burn_connections = wtools_lib.configget(config, 'parameters', 'burn_connections', 100, datatype='int') burn_gauges = wtools_lib.configget(config, 'parameters', 'burn_gauges', 100, datatype='int') minorder = wtools_lib.configget(config, 'parameters', 'riverorder_min', 3, datatype='int') percentiles = np.array( config.get('parameters', 'statisticmaps', '0, 100').replace( ' ', '').split(','), dtype='float') # read the parameters for generating a temporary very high resolution grid if unit_clone == 'degree': cellsize_hr = wtools_lib.configget(config, 'parameters', 'highres_degree', 0.0005, datatype='float') elif (unit_clone == 'metre') or (unit_clone == 'meter'): cellsize_hr = wtools_lib.configget(config, 'parameters', 'highres_metre', 50, datatype='float') cols_hr = int((float(xmax)-float(xmin))/cellsize_hr + 2) rows_hr = int((float(ymax)-float(ymin))/cellsize_hr + 2) hr_trans = (float(xmin), cellsize_hr, float(0), float(ymax), 0, -cellsize_hr) clone_hr = os.path.join(options.destination, 'clone_highres.tif') # make a highres clone as well! wtools_lib.CreateTif(clone_hr, rows_hr, cols_hr, hr_trans, srs, 0) # read staticmap locations catchment_map = wtools_lib.configget(config, 'staticmaps', 'catchment', 'wflow_catchment.map') dem_map = wtools_lib.configget(config, 'staticmaps', 'dem', 'wflow_dem.map') demmax_map = wtools_lib.configget(config, 'staticmaps', 'demmax', 'wflow_demmax.map') demmin_map = wtools_lib.configget(config, 'staticmaps', 'demmin', 'wflow_demmin.map') gauges_map = wtools_lib.configget(config, 'staticmaps', 'gauges', 'wflow_gauges.map') landuse_map = wtools_lib.configget(config, 'staticmaps', 'landuse', 'wflow_landuse.map') ldd_map = wtools_lib.configget(config, 'staticmaps', 'ldd', 'wflow_ldd.map') river_map = wtools_lib.configget(config, 'staticmaps', 'river', 'wflow_river.map') outlet_map = wtools_lib.configget(config, 'staticmaps', 'outlet', 'wflow_outlet.map') riverlength_fact_map = wtools_lib.configget(config, 'staticmaps', 'riverlength_fact', 'wflow_riverlength_fact.map') soil_map = wtools_lib.configget(config, 'staticmaps', 'soil', 'wflow_soil.map') streamorder_map = wtools_lib.configget(config, 'staticmaps', 'streamorder', 'wflow_streamorder.map') subcatch_map = wtools_lib.configget(config, 'staticmaps', 'subcatch', 'wflow_subcatch.map') # read mask location (optional) masklayer = wtools_lib.configget(config, 'mask', 'masklayer', options.catchshp) # ???? empty = pcr.ifthen(ones == 0, pcr.scalar(0)) # TODO: check if extents are correct this way # TODO: check what the role of missing values is in zeros and ones (l. 123 in old code) # first add a missing value to dem_in ds = gdal.Open(options.dem_in, gdal.GA_Update) RasterBand = ds.GetRasterBand(1) fill_val = RasterBand.GetNoDataValue() if fill_val is None: RasterBand.SetNoDataValue(-9999) ds = None # reproject to clone map: see http://stackoverflow.com/questions/10454316/how-to-project-and-resample-a-grid-to-match-another-grid-with-gdal-python # resample DEM logger.info('Resampling dem from {:s} to {:s}'.format(os.path.abspath(options.dem_in), os.path.join(options.destination, dem_map))) gis.gdal_warp(options.dem_in, clone_map, os.path.join(options.destination, dem_map), format='PCRaster', gdal_interp=gdalconst.GRA_Average) # retrieve amount of rows and columns from clone # TODO: make windowstats applicable to source/target with different projections. This does not work yet. # retrieve srs from DEM try: srs_dem = wtools_lib.get_projection(options.dem_in) except: logger.warning('No projection found in DEM, assuming WGS 1984 lat long') srs_dem = osr.SpatialReference() srs_dem.ImportFromEPSG(4326) clone2dem_transform = osr.CoordinateTransformation(srs,srs_dem) #if srs.ExportToProj4() == srs_dem.ExportToProj4(): for percentile in percentiles: if percentile >= 100: logger.info('computing window maximum') percentile_dem = os.path.join(options.destination, 'wflow_dem_max.map') elif percentile <= 0: logger.info('computing window minimum') percentile_dem = os.path.join(options.destination, 'wflow_dem_min.map') else: logger.info('computing window {:d} percentile'.format(int(percentile))) percentile_dem = os.path.join(options.destination, 'wflow_dem_{:03d}.map'.format(int(percentile))) percentile_dem = os.path.join(options.destination, 'wflow_dem_{:03d}.map'.format(int(percentile))) stats = wtools_lib.windowstats(options.dem_in, len(yax), len(xax), trans, srs, percentile_dem, percentile, transform=clone2dem_transform,logger=logger) # else: # logger.warning('Projections of DEM and clone are different. DEM statistics for different projections is not yet implemented') """ # burn in rivers # first convert and clip the river shapefile # retrieve river shape projection, if not available assume EPSG:4326 file_att = os.path.splitext(os.path.basename(options.rivshp))[0] ds = ogr.Open(options.rivshp) lyr = ds.GetLayerByName(file_att) extent = lyr.GetExtent() extent_in = [extent[0], extent[2], extent[1], extent[3]] try: # get spatial reference from shapefile srs_rivshp = lyr.GetSpatialRef() logger.info('Projection in river shapefile is {:s}'.format(srs_rivshp.ExportToProj4())) except: logger.warning('No projection found in {:s}, assuming WGS 1984 lat-lon'.format(options.rivshp)) srs_rivshp = osr.SpatialReference() srs_rivshp.ImportFromEPSG(4326) rivprojshp = os.path.join(options.destination, 'rivshp_proj.shp') logger.info('Projecting and clipping {:s} to {:s}'.format(options.rivshp, rivprojshp)) # TODO: Line below takes a very long time to process, the bigger the shapefile, the more time. How do we deal with this? call(('ogr2ogr','-s_srs', srs_rivshp.ExportToProj4(),'-t_srs', srs.ExportToProj4(), '-clipsrc', '{:f}'.format(xmin), '{:f}'.format(ymin), '{:f}'.format(xmax), '{:f}'.format(ymax), rivprojshp, options.rivshp)) """ # TODO: BURNING!! # project catchment layer to projection of clone file_att = os.path.splitext(os.path.basename(options.catchshp))[0] print options.catchshp ds = ogr.Open(options.catchshp) lyr = ds.GetLayerByName(file_att) extent = lyr.GetExtent() extent_in = [extent[0], extent[2], extent[1], extent[3]] try: # get spatial reference from shapefile srs_catchshp = lyr.GetSpatialRef() logger.info('Projection in catchment shapefile is {:s}'.format(srs_catchshp.ExportToProj4())) except: logger.warning('No projection found in {:s}, assuming WGS 1984 lat-lon'.format(options.catchshp)) srs_catchshp = osr.SpatialReference() srs_catchshp.ImportFromEPSG(4326) catchprojshp = os.path.join(options.destination, 'catchshp_proj.shp') logger.info('Projecting {:s} to {:s}'.format(options.catchshp, catchprojshp)) call(('ogr2ogr','-s_srs', srs_catchshp.ExportToProj4(),'-t_srs', srs.ExportToProj4(), '-clipsrc', '{:f}'.format(xmin), '{:f}'.format(ymin), '{:f}'.format(xmax), '{:f}'.format(ymax), catchprojshp, options.catchshp)) # logger.info('Calculating ldd') ldddem = pcr.readmap(os.path.join(options.destination, dem_map)) ldd_select=pcr.lddcreate(ldddem, 1e35, 1e35, 1e35, 1e35) pcr.report(ldd_select, os.path.join(options.destination, 'wflow_ldd.map')) # compute stream order, identify river cells streamorder = pcr.ordinal(pcr.streamorder(ldd_select)) river = pcr.ifthen(streamorder >= pcr.ordinal(minorder), pcr.boolean(1)) # find the minimum value in the DEM and cover missing values with a river with this value. Effect is none!! so now left out! # mindem = int(np.min(pcr.pcr2numpy(pcr.ordinal(os.path.join(options.destination, dem_map)),9999999))) # dem_resample_map = pcr.cover(os.path.join(options.destination, dem_map), pcr.scalar(river)*0+mindem) # pcr.report(dem_resample_map, os.path.join(options.destination, dem_map)) pcr.report(streamorder, os.path.join(options.destination, streamorder_map)) pcr.report(river, os.path.join(options.destination, river_map)) # deal with your catchments if options.gaugeshp == None: logger.info('No gauges defined, using outlets instead') gauges = pcr.ordinal( pcr.uniqueid( pcr.boolean( pcr.ifthen(pcr.scalar(ldd_select)==5, pcr.boolean(1) ) ) ) ) pcr.report(gauges, os.path.join(options.destination, gauges_map)) # TODO: Add the gauge shape code from StaticMaps.py (line 454-489) # TODO: add river length map (see SticMaps.py, line 492-499) # report river length # make a high resolution empty map dem_hr_file = os.path.join(options.destination, 'dem_highres.tif') burn_hr_file = os.path.join(options.destination, 'burn_highres.tif') demburn_hr_file = os.path.join(options.destination, 'demburn_highres.map') riv_hr_file = os.path.join(options.destination, 'riv_highres.map') gis.gdal_warp(options.dem_in, clone_hr, dem_hr_file) # wtools_lib.CreateTif(riv_hr, rows_hr, cols_hr, hr_trans, srs, 0) file_att = os.path.splitext(os.path.basename(options.rivshp))[0] # open the shape layer ds = ogr.Open(options.rivshp) lyr = ds.GetLayerByName(file_att) gis.ogr_burn(lyr, clone_hr, -100, file_out=burn_hr_file, format='GTiff', gdal_type=gdal.GDT_Float32, fill_value=0) # read dem and burn values and add xax_hr, yax_hr, burn_hr, fill = gis.gdal_readmap(burn_hr_file, 'GTiff') burn_hr[burn_hr==fill] = 0 xax_hr, yax_hr, dem_hr, fill = gis.gdal_readmap(dem_hr_file, 'GTiff') dem_hr[dem_hr==fill] = np.nan demburn_hr = dem_hr + burn_hr demburn_hr[np.isnan(demburn_hr)] = -9999 gis.gdal_writemap(demburn_hr_file, 'PCRaster', xax_hr, yax_hr, demburn_hr, -9999.) pcr.setclone(demburn_hr_file) demburn_hr = pcr.readmap(demburn_hr_file) ldd_hr = pcr.lddcreate(demburn_hr, 1e35, 1e35, 1e35, 1e35) pcr.report(ldd_hr, os.path.join(options.destination, 'ldd_hr.map')) pcr.setglobaloption('unitcell') riv_hr = pcr.scalar(pcr.streamorder(ldd_hr) >= minorder)*pcr.downstreamdist(ldd_hr) pcr.report(riv_hr, riv_hr_file) pcr.setglobaloption('unittrue') pcr.setclone(clone_map) logger.info('Computing river length') #riverlength = wt.windowstats(riv_hr,clone_rows,clone_columns,clone_trans,srs_clone,resultdir,'frac',clone2dem_transform) riverlength = wtools_lib.windowstats(riv_hr_file, len(yax), len(xax), trans, srs, os.path.join(options.destination, riverlength_fact_map), stat='fact', logger=logger) # TODO: nothing happends with the river lengths yet. Need to decide how to use these # report outlet map pcr.report(pcr.ifthen(pcr.ordinal(ldd_select)==5, pcr.ordinal(1)), os.path.join(options.destination, outlet_map)) # report subcatchment map subcatchment = pcr.subcatchment(ldd_select, gauges) pcr.report(pcr.ordinal(subcatchment), os.path.join(options.destination, subcatch_map)) # Report land use map if options.landuse == None: logger.info('No land use map used. Preparing {:s} with only ones.'. format(os.path.join(options.destination, landuse_map))) pcr.report(pcr.nominal(ones), os.path.join(options.destination, landuse_map)) else: logger.info('Resampling land use from {:s} to {:s}'. format(os.path.abspath(options.landuse), os.path.join(options.destination, os.path.abspath(landuse_map)))) gis.gdal_warp(options.landuse, clone_map, os.path.join(options.destination, landuse_map), format='PCRaster', gdal_interp=gdalconst.GRA_Mode, gdal_type=gdalconst.GDT_Int32) # report soil map if options.soil == None: logger.info('No soil map used. Preparing {:s} with only ones.'. format(os.path.join(options.destination, soil_map))) pcr.report(pcr.nominal(ones), os.path.join(options.destination, soil_map)) else: logger.info('Resampling soil from {:s} to {:s}'. format(os.path.abspath(options.soil), os.path.join(options.destination, os.path.abspath(soil_map)))) gis.gdal_warp(options.soil, clone_map, os.path.join(options.destination, soil_map), format='PCRaster', gdal_interp=gdalconst.GRA_Mode, gdal_type=gdalconst.GDT_Int32) if options.lai == None: logger.info('No vegetation LAI maps used. Preparing default maps {:s} with only ones.'. format(os.path.join(options.destination, soil_map))) pcr.report(pcr.nominal(ones), os.path.join(options.destination, soil_map)) else: dest_lai = os.path.join(options.destination, 'clim') os.makedirs(dest_lai) for month in range(12): lai_in = os.path.join(options.lai, 'LAI00000.{:03d}'.format(month + 1)) lai_out = os.path.join(dest_lai, 'LAI00000.{:03d}'.format(month + 1)) logger.info('Resampling vegetation LAI from {:s} to {:s}'. format(os.path.abspath(lai_in), os.path.abspath(lai_out))) gis.gdal_warp(lai_in, clone_map, lai_out, format='PCRaster', gdal_interp=gdalconst.GRA_Bilinear, gdal_type=gdalconst.GDT_Float32) # report soil map if options.other_maps == None: logger.info('No other maps used. Skipping other maps.') else: logger.info('Resampling list of other maps...') for map_file in options.other_maps: map_name = os.path.split(map_file)[1] logger.info('Resampling a map from {:s} to {:s}'. format(os.path.abspath(map_file), os.path.join(options.destination, map_name))) gis.gdal_warp(map_file, clone_map, os.path.join(options.destination, map_name), format='PCRaster', gdal_interp=gdalconst.GRA_Mode, gdal_type=gdalconst.GDT_Float32) if options.clean: wtools_lib.DeleteList(glob.glob(os.path.join(options.destination, '*.xml')), logger=logger) wtools_lib.DeleteList(glob.glob(os.path.join(options.destination, 'clim', '*.xml')), logger=logger) wtools_lib.DeleteList(glob.glob(os.path.join(options.destination, '*highres*')), logger=logger)
def naturalLake( waterlevel, LakeLocs, LinkedLakeLocs, LakeArea, LakeThreshold, LakeStorFunc, LakeOutflowFunc, sh, hq, lake_b, lake_e, inflow, precip, pet, LakeAreasMap, JDOY, timestepsecs=86400, ): """ Run Natural Lake module to compute the new waterlevel and outflow. Solves lake water balance with linearisation and iteration procedure, for any rating and storage curve. For the case where storage curve is S = AH and Q=b(H-Ho)^2, uses the direct solution from the Modified Puls Approach (LISFLOOD). :ivar waterlevel: water level H in the lake :ivar LakeLocs: location of lake's outlet :ivar LinkedLakeLocs: ID of linked lakes :ivar LakeArea: total lake area :ivar LakeThreshold: water level threshold Ho under which outflow is zero :ivar LakeStorFunc: type of lake storage curve 1: S = AH 2: S = f(H) from lake data and interpolation :ivar LakeOutflowFunc: type of lake rating curve 1: Q = f(H) from lake data and interpolation 2: General Q = b(H - Ho)^e 3: Case of Puls Approach Q = b(H - Ho)^2 :ivar sh: data for storage curve :ivar hq: data for rating curve :ivar lake_b: rating curve coefficient :ivar lake_e: rating curve exponent :ivar inflow: inflow to the lake (surface runoff + river discharge + seepage) :ivar precip: precipitation map :ivar pet: PET map :ivar LakeAreasMap: lake extent map (for filtering P and PET) :ivar JDOY: Julian Day of Year to read storage/rating curve from data :ivar timestepsecs: model timestep in seconds :returns: waterlevel, outflow, prec_av, pet_av, storage """ mv = -999.0 LakeZeros = LakeArea * 0.0 waterlevel_start = waterlevel inflow = pcr.ifthen(pcr.boolean(LakeLocs), inflow) prec_av = pcr.ifthen(pcr.boolean(LakeLocs), pcr.areaaverage(precip, LakeAreasMap)) pet_av = pcr.ifthen(pcr.boolean(LakeLocs), pcr.areaaverage(pet, LakeAreasMap)) ### Modified Puls Approach (Burek et al., 2013, LISFLOOD) ### #ResOutflowFunc = 3 #Calculate lake factor and SI parameter LakeFactor = pcr.ifthenelse(LakeOutflowFunc == 3, LakeArea / (timestepsecs * (lake_b)**0.5), mv) storage_start = pcr.ifthenelse( LakeStorFunc == 1, LakeArea * waterlevel_start, lookupResFunc(LakeLocs, waterlevel_start, sh, "0-1"), ) SIFactor = pcr.ifthenelse( LakeOutflowFunc == 3, ((storage_start + (prec_av - pet_av) * LakeArea / 1000.0) / timestepsecs + inflow), mv) #Adjust SIFactor for ResThreshold != 0 SIFactorAdj = SIFactor - LakeArea * LakeThreshold / timestepsecs #Calculate the new lake outflow/waterlevel/storage outflow = pcr.ifthenelse( LakeOutflowFunc == 3, pcr.ifthenelse(SIFactorAdj > 0.0, (-LakeFactor + (LakeFactor**2 + 2 * SIFactorAdj)**0.5)**2, 0.0), LakeZeros) storage = pcr.ifthenelse(LakeOutflowFunc == 3, (SIFactor - outflow) * timestepsecs, LakeZeros) waterlevel = pcr.ifthenelse(LakeOutflowFunc == 3, storage / LakeArea, LakeZeros) ### Linearisation and iteration for specific storage/rating curves ### np_lakeoutflowfunc = pcr.pcr2numpy(LakeOutflowFunc, 0.0) if ((bool(np.isin(1, np.unique(np_lakeoutflowfunc)))) or (bool(np.isin(2, np.unique(np_lakeoutflowfunc))))): np_lakelocs = pcr.pcr2numpy(LakeLocs, 0.0) np_linkedlakelocs = pcr.pcr2numpy(LinkedLakeLocs, 0.0) waterlevel_loop = waterlevel_start _outflow = [] nr_loop = np.max([int(timestepsecs / 21600), 1]) for n in range(0, nr_loop): np_waterlevel = pcr.pcr2numpy(waterlevel_loop, np.nan) np_waterlevel_lower = np_waterlevel.copy() for val in np.unique(np_linkedlakelocs): if val > 0: np_waterlevel_lower[np_linkedlakelocs == val] = np_waterlevel[np.where( np_lakelocs == val)] diff_wl = np_waterlevel - np_waterlevel_lower diff_wl[np.isnan(diff_wl)] = mv np_waterlevel_lower[np.isnan(np_waterlevel_lower)] = mv pcr_diff_wl = pcr.numpy2pcr(pcr.Scalar, diff_wl, mv) pcr_wl_lower = pcr.numpy2pcr(pcr.Scalar, np_waterlevel_lower, mv) storage_start_loop = pcr.ifthenelse( LakeStorFunc == 1, LakeArea * waterlevel_loop, lookupResFunc(LakeLocs, waterlevel_loop, sh, "0-1"), ) outflow_loop = pcr.ifthenelse( LakeOutflowFunc == 1, lookupResRegMatr(LakeLocs, waterlevel_loop, hq, JDOY), pcr.ifthenelse( pcr_diff_wl >= 0, pcr.max(lake_b * (waterlevel_loop - LakeThreshold)**lake_e, 0), pcr.min( -1 * lake_b * (pcr_wl_lower - LakeThreshold)**lake_e, 0), ), ) np_outflow = pcr.pcr2numpy(outflow_loop, np.nan) np_outflow_linked = np_lakelocs * 0.0 with np.errstate(invalid="ignore"): if np_outflow[np_outflow < 0] is not None: np_outflow_linked[np.in1d( np_lakelocs, np_linkedlakelocs[np_outflow < 0]).reshape( np_linkedlakelocs.shape)] = np_outflow[ np_outflow < 0] outflow_linked = pcr.numpy2pcr(pcr.Scalar, np_outflow_linked, 0.0) fl_nr_loop = float(nr_loop) storage_loop = ( storage_start_loop + (inflow * timestepsecs / fl_nr_loop) + (prec_av / fl_nr_loop / 1000.0) * LakeArea - (pet_av / fl_nr_loop / 1000.0) * LakeArea - (pcr.cover(outflow_loop, 0.0) * timestepsecs / fl_nr_loop) + (pcr.cover(outflow_linked, 0.0) * timestepsecs / fl_nr_loop)) waterlevel_loop = pcr.ifthenelse( LakeStorFunc == 1, waterlevel_loop + (storage_loop - storage_start_loop) / LakeArea, lookupResFunc(LakeLocs, storage_loop, sh, "1-0"), ) np_outflow_nz = np_outflow * 0.0 with np.errstate(invalid="ignore"): np_outflow_nz[np_outflow > 0] = np_outflow[np_outflow > 0] _outflow.append(np_outflow_nz) outflow_av_temp = np.average(_outflow, 0) outflow_av_temp[np.isnan(outflow_av_temp)] = mv outflow_av = pcr.numpy2pcr(pcr.Scalar, outflow_av_temp, mv) #Add the discharge/waterlevel/storage from the loop to the one from puls approach outflow = pcr.ifthenelse(LakeOutflowFunc == 3, outflow, outflow_av) waterlevel = pcr.ifthenelse(LakeOutflowFunc == 3, waterlevel, waterlevel_loop) storage = pcr.ifthenelse(LakeOutflowFunc == 3, storage, storage_loop) return waterlevel, outflow, prec_av, pet_av, storage
vos.cmd_line(cmd, using_subprocess = False) clone_map_file = "clone_low_resolution_30min.map" # - set the clone and landmask map pcr.setclone(clone_map_file) landmask = pcr.boolean(1.0) # # save numpy arrays for i_file in range(0, len(file_names)): # rename 5 arc-min file file_name = file_names[i_file] cmd = 'mv ' + file_name + " " + file_name + ".5min.map" vos.cmd_line(cmd, using_subprocess = False) # report it to pcraster files print(file_name) os.system('pwd') pcr.report(pcr.numpy2pcr(pcr.Scalar, extreme_value_30min[file_name], vos.MV), file_name) # # prepare ldd at 30 arcmin resolution (we need this, only for the compatibility with the downscaling script) # - rename ldd cmd = 'mv resampled_low_resolution_ldd.map resampled_low_resolution_ldd.5min.map' # - using 30 arcmin ldd ldd_map_low_resolution_file_name = "/projects/0/dfguu/data/hydroworld/PCRGLOBWB20/input30min/routing/lddsound_30min.map" ldd_map_low_resolution = vos.readPCRmapClone(ldd_map_low_resolution_file_name, \ clone_map_file, \ tmp_folder, \ None, True, None, False) ldd_map_low_resolution = pcr.lddrepair(pcr.ldd(ldd_map_low_resolution)) ldd_map_low_resolution = pcr.lddrepair(ldd_map_low_resolution) pcr.report(ldd_map_low_resolution, "resampled_low_resolution_ldd.map")
def main(): ### Read input arguments ##### parser = OptionParser() usage = "usage: %prog [options]" parser = OptionParser(usage=usage) parser.add_option('-q', '--quiet', dest='verbose', default=True, action='store_false', help='do not print status messages to stdout') parser.add_option('-i', '--ini', dest='inifile', default='hand_contour_inun.ini', nargs=1, help='ini configuration file') parser.add_option('-f', '--flood_map', nargs=1, dest='flood_map', help='Flood map file (NetCDF point time series file') parser.add_option('-v', '--flood_variable', nargs=1, dest='flood_variable', default='water_level', help='variable name of flood water level') parser.add_option('-b', '--bankfull_map', dest='bankfull_map', default='', help='Map containing bank full level (is subtracted from flood map, in NetCDF)') parser.add_option('-c', '--catchment', dest='catchment_strahler', default=7, type='int', help='Strahler order threshold >= are selected as catchment boundaries') parser.add_option('-t', '--time', dest='time', default='', help='time in YYYYMMDDHHMMSS, overrides time in NetCDF input if set') # parser.add_option('-s', '--hand_strahler', # dest='hand_strahler', default=7, type='int', # help='Strahler order threshold >= selected as riverine') parser.add_option('-m', '--max_strahler', dest = 'max_strahler', default=1000, type='int', help='Maximum Strahler order to loop over') parser.add_option('-d', '--destination', dest='dest_path', default='inun', help='Destination path') parser.add_option('-H', '--hand_file_prefix', dest='hand_file_prefix', default='', help='optional HAND file prefix of already generated HAND files') parser.add_option('-n', '--neg_HAND', dest='neg_HAND', default=0, type='int', help='if set to 1, allow for negative HAND values in HAND maps') (options, args) = parser.parse_args() if not os.path.exists(options.inifile): print 'path to ini file cannot be found' sys.exit(1) options.dest_path = os.path.abspath(options.dest_path) if not(os.path.isdir(options.dest_path)): os.makedirs(options.dest_path) # set up the logger flood_name = os.path.split(options.flood_map)[1].split('.')[0] # case_name = 'inun_{:s}_hand_{:02d}_catch_{:02d}'.format(flood_name, options.hand_strahler, options.catchment_strahler) case_name = 'inun_{:s}_catch_{:02d}'.format(flood_name, options.catchment_strahler) logfilename = os.path.join(options.dest_path, 'hand_contour_inun.log') logger, ch = inun_lib.setlogger(logfilename, 'HAND_INUN', options.verbose) logger.info('$Id: $') logger.info('Flood map: {:s}'.format(options.flood_map)) logger.info('Bank full map: {:s}'.format(options.bankfull_map)) logger.info('Destination path: {:s}'.format(options.dest_path)) # read out ini file ### READ CONFIG FILE # open config-file config = inun_lib.open_conf(options.inifile) # read settings options.dem_file = inun_lib.configget(config, 'HighResMaps', 'dem_file', True) options.ldd_file = inun_lib.configget(config, 'HighResMaps', 'ldd_file', True) options.stream_file = inun_lib.configget(config, 'HighResMaps', 'stream_file', True) options.riv_length_fact_file = inun_lib.configget(config, 'wflowResMaps', 'riv_length_fact_file', True) options.ldd_wflow = inun_lib.configget(config, 'wflowResMaps', 'ldd_wflow', True) options.riv_width_file = inun_lib.configget(config, 'wflowResMaps', 'riv_width_file', True) options.file_format = inun_lib.configget(config, 'file_settings', 'file_format', 0, datatype='int') options.out_format = inun_lib.configget(config, 'file_settings', 'out_format', 0, datatype='int') options.latlon = inun_lib.configget(config, 'file_settings', 'latlon', 0, datatype='int') options.x_tile = inun_lib.configget(config, 'tiling', 'x_tile', 10000, datatype='int') options.y_tile = inun_lib.configget(config, 'tiling', 'y_tile', 10000, datatype='int') options.x_overlap = inun_lib.configget(config, 'tiling', 'x_overlap', 1000, datatype='int') options.y_overlap = inun_lib.configget(config, 'tiling', 'y_overlap', 1000, datatype='int') options.iterations = inun_lib.configget(config, 'inundation', 'iterations', 20, datatype='int') options.initial_level = inun_lib.configget(config, 'inundation', 'initial_level', 32., datatype='float') options.flood_volume_type = inun_lib.configget(config, 'inundation', 'flood_volume_type', 0, datatype='int') # options.area_multiplier = inun_lib.configget(config, 'inundation', # 'area_multiplier', 1., datatype='float') logger.info('DEM file: {:s}'.format(options.dem_file)) logger.info('LDD file: {:s}'.format(options.ldd_file)) logger.info('Columns per tile: {:d}'.format(options.x_tile)) logger.info('Rows per tile: {:d}'.format(options.y_tile)) logger.info('Columns overlap: {:d}'.format(options.x_overlap)) logger.info('Rows overlap: {:d}'.format(options.y_overlap)) metadata_global = {} # add metadata from the section [metadata] meta_keys = config.options('metadata_global') for key in meta_keys: metadata_global[key] = config.get('metadata_global', key) # add a number of metadata variables that are mandatory metadata_global['config_file'] = os.path.abspath(options.inifile) metadata_var = {} metadata_var['units'] = 'm' metadata_var['standard_name'] = 'water_surface_height_above_reference_datum' metadata_var['long_name'] = 'flooding' metadata_var['comment'] = 'water_surface_reference_datum_altitude is given in file {:s}'.format(options.dem_file) if not os.path.exists(options.dem_file): logger.error('path to dem file {:s} cannot be found'.format(options.dem_file)) sys.exit(1) if not os.path.exists(options.ldd_file): logger.error('path to ldd file {:s} cannot be found'.format(options.ldd_file)) sys.exit(1) # Read extent from a GDAL compatible file try: extent = inun_lib.get_gdal_extent(options.dem_file) except: msg = 'Input file {:s} not a gdal compatible file'.format(options.dem_file) inun_lib.close_with_error(logger, ch, msg) sys.exit(1) try: x, y = inun_lib.get_gdal_axes(options.dem_file, logging=logger) srs = inun_lib.get_gdal_projection(options.dem_file, logging=logger) except: msg = 'Input file {:s} not a gdal compatible file'.format(options.dem_file) inun_lib.close_with_error(logger, ch, msg) sys.exit(1) # read history from flood file if options.file_format == 0: a = nc.Dataset(options.flood_map, 'r') metadata_global['history'] = 'Created by: $Id: $, boundary conditions from {:s},\nhistory: {:s}'.format(os.path.abspath(options.flood_map), a.history) a.close() else: metadata_global['history'] = 'Created by: $Id: $, boundary conditions from {:s},\nhistory: {:s}'.format(os.path.abspath(options.flood_map), 'PCRaster file, no history') # first write subcatch maps and hand maps ############### TODO ###### # setup a HAND file for each strahler order max_s = inun_lib.define_max_strahler(options.stream_file, logging=logger) stream_max = np.minimum(max_s, options.max_strahler) for hand_strahler in range(options.catchment_strahler, stream_max + 1, 1): dem_name = os.path.split(options.dem_file)[1].split('.')[0] if os.path.isfile('{:s}_{:02d}.tif'.format(options.hand_file_prefix, hand_strahler)): hand_file = '{:s}_{:02d}.tif'.format(options.hand_file_prefix, hand_strahler) else: logger.info('No HAND files with HAND prefix were found, checking {:s}_hand_strahler_{:02d}.tif'.format(dem_name, hand_strahler)) hand_file = os.path.join(options.dest_path, '{:s}_hand_strahler_{:02d}.tif'.format(dem_name, hand_strahler)) if not(os.path.isfile(hand_file)): # hand file does not exist yet! Generate it, otherwise skip! logger.info('HAND file {:s} not found, start setting up...please wait...'.format(hand_file)) hand_file_tmp = os.path.join(options.dest_path, '{:s}_hand_strahler_{:02d}.tif.tmp'.format(dem_name, hand_strahler)) ds_hand, band_hand = inun_lib.prepare_gdal(hand_file_tmp, x, y, logging=logger, srs=srs) # band_hand = ds_hand.GetRasterBand(1) # Open terrain data for reading ds_dem, rasterband_dem = inun_lib.get_gdal_rasterband(options.dem_file) ds_ldd, rasterband_ldd = inun_lib.get_gdal_rasterband(options.ldd_file) ds_stream, rasterband_stream = inun_lib.get_gdal_rasterband(options.stream_file) n = 0 for x_loop in range(0, len(x), options.x_tile): x_start = np.maximum(x_loop, 0) x_end = np.minimum(x_loop + options.x_tile, len(x)) # determine actual overlap for cutting for y_loop in range(0, len(y), options.y_tile): x_overlap_min = x_start - np.maximum(x_start - options.x_overlap, 0) x_overlap_max = np.minimum(x_end + options.x_overlap, len(x)) - x_end n += 1 # print('tile {:001d}:'.format(n)) y_start = np.maximum(y_loop, 0) y_end = np.minimum(y_loop + options.y_tile, len(y)) y_overlap_min = y_start - np.maximum(y_start - options.y_overlap, 0) y_overlap_max = np.minimum(y_end + options.y_overlap, len(y)) - y_end # cut out DEM logger.debug('Computing HAND for xmin: {:d} xmax: {:d} ymin {:d} ymax {:d}'.format(x_start, x_end,y_start, y_end)) terrain = rasterband_dem.ReadAsArray(x_start - x_overlap_min, y_start - y_overlap_min, (x_end + x_overlap_max) - (x_start - x_overlap_min), (y_end + y_overlap_max) - (y_start - y_overlap_min) ) drainage = rasterband_ldd.ReadAsArray(x_start - x_overlap_min, y_start - y_overlap_min, (x_end + x_overlap_max) - (x_start - x_overlap_min), (y_end + y_overlap_max) - (y_start - y_overlap_min) ) stream = rasterband_stream.ReadAsArray(x_start - x_overlap_min, y_start - y_overlap_min, (x_end + x_overlap_max) - (x_start - x_overlap_min), (y_end + y_overlap_max) - (y_start - y_overlap_min) ) # write to temporary file terrain_temp_file = os.path.join(options.dest_path, 'terrain_temp.map') drainage_temp_file = os.path.join(options.dest_path, 'drainage_temp.map') stream_temp_file = os.path.join(options.dest_path, 'stream_temp.map') if rasterband_dem.GetNoDataValue() is not None: inun_lib.gdal_writemap(terrain_temp_file, 'PCRaster', np.arange(0, terrain.shape[1]), np.arange(0, terrain.shape[0]), terrain, rasterband_dem.GetNoDataValue(), gdal_type=gdal.GDT_Float32, logging=logger) else: # in case no nodata value is found logger.warning('No nodata value found in {:s}. assuming -9999'.format(options.dem_file)) inun_lib.gdal_writemap(terrain_temp_file, 'PCRaster', np.arange(0, terrain.shape[1]), np.arange(0, terrain.shape[0]), terrain, -9999., gdal_type=gdal.GDT_Float32, logging=logger) inun_lib.gdal_writemap(drainage_temp_file, 'PCRaster', np.arange(0, terrain.shape[1]), np.arange(0, terrain.shape[0]), drainage, rasterband_ldd.GetNoDataValue(), gdal_type=gdal.GDT_Int32, logging=logger) inun_lib.gdal_writemap(stream_temp_file, 'PCRaster', np.arange(0, terrain.shape[1]), np.arange(0, terrain.shape[0]), stream, rasterband_ldd.GetNoDataValue(), gdal_type=gdal.GDT_Int32, logging=logger) # read as pcr objects pcr.setclone(terrain_temp_file) terrain_pcr = pcr.readmap(terrain_temp_file) drainage_pcr = pcr.lddrepair(pcr.ldd(pcr.readmap(drainage_temp_file))) # convert to ldd type map stream_pcr = pcr.scalar(pcr.readmap(stream_temp_file)) # convert to ldd type map #check if the highest stream order of the tile is below the hand_strahler # if the highest stream order of the tile is smaller than hand_strahler, than DEM values are taken instead of HAND values. max_stream_tile = inun_lib.define_max_strahler(stream_temp_file, logging=logger) if max_stream_tile < hand_strahler: hand_pcr = terrain_pcr logger.info('For this tile, DEM values are used instead of HAND because there is no stream order larger than {:02d}'.format(hand_strahler)) else: # compute streams stream_ge, subcatch = inun_lib.subcatch_stream(drainage_pcr, hand_strahler, stream=stream_pcr) # generate streams # compute basins stream_ge_dummy, subcatch = inun_lib.subcatch_stream(drainage_pcr, options.catchment_strahler, stream=stream_pcr) # generate streams basin = pcr.boolean(subcatch) hand_pcr, dist_pcr = inun_lib.derive_HAND(terrain_pcr, drainage_pcr, 3000, rivers=pcr.boolean(stream_ge), basin=basin, neg_HAND=options.neg_HAND) # convert to numpy hand = pcr.pcr2numpy(hand_pcr, -9999.) # cut relevant part if y_overlap_max == 0: y_overlap_max = -hand.shape[0] if x_overlap_max == 0: x_overlap_max = -hand.shape[1] hand_cut = hand[0+y_overlap_min:-y_overlap_max, 0+x_overlap_min:-x_overlap_max] band_hand.WriteArray(hand_cut, x_start, y_start) os.unlink(terrain_temp_file) os.unlink(drainage_temp_file) os.unlink(stream_temp_file) band_hand.FlushCache() ds_dem = None ds_ldd = None ds_stream = None band_hand.SetNoDataValue(-9999.) ds_hand = None logger.info('Finalizing {:s}'.format(hand_file)) # rename temporary file to final hand file os.rename(hand_file_tmp, hand_file) else: logger.info('HAND file {:s} already exists...skipping...'.format(hand_file)) ##################################################################################### # HAND file has now been prepared, moving to flood mapping part # ##################################################################################### # set the clone pcr.setclone(options.ldd_wflow) # read wflow ldd as pcraster object ldd_pcr = pcr.readmap(options.ldd_wflow) xax, yax, riv_width, fill_value = inun_lib.gdal_readmap(options.riv_width_file, 'GTiff', logging=logger) # determine cell length in meters using ldd_pcr as clone (if latlon=True, values are converted to m2 x_res, y_res, reallength_wflow = pcrut.detRealCellLength(pcr.scalar(ldd_pcr), not(bool(options.latlon))) cell_surface_wflow = pcr.pcr2numpy(x_res * y_res, 0) if options.flood_volume_type == 0: # load the staticmaps needed to estimate volumes across all # xax, yax, riv_length, fill_value = inun_lib.gdal_readmap(options.riv_length_file, 'GTiff', logging=logger) # riv_length = np.ma.masked_where(riv_length==fill_value, riv_length) xax, yax, riv_width, fill_value = inun_lib.gdal_readmap(options.riv_width_file, 'GTiff', logging=logger) riv_width[riv_width == fill_value] = 0 # read river length factor file (multiplier) xax, yax, riv_length_fact, fill_value = inun_lib.gdal_readmap(options.riv_length_fact_file, 'GTiff', logging=logger) riv_length_fact = np.ma.masked_where(riv_length_fact==fill_value, riv_length_fact) drain_length = wflow_lib.detdrainlength(ldd_pcr, x_res, y_res) # compute river length in each cell riv_length = pcr.pcr2numpy(drain_length, 0) * riv_length_fact # riv_length_pcr = pcr.numpy2pcr(pcr.Scalar, riv_length, 0) flood_folder = os.path.join(options.dest_path, case_name) flood_vol_map = os.path.join(flood_folder, '{:s}_vol.tif'.format(os.path.split(options.flood_map)[1].split('.')[0])) if not(os.path.isdir(flood_folder)): os.makedirs(flood_folder) if options.out_format == 0: inun_file_tmp = os.path.join(flood_folder, '{:s}.tif.tmp'.format(case_name)) inun_file = os.path.join(flood_folder, '{:s}.tif'.format(case_name)) else: inun_file_tmp = os.path.join(flood_folder, '{:s}.nc.tmp'.format(case_name)) inun_file = os.path.join(flood_folder, '{:s}.nc'.format(case_name)) hand_temp_file = os.path.join(flood_folder, 'hand_temp.map') drainage_temp_file = os.path.join(flood_folder, 'drainage_temp.map') stream_temp_file = os.path.join(flood_folder, 'stream_temp.map') flood_vol_temp_file = os.path.join(flood_folder, 'flood_warp_temp.tif') # load the data with river levels and compute the volumes if options.file_format == 0: # assume we need the maximum value in a NetCDF time series grid logger.info('Reading flood from {:s} NetCDF file'.format(options.flood_map)) a = nc.Dataset(options.flood_map, 'r') if options.latlon == 0: xax = a.variables['x'][:] yax = a.variables['y'][:] else: xax = a.variables['lon'][:] yax = a.variables['lat'][:] if options.time == '': time_list = nc.num2date(a.variables['time'][:], units = a.variables['time'].units, calendar=a.variables['time'].calendar) time = [time_list[len(time_list)/2]] else: time = [dt.datetime.strptime(options.time, '%Y%m%d%H%M%S')] flood_series = a.variables[options.flood_variable][:] flood_data = flood_series.max(axis=0) if np.ma.is_masked(flood_data): flood = flood_data.data flood[flood_data.mask] = 0 if yax[-1] > yax[0]: yax = np.flipud(yax) flood = np.flipud(flood) a.close() elif options.file_format == 1: logger.info('Reading flood from {:s} PCRaster file'.format(options.flood_map)) xax, yax, flood, flood_fill_value = inun_lib.gdal_readmap(options.flood_map, 'PCRaster', logging=logger) flood = np.ma.masked_equal(flood, flood_fill_value) if options.time == '': options.time = '20000101000000' time = [dt.datetime.strptime(options.time, '%Y%m%d%H%M%S')] flood[flood==flood_fill_value] = 0. # load the bankfull depths if options.bankfull_map == '': bankfull = np.zeros(flood.shape) else: if options.file_format == 0: logger.info('Reading bankfull from {:s} NetCDF file'.format(options.bankfull_map)) a = nc.Dataset(options.bankfull_map, 'r') xax = a.variables['x'][:] yax = a.variables['y'][:] bankfull_series = a.variables[options.flood_variable][:] bankfull_data = bankfull_series.max(axis=0) if np.ma.is_masked(bankfull_data): bankfull = bankfull_data.data bankfull[bankfull_data.mask] = 0 if yax[-1] > yax[0]: yax = np.flipud(yax) bankfull = np.flipud(bankfull) a.close() elif options.file_format == 1: logger.info('Reading bankfull from {:s} PCRaster file'.format(options.bankfull_map)) xax, yax, bankfull, bankfull_fill_value = inun_lib.gdal_readmap(options.bankfull_map, 'PCRaster', logging=logger) bankfull = np.ma.masked_equal(bankfull, bankfull_fill_value) # flood = bankfull*2 # res_x = 2000 # res_y = 2000 # subtract the bankfull water level to get flood levels (above bankfull) flood_vol = np.maximum(flood-bankfull, 0) if options.flood_volume_type == 0: flood_vol_m = riv_length*riv_width*flood_vol/cell_surface_wflow # volume expressed in meters water disc flood_vol_m_pcr = pcr.numpy2pcr(pcr.Scalar, flood_vol_m, 0) else: flood_vol_m = flood_vol/cell_surface_wflow flood_vol_m_data = flood_vol_m.data flood_vol_m_data[flood_vol_m.mask] = -999. logger.info('Saving water layer map to {:s}'.format(flood_vol_map)) # write to a tiff file inun_lib.gdal_writemap(flood_vol_map, 'GTiff', xax, yax, np.maximum(flood_vol_m_data, 0), -999., logging=logger) # this is placed later in the hand loop # ds_hand, rasterband_hand = inun_lib.get_gdal_rasterband(hand_file) ds_ldd, rasterband_ldd = inun_lib.get_gdal_rasterband(options.ldd_file) ds_stream, rasterband_stream = inun_lib.get_gdal_rasterband(options.stream_file) logger.info('Preparing flood map in {:s} ...please wait...'.format(inun_file)) if options.out_format == 0: ds_inun, band_inun = inun_lib.prepare_gdal(inun_file_tmp, x, y, logging=logger, srs=srs) # band_inun = ds_inun.GetRasterBand(1) else: ds_inun, band_inun = inun_lib.prepare_nc(inun_file_tmp, time, x, np.flipud(y), metadata=metadata_global, metadata_var=metadata_var, logging=logger) # loop over all the tiles n = 0 for x_loop in range(0, len(x), options.x_tile): x_start = np.maximum(x_loop, 0) x_end = np.minimum(x_loop + options.x_tile, len(x)) # determine actual overlap for cutting for y_loop in range(0, len(y), options.y_tile): x_overlap_min = x_start - np.maximum(x_start - options.x_overlap, 0) x_overlap_max = np.minimum(x_end + options.x_overlap, len(x)) - x_end n += 1 # print('tile {:001d}:'.format(n)) y_start = np.maximum(y_loop, 0) y_end = np.minimum(y_loop + options.y_tile, len(y)) y_overlap_min = y_start - np.maximum(y_start - options.y_overlap, 0) y_overlap_max = np.minimum(y_end + options.y_overlap, len(y)) - y_end x_tile_ax = x[x_start - x_overlap_min:x_end + x_overlap_max] y_tile_ax = y[y_start - y_overlap_min:y_end + y_overlap_max] # cut out DEM logger.debug('handling xmin: {:d} xmax: {:d} ymin {:d} ymax {:d}'.format(x_start, x_end, y_start, y_end)) drainage = rasterband_ldd.ReadAsArray(x_start - x_overlap_min, y_start - y_overlap_min, (x_end + x_overlap_max) - (x_start - x_overlap_min), (y_end + y_overlap_max) - (y_start - y_overlap_min) ) stream = rasterband_stream.ReadAsArray(x_start - x_overlap_min, y_start - y_overlap_min, (x_end + x_overlap_max) - (x_start - x_overlap_min), (y_end + y_overlap_max) - (y_start - y_overlap_min) ) # stream_max = np.minimum(stream.max(), options.max_strahler) inun_lib.gdal_writemap(drainage_temp_file, 'PCRaster', x_tile_ax, y_tile_ax, drainage, rasterband_ldd.GetNoDataValue(), gdal_type=gdal.GDT_Int32, logging=logger) inun_lib.gdal_writemap(stream_temp_file, 'PCRaster', x_tile_ax, y_tile_ax, stream, rasterband_stream.GetNoDataValue(), gdal_type=gdal.GDT_Int32, logging=logger) # read as pcr objects pcr.setclone(stream_temp_file) drainage_pcr = pcr.lddrepair(pcr.ldd(pcr.readmap(drainage_temp_file))) # convert to ldd type map stream_pcr = pcr.scalar(pcr.readmap(stream_temp_file)) # convert to ldd type map # warp of flood volume to inundation resolution inun_lib.gdal_warp(flood_vol_map, stream_temp_file, flood_vol_temp_file, gdal_interp=gdalconst.GRA_NearestNeighbour) # , x_tile_ax, y_tile_ax, flood_meter, fill_value = inun_lib.gdal_readmap(flood_vol_temp_file, 'GTiff', logging=logger) # make sure that the option unittrue is on !! (if unitcell was is used in another function) x_res_tile, y_res_tile, reallength = pcrut.detRealCellLength(pcr.scalar(stream_pcr), not(bool(options.latlon))) cell_surface_tile = pcr.pcr2numpy(x_res_tile * y_res_tile, 0) # convert meter depth to volume [m3] flood_vol = pcr.numpy2pcr(pcr.Scalar, flood_meter*cell_surface_tile, fill_value) # first prepare a basin map, belonging to the lowest order we are looking at inundation_pcr = pcr.scalar(stream_pcr) * 0 for hand_strahler in range(options.catchment_strahler, stream_max + 1, 1): # hand_temp_file = os.path.join(flood_folder, 'hand_temp.map') if os.path.isfile(os.path.join(options.dest_path, '{:s}_hand_strahler_{:02d}.tif'.format(dem_name, hand_strahler))): hand_file = os.path.join(options.dest_path, '{:s}_hand_strahler_{:02d}.tif'.format(dem_name, hand_strahler)) else: hand_file = '{:s}_{:02d}.tif'.format(options.hand_file_prefix, hand_strahler) ds_hand, rasterband_hand = inun_lib.get_gdal_rasterband(hand_file) hand = rasterband_hand.ReadAsArray(x_start - x_overlap_min, y_start - y_overlap_min, (x_end + x_overlap_max) - (x_start - x_overlap_min), (y_end + y_overlap_max) - (y_start - y_overlap_min) ) print('len x-ax: {:d} len y-ax {:d} x-shape {:d} y-shape {:d}'.format(len(x_tile_ax), len(y_tile_ax), hand.shape[1], hand.shape[0])) inun_lib.gdal_writemap(hand_temp_file, 'PCRaster', x_tile_ax, y_tile_ax, hand, rasterband_hand.GetNoDataValue(), gdal_type=gdal.GDT_Float32, logging=logger) hand_pcr = pcr.readmap(hand_temp_file) stream_ge_hand, subcatch_hand = inun_lib.subcatch_stream(drainage_pcr, options.catchment_strahler, stream=stream_pcr) # stream_ge_hand, subcatch_hand = inun_lib.subcatch_stream(drainage_pcr, hand_strahler, stream=stream_pcr) stream_ge, subcatch = inun_lib.subcatch_stream(drainage_pcr, options.catchment_strahler, stream=stream_pcr, basin=pcr.boolean(pcr.cover(subcatch_hand, 0)), assign_existing=True, min_strahler=hand_strahler, max_strahler=hand_strahler) # generate subcatchments, only within basin for HAND flood_vol_strahler = pcr.ifthenelse(pcr.boolean(pcr.cover(subcatch, 0)), flood_vol, 0) # mask the flood volume map with the created subcatch map for strahler order = hand_strahler inundation_pcr_step = inun_lib.volume_spread(drainage_pcr, hand_pcr, pcr.subcatchment(drainage_pcr, subcatch), # to make sure backwater effects can occur from higher order rivers to lower order rivers flood_vol_strahler, volume_thres=0., iterations=options.iterations, cell_surface=pcr.numpy2pcr(pcr.Scalar, cell_surface_tile, -9999), logging=logger, order=hand_strahler, neg_HAND=options.neg_HAND) # 1166400000. # use maximum value of inundation_pcr_step and new inundation for higher strahler order inundation_pcr = pcr.max(inundation_pcr, inundation_pcr_step) inundation = pcr.pcr2numpy(inundation_pcr, -9999.) # cut relevant part if y_overlap_max == 0: y_overlap_max = -inundation.shape[0] if x_overlap_max == 0: x_overlap_max = -inundation.shape[1] inundation_cut = inundation[0+y_overlap_min:-y_overlap_max, 0+x_overlap_min:-x_overlap_max] # inundation_cut if options.out_format == 0: band_inun.WriteArray(inundation_cut, x_start, y_start) band_inun.FlushCache() else: # with netCDF, data is up-side-down. inun_lib.write_tile_nc(band_inun, inundation_cut, x_start, y_start) # clean up os.unlink(flood_vol_temp_file) os.unlink(drainage_temp_file) os.unlink(hand_temp_file) os.unlink(stream_temp_file) #also remove temp stream file from output folder # if n == 35: # band_inun.SetNoDataValue(-9999.) # ds_inun = None # sys.exit(0) # os.unlink(flood_vol_map) logger.info('Finalizing {:s}'.format(inun_file)) # add the metadata to the file and band # band_inun.SetNoDataValue(-9999.) # ds_inun.SetMetadata(metadata_global) # band_inun.SetMetadata(metadata_var) if options.out_format == 0: ds_inun = None ds_hand = None else: ds_inun.close() ds_ldd = None # rename temporary file to final hand file if os.path.isfile(inun_file): # remove an old result if available os.unlink(inun_file) os.rename(inun_file_tmp, inun_file) logger.info('Done! Thank you for using hand_contour_inun.py') logger, ch = inun_lib.closeLogger(logger, ch) del logger, ch sys.exit(0)
def test_numpy2pcr(self): nrRows, nrCols, cellSize = 3, 2, 1.0 west, north = 0.0, 0.0 pcraster.setclone(nrRows, nrCols, cellSize, west, north) # Values in array must fit the value scale of the raster exactly. This # will be checked. # Valid boolean values are: 0, 1, missing_value. # Valid ldd values are: 1, 2, 3, 4, 5, 6, 7, 8, 9, missing_value. # Valid nominal values are: [-2^31 + 1, 2^31], missing_value. # Valid ordinal values are: [-2^31 + 1, 2^31], missing_value. # Valid scalar values are: All 32 bit float values. # Valid directional values are: All 32 bit float values. # bool_min = 0 # bool_max = 1 int8_min = numpy.iinfo(numpy.int8).min int8_max = numpy.iinfo(numpy.int8).max int32_min = numpy.iinfo(numpy.int32).min int32_max = numpy.iinfo(numpy.int32).max int64_min = numpy.iinfo(numpy.int64).min int64_max = numpy.iinfo(numpy.int64).max # bool -> Boolean (uint8) raster = pcraster.numpy2pcr(pcraster.Boolean, numpy.array([ [1, 1], [0, 5], [1, 1]], numpy.bool), 5) self.assertEqual(pcraster.cellvalue(raster, 1, 1), (True, True)) self.assertEqual(pcraster.cellvalue(raster, 2, 1), (False, True)) # It is not possible to create a bool array with other values than # 0 and 1. Passing 5 as missing value has no effect. self.assertEqual(pcraster.cellvalue(raster, 2, 2), (True, True)) # int8 -> Boolean (uint8) raster = pcraster.numpy2pcr(pcraster.Boolean, numpy.array([ [1, 1], [0, 5], [1, 1]], numpy.int8), 5) self.assertEqual(pcraster.cellvalue(raster, 1, 1), (True, True)) self.assertEqual(pcraster.cellvalue(raster, 2, 1), (False, True)) self.assertEqual(pcraster.cellvalue(raster, 2, 2)[1], False) with self.assertRaises(Exception) as context_manager: raster = pcraster.numpy2pcr(pcraster.Boolean, numpy.array([ [1, 1], [9, 5], [1, 1]], numpy.int8), 5) self.assertEqual(str(context_manager.exception), "Incorrect value 9 at input array [1][0] for Boolean map") # int8 -> Ldd (uint8) raster = pcraster.numpy2pcr(pcraster.Ldd, numpy.array([ [1, 2], [5, 15], [8, 9]], numpy.int8), 15) self.assertEqual(pcraster.cellvalue(raster, 1, 1), (1, True)) self.assertEqual(pcraster.cellvalue(raster, 2, 1), (5, True)) self.assertEqual(pcraster.cellvalue(raster, 2, 2)[1], False) with self.assertRaises(Exception) as context_manager: raster = pcraster.numpy2pcr(pcraster.Ldd, numpy.array([ [ 1, 2], [10, 15], [ 8, 9]], numpy.int8), 15) self.assertEqual(str(context_manager.exception), "Incorrect value 10 at input array [1][0] for LDD map") # int8 -> Nominal (int32) # All valid int8 values are valid nominal values. raster = pcraster.numpy2pcr(pcraster.Nominal, numpy.array([ [int8_min, 2], [ 5, 15], [ 8, int8_max]], numpy.int8), 15) self.assertEqual(pcraster.cellvalue(raster, 1, 1), (int8_min, True)) self.assertEqual(pcraster.cellvalue(raster, 2, 1), ( 5, True)) self.assertEqual(pcraster.cellvalue(raster, 2, 2)[1], False) self.assertEqual(pcraster.cellvalue(raster, 3, 2), (int8_max, True)) # int8 -> Ordinal (int32) # All valid int8 values are valid ordinal values. raster = pcraster.numpy2pcr(pcraster.Ordinal, numpy.array([ [int8_min, 2], [ 5, 15], [ 8, int8_max]], numpy.int8), 15) self.assertEqual(pcraster.cellvalue(raster, 1, 1), (int8_min, True)) self.assertEqual(pcraster.cellvalue(raster, 2, 1), ( 5, True)) self.assertEqual(pcraster.cellvalue(raster, 2, 2)[1], False) self.assertEqual(pcraster.cellvalue(raster, 3, 2), (int8_max, True)) # int8 -> Scalar (float32) # All valid int8 values are valid scalar values. raster = pcraster.numpy2pcr(pcraster.Scalar, numpy.array([ [int8_min, 2], [ 5, 15], [ 8, int8_max]], numpy.int8), 15) self.assertEqual(pcraster.cellvalue(raster, 1, 1), (int8_min, True)) self.assertEqual(pcraster.cellvalue(raster, 2, 1), ( 5, True)) self.assertEqual(pcraster.cellvalue(raster, 2, 2)[1], False) self.assertEqual(pcraster.cellvalue(raster, 3, 2), (int8_max, True)) # int8 -> Directional (float32) # All valid int8 values are valid directional values. raster = pcraster.numpy2pcr(pcraster.Directional, numpy.array([ [int8_min, 2], [ 5, 15], [ 8, int8_max]], numpy.int8), 15) self.assertEqual(pcraster.cellvalue(raster, 1, 1), (int8_min, True)) self.assertEqual(pcraster.cellvalue(raster, 2, 1), ( 5, True)) self.assertEqual(pcraster.cellvalue(raster, 2, 2)[1], False) self.assertEqual(pcraster.cellvalue(raster, 3, 2), (int8_max, True)) # int16 TODO # int32 TODO # int64 -> Boolean (uint8) raster = pcraster.numpy2pcr(pcraster.Boolean, numpy.array([ [1, 1], [0, 5], [1, 1]], numpy.int64), 5) self.assertEqual(pcraster.cellvalue(raster, 1, 1), (True, True)) self.assertEqual(pcraster.cellvalue(raster, 2, 1), (False, True)) self.assertEqual(pcraster.cellvalue(raster, 2, 2)[1], False) with self.assertRaises(Exception) as context_manager: raster = pcraster.numpy2pcr(pcraster.Boolean, numpy.array([ [1, 1], [9, 5], [1, 1]], numpy.int64), 5) self.assertEqual(str(context_manager.exception), "Incorrect value 9 at input array [1][0] for Boolean map") # int64 -> Ldd (uint8) raster = pcraster.numpy2pcr(pcraster.Ldd, numpy.array([ [1, 2], [5, 15], [8, 9]], numpy.int64), 15) self.assertEqual(pcraster.cellvalue(raster, 1, 1), (1, True)) self.assertEqual(pcraster.cellvalue(raster, 2, 1), (5, True)) self.assertEqual(pcraster.cellvalue(raster, 2, 2)[1], False) with self.assertRaises(Exception) as context_manager: raster = pcraster.numpy2pcr(pcraster.Ldd, numpy.array([ [ 1, 2], [10, 15], [ 8, 9]], numpy.int64), 15) self.assertEqual(str(context_manager.exception), "Incorrect value 10 at input array [1][0] for LDD map") # int64 -> Nominal (int32) raster = pcraster.numpy2pcr(pcraster.Nominal, numpy.array([ [int32_min+1, 2], [ 5, 15], [ 8, int32_max]], numpy.int64), 15) self.assertEqual(pcraster.cellvalue(raster, 1, 1), (int32_min + 1, True)) self.assertEqual(pcraster.cellvalue(raster, 2, 1), ( 5, True)) self.assertEqual(pcraster.cellvalue(raster, 2, 2)[1], False) self.assertEqual(pcraster.cellvalue(raster, 3, 2), ( int32_max, True)) with self.assertRaises(Exception) as context_manager: raster = pcraster.numpy2pcr(pcraster.Nominal, numpy.array([ [int32_min+1, 2], [ int32_min, 15], [ 8, int32_max]], numpy.int64), 15) self.assertEqual(str(context_manager.exception), "Incorrect value -2147483648 at input array [1][0] for Nominal map") # uint8 TODO # uint16 TODO # uint32 TODO # uint64 TODO # float16 TODO # float32 -> Scalar (float32) raster = pcraster.numpy2pcr(pcraster.Scalar, numpy.array([ [-2, -1], [ 0, numpy.nan], [ 1, 2]], numpy.float32), numpy.nan) self.assertEqual(pcraster.cellvalue(raster, 1, 1), (-2, True)) self.assertEqual(pcraster.cellvalue(raster, 2, 1), ( 0, True)) self.assertEqual(pcraster.cellvalue(raster, 2, 2)[1], False) self.assertEqual(pcraster.cellvalue(raster, 3, 2), (2, True)) # float64 -> Scalar (float32) raster = pcraster.numpy2pcr(pcraster.Scalar, numpy.array([ [-2, -1], [ 0, numpy.nan], [ 1, 2]], numpy.float64), numpy.nan) self.assertEqual(pcraster.cellvalue(raster, 1, 1), (-2, True)) self.assertEqual(pcraster.cellvalue(raster, 2, 1), ( 0, True)) self.assertEqual(pcraster.cellvalue(raster, 2, 2)[1], False) self.assertEqual(pcraster.cellvalue(raster, 3, 2), (2, True)) # complex64: Not supported. with self.assertRaises(Exception) as context_manager: raster = pcraster.numpy2pcr(pcraster.Nominal, numpy.array([ [-2, -1], [ 0, 15], [ 1, 2]], numpy.complex64), 15) self.assertEqual(str(context_manager.exception), "Unsupported array type")
def main(source, destination, inifile, dem_in, rivshp, catchshp, gaugeshp=None, landuse=None, soil=None, lai=None, other_maps=None, logfilename='wtools_static_maps.log', verbose=True, clean=True, alltouch=False, outlets=([], [])): # parse other maps into an array if not other_maps == None: if type(other_maps) == str: print other_maps other_maps = other_maps.replace(' ', '').replace('[', '').replace( ']', '').split(',') source = os.path.abspath(source) clone_map = os.path.join(source, 'mask.map') clone_shp = os.path.join(source, 'mask.shp') clone_prj = os.path.join(source, 'mask.prj') if None in (rivshp, catchshp, dem_in): msg = """The following files are compulsory: - DEM (raster) - river (shape) - catchment (shape) """ print(msg) parser.print_help() sys.exit(1) if (inifile is not None) and (not os.path.exists(inifile)): print 'path to ini file cannot be found' sys.exit(1) if not os.path.exists(rivshp): print 'path to river shape cannot be found' sys.exit(1) if not os.path.exists(catchshp): print 'path to catchment shape cannot be found' sys.exit(1) if not os.path.exists(dem_in): print 'path to DEM cannot be found' sys.exit(1) # open a logger, dependent on verbose print to screen or not logger, ch = wt.setlogger(logfilename, 'WTOOLS', verbose) # create directories # TODO: check if workdir is still necessary, try to # keep in memory as much as possible # delete old files (when the source and destination folder are different) if np.logical_and(os.path.isdir(destination), destination is not source): shutil.rmtree(destination) if destination is not source: os.makedirs(destination) # Read mask if not (os.path.exists(clone_map)): logger.error( 'Clone file {:s} not found. Please run create_grid first.'.format( clone_map)) sys.exit(1) else: # set clone pcr.setclone(clone_map) # get the extent from clone.tif xax, yax, clone, fill_value = wt.gdal_readmap(clone_map, 'GTiff') trans = wt.get_geotransform(clone_map) extent = wt.get_extent(clone_map) xmin, ymin, xmax, ymax = extent zeros = np.zeros(clone.shape) ones = pcr.numpy2pcr(pcr.Scalar, np.ones(clone.shape), -9999) # get the projection from clone.tif srs = wt.get_projection(clone_map) unit_clone = srs.GetAttrValue('UNIT').lower() # READ CONFIG FILE # open config-file if inifile is None: config = ConfigParser.SafeConfigParser() config.optionxform = str else: config = wt.OpenConf(inifile) # read settings snapgaugestoriver = wt.configget(config, 'settings', 'snapgaugestoriver', True, datatype='boolean') burnalltouching = wt.configget(config, 'settings', 'burncatchalltouching', True, datatype='boolean') burninorder = wt.configget(config, 'settings', 'burncatchalltouching', False, datatype='boolean') verticetollerance = wt.configget(config, 'settings', 'vertice_tollerance', 0.0001, datatype='float') ''' read parameters ''' burn_outlets = wt.configget(config, 'parameters', 'burn_outlets', 10000, datatype='int') burn_rivers = wt.configget(config, 'parameters', 'burn_rivers', 200, datatype='int') burn_connections = wt.configget(config, 'parameters', 'burn_connections', 100, datatype='int') burn_gauges = wt.configget(config, 'parameters', 'burn_gauges', 100, datatype='int') minorder = wt.configget(config, 'parameters', 'riverorder_min', 3, datatype='int') try: percentiles = np.array(config.get('parameters', 'statisticmaps', '0, 100').replace(' ', '').split(','), dtype='float') except ConfigParser.NoOptionError: percentiles = [0.0, 100.0] # read the parameters for generating a temporary very high resolution grid if unit_clone == 'degree': cellsize_hr = wt.configget(config, 'parameters', 'highres_degree', 0.0005, datatype='float') elif (unit_clone == 'metre') or (unit_clone == 'meter'): cellsize_hr = wt.configget(config, 'parameters', 'highres_metre', 50, datatype='float') cols_hr = int((float(xmax) - float(xmin)) / cellsize_hr + 2) rows_hr = int((float(ymax) - float(ymin)) / cellsize_hr + 2) hr_trans = (float(xmin), cellsize_hr, float(0), float(ymax), 0, -cellsize_hr) clone_hr = os.path.join(destination, 'clone_highres.tif') # make a highres clone as well! wt.CreateTif(clone_hr, rows_hr, cols_hr, hr_trans, srs, 0) # read staticmap locations catchment_map = wt.configget(config, 'staticmaps', 'catchment', 'wflow_catchment.map') dem_map = wt.configget(config, 'staticmaps', 'dem', 'wflow_dem.map') demmax_map = wt.configget(config, 'staticmaps', 'demmax', 'wflow_demmax.map') demmin_map = wt.configget(config, 'staticmaps', 'demmin', 'wflow_demmin.map') gauges_map = wt.configget(config, 'staticmaps', 'gauges', 'wflow_gauges.map') landuse_map = wt.configget(config, 'staticmaps', 'landuse', 'wflow_landuse.map') ldd_map = wt.configget(config, 'staticmaps', 'ldd', 'wflow_ldd.map') river_map = wt.configget(config, 'staticmaps', 'river', 'wflow_river.map') outlet_map = wt.configget(config, 'staticmaps', 'outlet', 'wflow_outlet.map') riverlength_fact_map = wt.configget(config, 'staticmaps', 'riverlength_fact', 'wflow_riverlength_fact.map') soil_map = wt.configget(config, 'staticmaps', 'soil', 'wflow_soil.map') streamorder_map = wt.configget(config, 'staticmaps', 'streamorder', 'wflow_streamorder.map') subcatch_map = wt.configget(config, 'staticmaps', 'subcatch', 'wflow_subcatch.map') # read mask location (optional) masklayer = wt.configget(config, 'mask', 'masklayer', catchshp) # ???? empty = pcr.ifthen(ones == 0, pcr.scalar(0)) # TODO: check if extents are correct this way # TODO: check what the role of missing values is in zeros and ones (l. 123 # in old code) # first add a missing value to dem_in ds = gdal.Open(dem_in, gdal.GA_Update) RasterBand = ds.GetRasterBand(1) fill_val = RasterBand.GetNoDataValue() if fill_val is None: RasterBand.SetNoDataValue(-9999) ds = None # reproject to clone map: see http://stackoverflow.com/questions/10454316/how-to-project-and-resample-a-grid-to-match-another-grid-with-gdal-python # resample DEM logger.info('Resampling dem from {:s} to {:s}'.format( os.path.abspath(dem_in), os.path.join(destination, dem_map))) wt.gdal_warp(dem_in, clone_map, os.path.join(destination, dem_map), format='PCRaster', gdal_interp=gdalconst.GRA_Average) # retrieve amount of rows and columns from clone # TODO: make windowstats applicable to source/target with different projections. This does not work yet. # retrieve srs from DEM try: srs_dem = wt.get_projection(dem_in) except: logger.warning( 'No projection found in DEM, assuming WGS 1984 lat long') srs_dem = osr.SpatialReference() srs_dem.ImportFromEPSG(4326) clone2dem_transform = osr.CoordinateTransformation(srs, srs_dem) # if srs.ExportToProj4() == srs_dem.ExportToProj4(): wt.windowstats(dem_in, len(yax), len(xax), trans, srs, destination, percentiles, transform=clone2dem_transform, logger=logger) ## read catchment shape-file to create catchment map src = rasterio.open(clone_map) shapefile = fiona.open(catchshp, "r") catchment_shapes = [feature["geometry"] for feature in shapefile] image = features.rasterize(catchment_shapes, out_shape=src.shape, all_touched=True, transform=src.transform) catchment_domain = pcr.numpy2pcr(pcr.Ordinal, image.copy(), 0) ## read river shape-file and create burn layer shapefile = fiona.open(rivshp, "r") river_shapes = [feature["geometry"] for feature in shapefile] image = features.rasterize(river_shapes, out_shape=src.shape, all_touched=False, transform=src.transform) rivers = pcr.numpy2pcr(pcr.Nominal, image.copy(), 0) riverdem = pcr.scalar(rivers) * pcr.readmap( os.path.join(destination, dem_map)) pcr.setglobaloption("lddin") riverldd = pcr.lddcreate(riverdem, 1e35, 1e35, 1e35, 1e35) riveroutlet = pcr.cover( pcr.ifthen(pcr.scalar(riverldd) == 5, pcr.scalar(1000)), 0) burn_layer = pcr.cover((pcr.scalar( pcr.ifthen(pcr.streamorder(riverldd) > 1, pcr.streamorder(riverldd))) - 1) * 1000 + riveroutlet, 0) outlets_x, outlets_y = outlets n_outlets = len(outlets_x) logger.info('Number of outlets: {}'.format(n_outlets)) if n_outlets >= 1: outlets_map_numbered = tr.points_to_map(pcr.scalar(0), outlets_x, outlets_y, 0.5) outlets_map = pcr.boolean(outlets_map_numbered) # snap outlets to closest river (max 1 cell closer to river) outlets_map = pcr.boolean( pcr.cover(tr.snaptomap(pcr.ordinal(outlets_map), rivers), 0)) ## create ldd per catchment logger.info('Calculating ldd') ldddem = pcr.scalar(clone_map) # per subcatchment, burn dem, then create modified dem that fits the ldd of the subcatchment # this ldd dem is merged over catchments, to create a global ldd that abides to the subcatchment boundaries for idx, shape in enumerate(catchment_shapes): logger.info('Computing ldd for catchment ' + str(idx + 1) + '/' + str(len(catchment_shapes))) image = features.rasterize([shape], out_shape=src.shape, all_touched=True, transform=src.transform) catchment = pcr.numpy2pcr(pcr.Scalar, image.copy(), 0) dem_burned_catchment = ( pcr.readmap(os.path.join(destination, dem_map)) * pcr.scalar(catchment_domain) * catchment) - burn_layer ldddem_catchment = pcr.lddcreatedem(dem_burned_catchment, 1e35, 1e35, 1e35, 1e35) ldddem = pcr.cover(ldddem, ldddem_catchment) wflow_ldd = pcr.lddcreate(ldddem, 1e35, 1e35, 1e35, 1e35) if n_outlets >= 1: # set outlets to pit wflow_ldd = pcr.ifthenelse(outlets_map, pcr.ldd(5), wflow_ldd) wflow_ldd = pcr.lddrepair(wflow_ldd) pcr.report(wflow_ldd, os.path.join(destination, 'wflow_ldd.map')) # compute stream order, identify river cells streamorder = pcr.ordinal(pcr.streamorder(wflow_ldd)) river = pcr.ifthen(streamorder >= pcr.ordinal(minorder), pcr.boolean(1)) # find the minimum value in the DEM and cover missing values with a river with this value. Effect is none!! so now left out! # mindem = int(np.min(pcr.pcr2numpy(pcr.ordinal(os.path.join(destination, dem_map)),9999999))) # dem_resample_map = pcr.cover(os.path.join(destination, dem_map), pcr.scalar(river)*0+mindem) # pcr.report(dem_resample_map, os.path.join(destination, dem_map)) pcr.report(streamorder, os.path.join(destination, streamorder_map)) pcr.report(river, os.path.join(destination, river_map)) # deal with your catchments if gaugeshp == None: logger.info('No gauges defined, using outlets instead') gauges = pcr.ordinal( pcr.uniqueid( pcr.boolean( pcr.ifthen(pcr.scalar(wflow_ldd) == 5, pcr.boolean(1))))) pcr.report(gauges, os.path.join(destination, gauges_map)) # TODO: Add the gauge shape code from StaticMaps.py (line 454-489) # TODO: add river length map (see SticMaps.py, line 492-499) # since the products here (river length fraction) are not yet used # this is disabled for now, as it also takes a lot of computation time if False: # report river length # make a high resolution empty map dem_hr_file = os.path.join(destination, 'dem_highres.tif') burn_hr_file = os.path.join(destination, 'burn_highres.tif') demburn_hr_file = os.path.join(destination, 'demburn_highres.map') riv_hr_file = os.path.join(destination, 'riv_highres.map') wt.gdal_warp(dem_in, clone_hr, dem_hr_file) # wt.CreateTif(riv_hr, rows_hr, cols_hr, hr_trans, srs, 0) # open the shape layer ds = ogr.Open(rivshp) lyr = ds.GetLayer(0) wt.ogr_burn(lyr, clone_hr, -100, file_out=burn_hr_file, format='GTiff', gdal_type=gdal.GDT_Float32, fill_value=0) # read dem and burn values and add xax_hr, yax_hr, burn_hr, fill = wt.gdal_readmap(burn_hr_file, 'GTiff') burn_hr[burn_hr == fill] = 0 xax_hr, yax_hr, dem_hr, fill = wt.gdal_readmap(dem_hr_file, 'GTiff') dem_hr[dem_hr == fill] = np.nan demburn_hr = dem_hr + burn_hr demburn_hr[np.isnan(demburn_hr)] = -9999 wt.gdal_writemap(demburn_hr_file, 'PCRaster', xax_hr, yax_hr, demburn_hr, -9999.) pcr.setclone(demburn_hr_file) demburn_hr = pcr.readmap(demburn_hr_file) logger.info('Calculating ldd to determine river length') ldd_hr = pcr.lddcreate(demburn_hr, 1e35, 1e35, 1e35, 1e35) pcr.report(ldd_hr, os.path.join(destination, 'ldd_hr.map')) pcr.setglobaloption('unitcell') riv_hr = pcr.scalar( pcr.streamorder(ldd_hr) >= minorder) * pcr.downstreamdist(ldd_hr) pcr.report(riv_hr, riv_hr_file) pcr.setglobaloption('unittrue') pcr.setclone(clone_map) logger.info('Computing river length') wt.windowstats(riv_hr_file, len(yax), len(xax), trans, srs, destination, stat='fact', transform=False, logger=logger) # TODO: nothing happens with the river lengths yet. Need to decide how to use these # report outlet map pcr.report(pcr.ifthen(pcr.ordinal(wflow_ldd) == 5, pcr.ordinal(1)), os.path.join(destination, outlet_map)) # report subcatchment map subcatchment = pcr.subcatchment(wflow_ldd, gauges) pcr.report(pcr.ordinal(subcatchment), os.path.join(destination, subcatch_map)) # Report land use map if landuse == None: logger.info( 'No land use map used. Preparing {:s} with only ones.'.format( os.path.join(destination, landuse_map))) pcr.report(pcr.nominal(ones), os.path.join(destination, landuse_map)) else: logger.info('Resampling land use from {:s} to {:s}'.format( os.path.abspath(landuse), os.path.join(destination, os.path.abspath(landuse_map)))) wt.gdal_warp(landuse, clone_map, os.path.join(destination, landuse_map), format='PCRaster', gdal_interp=gdalconst.GRA_Mode, gdal_type=gdalconst.GDT_Int32) # report soil map if soil == None: logger.info('No soil map used. Preparing {:s} with only ones.'.format( os.path.join(destination, soil_map))) pcr.report(pcr.nominal(ones), os.path.join(destination, soil_map)) else: logger.info('Resampling soil from {:s} to {:s}'.format( os.path.abspath(soil), os.path.join(destination, os.path.abspath(soil_map)))) wt.gdal_warp(soil, clone_map, os.path.join(destination, soil_map), format='PCRaster', gdal_interp=gdalconst.GRA_Mode, gdal_type=gdalconst.GDT_Int32) if lai == None: logger.info( 'No vegetation LAI maps used. Preparing default maps {:s} with only ones.' .format(os.path.join(destination, soil_map))) pcr.report(pcr.nominal(ones), os.path.join(destination, soil_map)) else: dest_lai = os.path.join(destination, 'clim') os.makedirs(dest_lai) for month in range(12): lai_in = os.path.join(lai, 'LAI00000.{:03d}'.format(month + 1)) lai_out = os.path.join(dest_lai, 'LAI00000.{:03d}'.format(month + 1)) logger.info('Resampling vegetation LAI from {:s} to {:s}'.format( os.path.abspath(lai_in), os.path.abspath(lai_out))) wt.gdal_warp(lai_in, clone_map, lai_out, format='PCRaster', gdal_interp=gdalconst.GRA_Bilinear, gdal_type=gdalconst.GDT_Float32) # report soil map if other_maps == None: logger.info('No other maps used. Skipping other maps.') else: logger.info('Resampling list of other maps...') for map_file in other_maps: map_name = os.path.split(map_file)[1] logger.info('Resampling a map from {:s} to {:s}'.format( os.path.abspath(map_file), os.path.join( destination, os.path.splitext(os.path.basename(map_file))[0] + '.map'))) wt.gdal_warp(map_file, clone_map, os.path.join( destination, os.path.splitext(os.path.basename(map_file))[0] + '.map'), format='PCRaster', gdal_interp=gdalconst.GRA_Mode, gdal_type=gdalconst.GDT_Float32) if clean: wt.DeleteList(glob.glob(os.path.join(destination, '*.xml')), logger=logger) wt.DeleteList(glob.glob(os.path.join(destination, 'clim', '*.xml')), logger=logger) wt.DeleteList(glob.glob(os.path.join(destination, '*highres*')), logger=logger)
def netcdf2PCRobjClone(ncFile,varName,dateInput,\ useDoy = None, cloneMapFileName = None,\ LatitudeLongitude = True,\ specificFillValue = None): # # EHS (19 APR 2013): To convert netCDF (tss) file to PCR file. # --- with clone checking # Only works if cells are 'square'. # Only works if cellsizeClone <= cellsizeInput # Get netCDF file and variable name: #~ print ncFile logger.debug('reading variable: '+str(varName)+' from the file: '+str(ncFile)) if ncFile in filecache.keys(): f = filecache[ncFile] #~ print "Cached: ", ncFile else: f = nc.Dataset(ncFile) filecache[ncFile] = f #~ print "New: ", ncFile varName = str(varName) if varName == "automatic": nc_dims = [dim for dim in f.dimensions] nc_vars = [var for var in f.variables] for var in nc_vars: if var not in nc_dims: varName = var logger.debug('reading variable: '+str(varName)+' from the file: '+str(ncFile)) if LatitudeLongitude == True: try: f.variables['lat'] = f.variables['latitude'] f.variables['lon'] = f.variables['longitude'] except: pass if varName == "evapotranspiration": try: f.variables['evapotranspiration'] = f.variables['referencePotET'] except: pass # date date = dateInput if useDoy == "Yes": idx = dateInput - 1 else: if isinstance(date, str) == True: date = \ datetime.datetime.strptime(str(date),'%Y-%m-%d') date = datetime.datetime(date.year,date.month,date.day) # time index (in the netCDF file) if useDoy == "month": idx = int(date.month) - 1 else: nctime = f.variables['time'] # A netCDF time variable object. if useDoy == "yearly": date = datetime.datetime(date.year,int(1),int(1)) if useDoy == "monthly": date = datetime.datetime(date.year,date.month,int(1)) if useDoy == "yearly" or useDoy == "monthly": # if the desired year is not available, use the first year or the last year that is available first_year_in_nc_file = findFirstYearInNCTime(nctime) last_year_in_nc_file = findLastYearInNCTime(nctime) # if date.year < first_year_in_nc_file: date = datetime.datetime(first_year_in_nc_file,date.month,date.day) msg = "\n" msg += "WARNING related to the netcdf file: "+str(ncFile)+" ; variable: "+str(varName)+" !!!!!!"+"\n" msg += "The date "+str(dateInput)+" is NOT available. " msg += "The date "+str(date.year)+"-"+str(date.month)+"-"+str(date.day)+" is used." msg += "\n" logger.warning(msg) if date.year > last_year_in_nc_file: date = datetime.datetime(last_year_in_nc_file,date.month,date.day) msg = "\n" msg += "WARNING related to the netcdf file: "+str(ncFile)+" ; variable: "+str(varName)+" !!!!!!"+"\n" msg += "The date "+str(dateInput)+" is NOT available. " msg += "The date "+str(date.year)+"-"+str(date.month)+"-"+str(date.day)+" is used." msg += "\n" logger.warning(msg) try: idx = nc.date2index(date, nctime, calendar = nctime.calendar, \ select='exact') except: try: idx = nc.date2index(date, nctime, calendar = nctime.calendar, \ select='before') msg = "\n" msg += "WARNING related to the netcdf file: "+str(ncFile)+" ; variable: "+str(varName)+" !!!!!!"+"\n" msg += "The date "+str(dateInput)+" is NOT available. The 'before' option is used while selecting netcdf time." msg += "\n" except: idx = nc.date2index(date, nctime, calendar = nctime.calendar, \ select='after') msg = "\n" msg += "WARNING related to the netcdf file: "+str(ncFile)+" ; variable: "+str(varName)+" !!!!!!"+"\n" msg += "The date "+str(dateInput)+" is NOT available. The 'after' option is used while selecting netcdf time." msg += "\n" logger.warning(msg) idx = int(idx) sameClone = True # check whether clone and input maps have the same attributes: if cloneMapFileName != None: # get the attributes of cloneMap attributeClone = getMapAttributesALL(cloneMapFileName) cellsizeClone = attributeClone['cellsize'] rowsClone = attributeClone['rows'] colsClone = attributeClone['cols'] xULClone = attributeClone['xUL'] yULClone = attributeClone['yUL'] # get the attributes of input (netCDF) cellsizeInput = f.variables['lat'][0]- f.variables['lat'][1] cellsizeInput = float(cellsizeInput) rowsInput = len(f.variables['lat']) colsInput = len(f.variables['lon']) xULInput = f.variables['lon'][0]-0.5*cellsizeInput yULInput = f.variables['lat'][0]+0.5*cellsizeInput # check whether both maps have the same attributes if cellsizeClone != cellsizeInput: sameClone = False if rowsClone != rowsInput: sameClone = False if colsClone != colsInput: sameClone = False if xULClone != xULInput: sameClone = False if yULClone != yULInput: sameClone = False cropData = f.variables[varName][int(idx),:,:] # still original data factor = 1 # needed in regridData2FinerGrid if sameClone == False: logger.debug('Crop to the clone map with lower left corner (x,y): '+str(xULClone)+' , '+str(yULClone)) # crop to cloneMap: #~ xIdxSta = int(np.where(f.variables['lon'][:] == xULClone + 0.5*cellsizeInput)[0]) minX = min(abs(f.variables['lon'][:] - (xULClone + 0.5*cellsizeInput))) # ; print(minX) xIdxSta = int(np.where(abs(f.variables['lon'][:] - (xULClone + 0.5*cellsizeInput)) == minX)[0]) xIdxEnd = int(math.ceil(xIdxSta + colsClone /(cellsizeInput/cellsizeClone))) #~ yIdxSta = int(np.where(f.variables['lat'][:] == yULClone - 0.5*cellsizeInput)[0]) minY = min(abs(f.variables['lat'][:] - (yULClone - 0.5*cellsizeInput))) # ; print(minY) yIdxSta = int(np.where(abs(f.variables['lat'][:] - (yULClone - 0.5*cellsizeInput)) == minY)[0]) yIdxEnd = int(math.ceil(yIdxSta + rowsClone /(cellsizeInput/cellsizeClone))) cropData = f.variables[varName][idx,yIdxSta:yIdxEnd,xIdxSta:xIdxEnd] factor = int(round(float(cellsizeInput)/float(cellsizeClone))) if factor > 1: logger.debug('Resample: input cell size = '+str(float(cellsizeInput))+' ; output/clone cell size = '+str(float(cellsizeClone))) # convert to PCR object and close f if specificFillValue != None: outPCR = pcr.numpy2pcr(pcr.Scalar, \ regridData2FinerGrid(factor,cropData,MV), \ float(specificFillValue)) else: outPCR = pcr.numpy2pcr(pcr.Scalar, \ regridData2FinerGrid(factor,cropData,MV), \ float(f.variables[varName]._FillValue)) #f.close(); f = None ; cropData = None # PCRaster object return (outPCR)
def static_maps( source, # source folder containing clone destination, # destination folder inifile, # ini file with various settings dem_in, # path to digital elevation model (raster) rivshp, # path to river network (line vector) catchshp, # path to catchment polygon (polygon vector) gaugeshp=None, # path to gauge point (point vector) landuse=None, # path to land use / land cover (raster) soil=None, # path to soil type (raster) lai=None, # path to vegetation LAI (containing 12 GeoTiffs LAI00000.XXX.tif) other_maps=None, # bracketed [] comma-separated list of paths to other maps that should be reprojected logfilename="wtools_static_maps.log", # log file name verbose=True, clean=True, # Clean the .xml files from static maps folder when finished alltouch=False, # option to burn catchments "all touching".\nUseful when catchment-size is small compared to cellsize outlets=([], []), ): # parse other maps into an array if not other_maps == None: if type(other_maps) == str: print(other_maps) other_maps = ( other_maps.replace(" ", "").replace("[", "").replace("]", "").split(",") ) source = os.path.abspath(source) clone_tif = os.path.join(source, "mask.tif") clone_map = os.path.join(source, "mask.map") clone_shp = os.path.join(source, "mask.shp") clone_prj = os.path.join(source, "mask.prj") # open a logger, dependent on verbose print to screen or not logger, ch = wt.setlogger(logfilename, "WTOOLS", verbose) # create directories # TODO: check if workdir is still necessary, try to # keep in memory as much as possible # delete old files (when the source and destination folder are different) if np.logical_and(os.path.isdir(destination), destination is not source): shutil.rmtree(destination) if destination is not source: os.makedirs(destination) # Read mask if not (os.path.exists(clone_map)): logger.error( "Clone file {:s} not found. Please run create_grid first.".format(clone_map) ) sys.exit(1) else: # set clone pcr.setclone(clone_map) # get the extent from clone.tif xax, yax, clone, fill_value = wt.gdal_readmap(clone_tif, "GTiff") trans = wt.get_geotransform(clone_tif) extent = wt.get_extent(clone_tif) xmin, ymin, xmax, ymax = extent zeros = np.zeros(clone.shape) ones = pcr.numpy2pcr(pcr.Scalar, np.ones(clone.shape), -9999) # get the projection from clone.tif srs = wt.get_projection(clone_tif) unit_clone = srs.GetAttrValue("UNIT").lower() # READ CONFIG FILE # open config-file if inifile is None: config = configparser.ConfigParser() config.optionxform = str else: config = wt.OpenConf(inifile) # read settings """ read parameters """ minorder = wt.configget(config, "parameters", "riverorder_min", 3, datatype="int") try: percentiles_str = wt.configget( config, "parameters", "statisticmaps", "0, 100", datatype="str" ) percentiles_split = percentiles_str.replace(" ", "").split(",") percentiles = np.array(percentiles_split, dtype="float") except configparser.NoOptionError: percentiles = [0.0, 100.0] # read the parameters for generating a temporary very high resolution grid if unit_clone == "degree": cellsize_hr = wt.configget( config, "parameters", "highres_degree", 0.0005, datatype="float" ) elif (unit_clone == "metre") or (unit_clone == "meter"): cellsize_hr = wt.configget( config, "parameters", "highres_metre", 50, datatype="float" ) cols_hr = int((float(xmax) - float(xmin)) / cellsize_hr + 2) rows_hr = int((float(ymax) - float(ymin)) / cellsize_hr + 2) hr_trans = (float(xmin), cellsize_hr, float(0), float(ymax), 0, -cellsize_hr) clone_hr = os.path.join(destination, "clone_highres.tif") # make a highres clone as well! wt.CreateTif(clone_hr, rows_hr, cols_hr, hr_trans, srs, 0) # read staticmap locations dem_map = wt.configget(config, "staticmaps", "dem", "wflow_dem.map") gauges_map = wt.configget(config, "staticmaps", "gauges", "wflow_gauges.map") landuse_map = wt.configget(config, "staticmaps", "landuse", "wflow_landuse.map") river_map = wt.configget(config, "staticmaps", "river", "wflow_river.map") outlet_map = wt.configget(config, "staticmaps", "outlet", "wflow_outlet.map") soil_map = wt.configget(config, "staticmaps", "soil", "wflow_soil.map") streamorder_map = wt.configget( config, "staticmaps", "streamorder", "wflow_streamorder.map" ) subcatch_map = wt.configget(config, "staticmaps", "subcatch", "wflow_subcatch.map") # first add a missing value to dem_in ds = gdal.Open(dem_in, gdal.GA_Update) RasterBand = ds.GetRasterBand(1) fill_val = RasterBand.GetNoDataValue() if fill_val is None: RasterBand.SetNoDataValue(-9999) ds = None # reproject to clone map: see http://stackoverflow.com/questions/10454316/how-to-project-and-resample-a-grid-to-match-another-grid-with-gdal-python # resample DEM logger.info( "Resampling dem from {:s} to {:s}".format( os.path.abspath(dem_in), os.path.join(destination, dem_map) ) ) wt.gdal_warp( dem_in, clone_map, os.path.join(destination, dem_map), format="PCRaster", gdal_interp=gdalconst.GRA_Average, ) # retrieve amount of rows and columns from clone # TODO: make windowstats applicable to source/target with different projections. This does not work yet. # retrieve srs from DEM try: srs_dem = wt.get_projection(dem_in) except: logger.warning("No projection found in DEM, assuming WGS 1984 lat long") srs_dem = osr.SpatialReference() srs_dem.ImportFromEPSG(4326) clone2dem_transform = osr.CoordinateTransformation(srs, srs_dem) # if srs.ExportToProj4() == srs_dem.ExportToProj4(): wt.windowstats( dem_in, len(yax), len(xax), trans, srs, destination, percentiles, transform=clone2dem_transform, logger=logger, ) ## read catchment shape-file to create catchment map src = rasterio.open(clone_tif) shapefile = fiona.open(catchshp, "r") catchment_shapes = [feature["geometry"] for feature in shapefile] image = features.rasterize( catchment_shapes, out_shape=src.shape, all_touched=True, transform=src.transform ) catchment_domain = pcr.numpy2pcr(pcr.Ordinal, image.copy(), 0) ## read river shape-file and create burn layer shapefile = fiona.open(rivshp, "r") river_shapes = [feature["geometry"] for feature in shapefile] image = features.rasterize( river_shapes, out_shape=src.shape, all_touched=False, transform=src.transform ) rivers = pcr.numpy2pcr(pcr.Nominal, image.copy(), 0) riverdem = pcr.scalar(rivers) * pcr.readmap(os.path.join(destination, dem_map)) pcr.setglobaloption("lddin") riverldd = pcr.lddcreate(riverdem, 1e35, 1e35, 1e35, 1e35) riveroutlet = pcr.cover(pcr.ifthen(pcr.scalar(riverldd) == 5, pcr.scalar(1000)), 0) burn_layer = pcr.cover( ( pcr.scalar( pcr.ifthen(pcr.streamorder(riverldd) > 1, pcr.streamorder(riverldd)) ) - 1 ) * 1000 + riveroutlet, 0, ) outlets_x, outlets_y = outlets n_outlets = len(outlets_x) logger.info("Number of outlets: {}".format(n_outlets)) if n_outlets >= 1: outlets_map_numbered = points_to_map(pcr.scalar(0), outlets_x, outlets_y, 0.5) outlets_map = pcr.boolean(outlets_map_numbered) # snap outlets to closest river (max 1 cell closer to river) outlets_map = pcr.boolean( pcr.cover(snaptomap(pcr.ordinal(outlets_map), rivers), 0) ) ## create ldd per catchment logger.info("Calculating ldd") ldddem = pcr.scalar(clone_map) # per subcatchment, burn dem, then create modified dem that fits the ldd of the subcatchment # this ldd dem is merged over catchments, to create a global ldd that abides to the subcatchment boundaries for idx, shape in enumerate(catchment_shapes): logger.info( "Computing ldd for catchment " + str(idx + 1) + "/" + str(len(catchment_shapes)) ) image = features.rasterize( [shape], out_shape=src.shape, all_touched=True, transform=src.transform ) catchment = pcr.numpy2pcr(pcr.Scalar, image.copy(), 0) dem_burned_catchment = ( pcr.readmap(os.path.join(destination, dem_map)) * pcr.scalar(catchment_domain) * catchment ) - burn_layer ldddem = pcr.cover(ldddem, dem_burned_catchment) wflow_ldd = pcr.lddcreate(ldddem, 1e35, 1e35, 1e35, 1e35) if n_outlets >= 1: # set outlets to pit wflow_ldd = pcr.ifthenelse(outlets_map, pcr.ldd(5), wflow_ldd) wflow_ldd = pcr.lddrepair(wflow_ldd) pcr.report(wflow_ldd, os.path.join(destination, "wflow_ldd.map")) # compute stream order, identify river cells streamorder = pcr.ordinal(pcr.streamorder(wflow_ldd)) river = pcr.ifthen(streamorder >= pcr.ordinal(minorder), pcr.boolean(1)) # find the minimum value in the DEM and cover missing values with a river with this value. Effect is none!! so now left out! # mindem = int(np.min(pcr.pcr2numpy(pcr.ordinal(os.path.join(destination, dem_map)),9999999))) # dem_resample_map = pcr.cover(os.path.join(destination, dem_map), pcr.scalar(river)*0+mindem) # pcr.report(dem_resample_map, os.path.join(destination, dem_map)) pcr.report(streamorder, os.path.join(destination, streamorder_map)) pcr.report(river, os.path.join(destination, river_map)) # deal with your catchments if gaugeshp == None: logger.info("No gauges defined, using outlets instead") gauges = pcr.ordinal( pcr.uniqueid( pcr.boolean(pcr.ifthen(pcr.scalar(wflow_ldd) == 5, pcr.boolean(1))) ) ) pcr.report(gauges, os.path.join(destination, gauges_map)) # TODO: Add the gauge shape code from StaticMaps.py (line 454-489) # TODO: add river length map (see SticMaps.py, line 492-499) # since the products here (river length fraction) are not yet used # this is disabled for now, as it also takes a lot of computation time if False: # report river length # make a high resolution empty map dem_hr_file = os.path.join(destination, "dem_highres.tif") burn_hr_file = os.path.join(destination, "burn_highres.tif") demburn_hr_file = os.path.join(destination, "demburn_highres.map") riv_hr_file = os.path.join(destination, "riv_highres.map") wt.gdal_warp(dem_in, clone_hr, dem_hr_file) # wt.CreateTif(riv_hr, rows_hr, cols_hr, hr_trans, srs, 0) # open the shape layer ds = ogr.Open(rivshp) lyr = ds.GetLayer(0) wt.ogr_burn( lyr, clone_hr, -100, file_out=burn_hr_file, format="GTiff", gdal_type=gdal.GDT_Float32, fill_value=0, ) # read dem and burn values and add xax_hr, yax_hr, burn_hr, fill = wt.gdal_readmap(burn_hr_file, "GTiff") burn_hr[burn_hr == fill] = 0 xax_hr, yax_hr, dem_hr, fill = wt.gdal_readmap(dem_hr_file, "GTiff") dem_hr[dem_hr == fill] = np.nan demburn_hr = dem_hr + burn_hr demburn_hr[np.isnan(demburn_hr)] = -9999 wt.gdal_writemap( demburn_hr_file, "PCRaster", xax_hr, yax_hr, demburn_hr, -9999.0 ) pcr.setclone(demburn_hr_file) demburn_hr = pcr.readmap(demburn_hr_file) logger.info("Calculating ldd to determine river length") ldd_hr = pcr.lddcreate(demburn_hr, 1e35, 1e35, 1e35, 1e35) pcr.report(ldd_hr, os.path.join(destination, "ldd_hr.map")) pcr.setglobaloption("unitcell") riv_hr = pcr.scalar(pcr.streamorder(ldd_hr) >= minorder) * pcr.downstreamdist( ldd_hr ) pcr.report(riv_hr, riv_hr_file) pcr.setglobaloption("unittrue") pcr.setclone(clone_map) logger.info("Computing river length") wt.windowstats( riv_hr_file, len(yax), len(xax), trans, srs, destination, stat="fact", transform=False, logger=logger, ) # TODO: nothing happens with the river lengths yet. Need to decide how to use these # report outlet map pcr.report( pcr.ifthen(pcr.ordinal(wflow_ldd) == 5, pcr.ordinal(1)), os.path.join(destination, outlet_map), ) # report subcatchment map subcatchment = pcr.subcatchment(wflow_ldd, gauges) pcr.report(pcr.ordinal(subcatchment), os.path.join(destination, subcatch_map)) # Report land use map if landuse == None: logger.info( "No land use map used. Preparing {:s} with only ones.".format( os.path.join(destination, landuse_map) ) ) pcr.report(pcr.nominal(ones), os.path.join(destination, landuse_map)) else: logger.info( "Resampling land use from {:s} to {:s}".format( os.path.abspath(landuse), os.path.join(destination, os.path.abspath(landuse_map)), ) ) wt.gdal_warp( landuse, clone_map, os.path.join(destination, landuse_map), format="PCRaster", gdal_interp=gdalconst.GRA_Mode, gdal_type=gdalconst.GDT_Int32, ) # report soil map if soil == None: logger.info( "No soil map used. Preparing {:s} with only ones.".format( os.path.join(destination, soil_map) ) ) pcr.report(pcr.nominal(ones), os.path.join(destination, soil_map)) else: logger.info( "Resampling soil from {:s} to {:s}".format( os.path.abspath(soil), os.path.join(destination, os.path.abspath(soil_map)), ) ) wt.gdal_warp( soil, clone_map, os.path.join(destination, soil_map), format="PCRaster", gdal_interp=gdalconst.GRA_Mode, gdal_type=gdalconst.GDT_Int32, ) if lai == None: logger.info( "No vegetation LAI maps used. Preparing default maps {:s} with only ones.".format( os.path.join(destination, soil_map) ) ) pcr.report(pcr.nominal(ones), os.path.join(destination, soil_map)) else: dest_lai = os.path.join(destination, "clim") os.makedirs(dest_lai) for month in range(12): lai_in = os.path.join(lai, "LAI00000.{:03d}".format(month + 1)) lai_out = os.path.join(dest_lai, "LAI00000.{:03d}".format(month + 1)) logger.info( "Resampling vegetation LAI from {:s} to {:s}".format( os.path.abspath(lai_in), os.path.abspath(lai_out) ) ) wt.gdal_warp( lai_in, clone_map, lai_out, format="PCRaster", gdal_interp=gdalconst.GRA_Bilinear, gdal_type=gdalconst.GDT_Float32, ) # report soil map if other_maps == None: logger.info("No other maps used. Skipping other maps.") else: logger.info("Resampling list of other maps...") for map_file in other_maps: logger.info( "Resampling a map from {:s} to {:s}".format( os.path.abspath(map_file), os.path.join( destination, os.path.splitext(os.path.basename(map_file))[0] + ".map", ), ) ) wt.gdal_warp( map_file, clone_map, os.path.join( destination, os.path.splitext(os.path.basename(map_file))[0] + ".map", ), format="PCRaster", gdal_interp=gdalconst.GRA_Mode, gdal_type=gdalconst.GDT_Float32, ) if clean: wt.DeleteList(glob.glob(os.path.join(destination, "*.xml")), logger=logger) wt.DeleteList( glob.glob(os.path.join(destination, "clim", "*.xml")), logger=logger ) wt.DeleteList(glob.glob(os.path.join(destination, "*highres*")), logger=logger)
def regridMapFile2FinerGrid (rescaleFac,coarse): if rescaleFac ==1: return coarse return pcr.numpy2pcr(pcr.Scalar, regridData2FinerGrid(rescaleFac,pcr.pcr2numpy(coarse,MV),MV),MV)
def dynamic(self): self.counter += 1 print( str(self.curdate.day) + '-' + str(self.curdate.month) + '-' + str(self.curdate.year) + ' t = ' + str(self.counter)) #-Snow and rain fraction settings for non-glacier part of model cell SnowFrac = pcr.ifthenelse(self.SnowStore > 0, pcr.scalar(1 - self.GlacFrac), 0) RainFrac = pcr.ifthenelse(self.SnowStore == 0, pcr.scalar(1 - self.GlacFrac), 0) #-Read the precipitation time-series if self.precNetcdfFLAG == 1: #-read forcing by netcdf input Precip = self.netcdf2PCraster.netcdf2pcrDynamic(self, pcr, 'Prec') else: #-read forcing by map input Precip = pcr.readmap(pcrm.generateNameT(self.Prec, self.counter)) PrecipTot = Precip #-Report Precip self.reporting.reporting(self, pcr, 'TotPrec', Precip) self.reporting.reporting(self, pcr, 'TotPrecF', Precip * (1 - self.GlacFrac)) #-Temperature and determine reference evapotranspiration if self.tempNetcdfFLAG == 1: #-read forcing by netcdf input Temp = self.netcdf2PCraster.netcdf2pcrDynamic(self, pcr, 'Temp') else: #-read forcing by map input Temp = pcr.readmap(pcrm.generateNameT(self.Tair, self.counter)) if self.ETREF_FLAG == 0: if self.TminNetcdfFLAG == 1: #-read forcing by netcdf input TempMin = self.netcdf2PCraster.netcdf2pcrDynamic( self, pcr, 'Tmin') else: #-read forcing by map input TempMin = pcr.readmap( pcrm.generateNameT(self.Tmin, self.counter)) if self.TmaxNetcdfFLAG == 1: #-read forcing by netcdf input TempMax = self.netcdf2PCraster.netcdf2pcrDynamic( self, pcr, 'Tmax') else: #-read forcing by map input TempMax = pcr.readmap( pcrm.generateNameT(self.Tmax, self.counter)) ETref = self.Hargreaves.Hargreaves( pcr, self.Hargreaves.extrarad(self, pcr), Temp, TempMax, TempMin) else: ETref = pcr.readmap(pcrm.generateNameT(self.ETref, self.counter)) self.reporting.reporting(self, pcr, 'TotETref', ETref) self.reporting.reporting(self, pcr, 'TotETrefF', ETref * (1 - self.GlacFrac)) #-Interception and effective precipitation if self.DynVegFLAG == 1: #-read dynamic processes dynamic vegetation Precip = self.dynamic_veg.dynamic(self, pcr, pcrm, np, Precip, ETref) elif self.KcStatFLAG == 0: #-Try to read the KC map series try: self.Kc = pcr.readmap( pcrm.generateNameT(self.Kcmaps, self.counter)) self.KcOld = self.Kc except: self.Kc = self.KcOld #-report mm effective precipitation for sub-basin averages if self.mm_rep_FLAG == 1 and self.Prec_mm_FLAG == 1 and ( self.RoutFLAG == 1 or self.ResFLAG == 1 or self.LakeFLAG == 1): self.PrecSubBasinTSS.sample( pcr.catchmenttotal(Precip * (1 - self.GlacFrac), self.FlowDir) / pcr.catchmenttotal(1, self.FlowDir)) #-Snow, rain, and glacier calculations for glacier fraction of cell if self.GlacFLAG: #-read dynamic processes glacier Rain_GLAC, Snow_GLAC, ActSnowMelt_GLAC, SnowR_GLAC, GlacMelt, GlacPerc, self.GlacR = self.glacier.dynamic( self, pcr, pd, Temp, Precip) #-If glacier module is not used, then else: Rain_GLAC = 0 Snow_GLAC = 0 ActSnowMelt_GLAC = 0 self.TotalSnowStore_GLAC = 0 SnowR_GLAC = 0 self.GlacR = 0 GlacMelt = 0 GlacPerc = 0 # Calculate snow and rain for non-glacier part of cell if self.SnowFLAG == 1: #-read dynamic processes snow Rain, self.SnowR, OldTotalSnowStore = self.snow.dynamic( self, pcr, Temp, Precip, Snow_GLAC, ActSnowMelt_GLAC, SnowFrac, RainFrac, SnowR_GLAC) else: Rain = Precip self.SnowR = 0 OldTotalSnowStore = 0 self.TotalSnowStore = 0 #-Report Rain self.reporting.reporting(self, pcr, 'TotRain', Rain) self.reporting.reporting(self, pcr, 'TotRainF', Rain * (1 - self.GlacFrac) + Rain_GLAC) # for entire cell #-Potential evapotranspiration ETpot = self.ET.ETpot(ETref, self.Kc) if self.ETOpenWaterFLAG == 1: self.ETOpenWater = self.ET.ETpot(ETref, self.kcOpenWater) #-Report ETpot self.reporting.reporting(self, pcr, 'TotETpot', ETpot) self.reporting.reporting(self, pcr, 'TotETpotF', ETpot * RainFrac) #-Rootzone calculations self.RootWater = self.RootWater + self.CapRise #-Calculate rootzone runoff tempvar = self.rootzone.RootRunoff(self, pcr, RainFrac, Rain) #-Rootzone runoff RootRunoff = tempvar[0] #-Infiltration Infil = tempvar[1] #-Report infiltration self.reporting.reporting(self, pcr, 'Infil', Infil) #-Updated rootwater content self.RootWater = pcr.ifthenelse(RainFrac > 0, self.RootWater + Infil, self.RootWater) #-Actual evapotranspiration if self.PlantWaterStressFLAG == 1: etreddry = self.ET.ks(self, pcr, ETpot) else: etreddry = pcr.max( pcr.min((self.RootWater - self.RootDry) / (self.RootWilt - self.RootDry), 1), 0) self.reporting.reporting(self, pcr, 'PlantStress', 1 - etreddry) ETact = self.ET.ETact(pcr, ETpot, self.RootWater, self.RootSat, etreddry, RainFrac) #-Report the actual evapotranspiration self.reporting.reporting( self, pcr, 'TotETact', ETact * (1 - self.openWaterFrac) + self.ETOpenWater * self.openWaterFrac) #-Actual evapotranspiration, corrected for rain fraction ActETact = ETact * RainFrac #-Report the actual evapotranspiration, corrected for rain fraction self.reporting.reporting(self, pcr, 'TotETactF', ActETact) if self.mm_rep_FLAG == 1 and self.ETa_mm_FLAG == 1 and ( self.RoutFLAG == 1 or self.ResFLAG == 1 or self.LakeFLAG == 1): self.ETaSubBasinTSS.sample( pcr.catchmenttotal(ActETact, self.FlowDir) / pcr.catchmenttotal(1, self.FlowDir)) #-Update rootwater content self.RootWater = pcr.max(self.RootWater - ETact, 0) #-Calculate drainage temp_RootDrain = self.rootzone.RootDrainage( pcr, self.RootWater, self.RootDrain, self.RootField, self.RootSat, self.RootDrainVel, self.RootTT) #-Calculate percolation temp_rootperc = self.rootzone.RootPercolation(pcr, self.RootWater, self.SubWater, self.RootField, self.RootTT, self.SubSat) #-Total sum of water able to leave the soil RootOut = temp_RootDrain + temp_rootperc #-Calculate new values for drainage and percolation (to be used when RootOut > RootExcess) newdrain, newperc = self.rootzone.CalcFrac(pcr, self.RootWater, self.RootField, temp_RootDrain, temp_rootperc) #-Determine whether the new values need to be used rootexcess = pcr.max(self.RootWater - self.RootField, 0) self.RootDrain = pcr.ifthenelse(RootOut > rootexcess, newdrain, temp_RootDrain) rootperc = pcr.ifthenelse(RootOut > rootexcess, newperc, temp_rootperc) #-Update the RootWater content # Roottemp = self.RootWater self.RootWater = self.RootWater - (self.RootDrain + rootperc) #-Report rootzone percolation, corrected for fraction self.reporting.reporting(self, pcr, 'TotRootPF', rootperc * (1 - self.GlacFrac)) #-Report rootwater content self.reporting.reporting(self, pcr, 'StorRootW', self.RootWater * (1 - self.openWaterFrac)) #-Sub soil calculations self.SubWater = self.SubWater + rootperc if self.GroundFLAG == 0: if self.SeepStatFLAG == 0: try: self.SeePage = pcr.readmap( pcrm.generateNameT(self.Seepmaps, self.counter)) self.SeepOld = self.SeePage except: self.SeePage = self.SeepOld #-Report seepage self.reporting.reporting(self, pcr, 'TotSeepF', pcr.scalar(self.SeePage)) self.SubWater = pcr.min(pcr.max(self.SubWater - self.SeePage, 0), self.SubSat) if self.mm_rep_FLAG == 1 and self.Seep_mm_FLAG == 1 and ( self.RoutFLAG == 1 or self.ResFLAG == 1 or self.LakeFLAG == 1): self.SeepSubBasinTSS.sample( pcr.catchmenttotal(self.SeePage, self.FlowDir) / pcr.catchmenttotal(1, self.FlowDir)) #-Capillary rise self.CapRise = self.subzone.CapilRise(pcr, self.SubField, self.SubWater, self.CapRiseMax, self.RootWater, self.RootSat, self.RootField) #-Report capillary rise, corrected for fraction self.reporting.reporting(self, pcr, 'TotCapRF', self.CapRise * (1 - self.GlacFrac)) #-Update sub soil water content self.SubWater = self.SubWater - self.CapRise if self.GroundFLAG == 1: # sub percolation will be calculated instead of subdrainage subperc = self.subzone.SubPercolation(pcr, self.SubWater, self.SubField, self.SubTT, self.Gw, self.GwSat) ActSubPerc = subperc * (1 - self.GlacFrac) #-Report the subzone percolation, corrected for the fraction self.reporting.reporting(self, pcr, 'TotSubPF', ActSubPerc) #-Update sub soil water content self.SubWater = self.SubWater - subperc else: # sub drainage will be calculated instead of sub percolation self.SubDrain = self.subzone.SubDrainage(pcr, self.SubWater, self.SubField, self.SubSat, self.SubDrainVel, self.SubDrain, self.SubTT) #-Report drainage from subzone self.reporting.reporting(self, pcr, 'TotSubDF', self.SubDrain) #-Update sub soil water content self.SubWater = self.SubWater - self.SubDrain #-Report rootwater content self.reporting.reporting(self, pcr, 'StorSubW', self.SubWater * (1 - self.openWaterFrac)) #-Changes in soil water storage OldSoilWater = self.SoilWater self.SoilWater = (self.RootWater + self.SubWater) * (1 - self.GlacFrac) #-Rootzone runoff self.RootRR = RootRunoff * RainFrac * (1 - self.openWaterFrac) #-Report rootzone runoff, corrected for fraction self.reporting.reporting(self, pcr, 'TotRootRF', self.RootRR) #-Rootzone drainage self.RootDR = self.RootDrain * (1 - self.GlacFrac) * ( 1 - self.openWaterFrac) #-Report rootzone drainage, corrected for fraction self.reporting.reporting(self, pcr, 'TotRootDF', self.RootDR) #-Rain runoff self.RainR = self.RootRR + self.RootDR #-Report rain runoff self.reporting.reporting(self, pcr, 'TotRainRF', self.RainR) #-Groundwater calculations if self.GroundFLAG == 1: #-read dynamic processes groundwater self.groundwater.dynamic(self, pcr, ActSubPerc, GlacPerc) else: #-Use drainage from subsoil as baseflow self.BaseR = self.SubDrain #-Groundwater level as scaled between min and max measured gwl SoilAct = self.RootWater + self.SubWater SoilRel = (SoilAct - self.SoilMin) / ( self.SoilMax - self.SoilMin ) # scale between 0 (dry) and 1 (wet) GWL = self.GWL_base - (SoilRel - 0.5) * self.GWL_base #-Report groundwater self.reporting.reporting(self, pcr, 'GWL', GWL) #-Report Total runoff TotR = self.BaseR + self.RainR + self.SnowR + self.GlacR self.reporting.reporting(self, pcr, 'TotRF', TotR) #-Routing for lake and/or reservoir modules if self.LakeFLAG == 1 or self.ResFLAG == 1: #-read dynamic processes advanced routing Q = self.advanced_routing.dynamic(self, pcr, pcrm, config, TotR, self.ETOpenWater, PrecipTot) #-Normal routing module elif self.RoutFLAG == 1: self.routing.dynamic(self, pcr, TotR) if self.GlacFLAG: #-read dynamic reporting processes glacier self.glacier.dynamic_reporting(self, pcr, pd, np) #-Water balance if self.GlacFLAG and self.GlacRetreat == 1: GlacTable_MODid = self.GlacTable.loc[:, ['FRAC_GLAC', 'ICE_DEPTH']] GlacTable_MODid['ICE_DEPTH'] = GlacTable_MODid[ 'ICE_DEPTH'] * GlacTable_MODid['FRAC_GLAC'] GlacTable_MODid = GlacTable_MODid.groupby( GlacTable_MODid.index).sum() GlacTable_MODid.fillna(0., inplace=True) #-Report pcraster map of glacier depth iceDepth = pcr.numpy.zeros(self.ModelID_1d.shape) iceDepth[self.GlacierKeys] = GlacTable_MODid['ICE_DEPTH'] iceDepth = iceDepth.reshape(self.ModelID.shape) iceDepth = pcr.numpy2pcr(pcr.Scalar, iceDepth, self.MV) iceDepth = pcr.ifthen( self.clone, iceDepth) #-only use values where clone is True iceDepth = iceDepth * 1000 # in mm #-change in storage dS = ((self.RootWater - self.oldRootWater) + (self.SubWater - self.oldSubWater)) * (1-self.GlacFrac) + (self.Gw - self.oldGw) + \ (self.TotalSnowStore-OldTotalSnowStore) + (iceDepth - self.oldIceDepth) #-set old state variables for glacier self.oldIceDepth = iceDepth iceDepth = None del iceDepth GlacTable_MODid = None del GlacTable_MODid elif self.GroundFLAG: #-change in storage dS = ((self.RootWater - self.oldRootWater) + (self.SubWater - self.oldSubWater)) * (1-self.GlacFrac) + (self.Gw - self.oldGw) + \ (self.TotalSnowStore-OldTotalSnowStore) # set old state variables for groundwater self.oldGw = self.Gw else: #-change in storage dS = ((self.RootWater - self.oldRootWater) + (self.SubWater - self.oldSubWater)) * (1 - self.GlacFrac) + ( self.TotalSnowStore - OldTotalSnowStore) #-water balance per time step if self.GroundFLAG: waterbalance = Precip - ActETact - self.BaseR - self.RainR - self.SnowR - self.GlacR - dS else: waterbalance = Precip - ActETact - self.BaseR - self.RainR - self.SnowR - dS - self.SeePage self.reporting.reporting(self, pcr, 'wbal', waterbalance) #-total water balance self.waterbalanceTot = self.waterbalanceTot + waterbalance #-report water balance and accumulated water balance if self.wbal_TSS_FLAG and (self.RoutFLAG == 1 or self.ResFLAG == 1 or self.LakeFLAG == 1): self.wbalTSS.sample( pcr.catchmenttotal(waterbalance, self.FlowDir) / pcr.catchmenttotal(1., self.FlowDir)) self.wbalTotTSS.sample( pcr.catchmenttotal(self.waterbalanceTot, self.FlowDir) / pcr.catchmenttotal(1., self.FlowDir)) # set old state variables self.oldRootWater = self.RootWater self.oldSubWater = self.SubWater waterbalance = None del waterbalance dS = None del dS #-End of water balance calculations #-Sediment yield if self.SedFLAG == 1: #-determine runoff in mm per day if self.RoutFLAG == 1 or self.ResFLAG == 1 or self.LakeFLAG == 1: Runoff = (Q * 3600 * 24) / pcr.cellarea() * 1000 else: Runoff = TotR #-MUSLE if self.SedModel == 1: #-read dynamic processes musle self.musle.dynamic(self, pcr, Runoff) #-sediment transport if self.SedTransFLAG == 1: #-read dynamic sediment transport processes musle self.sediment_transport.dynamic_musle(self, pcr) #-Modified Morgan-Morgan-Finney model if self.SedModel == 2: #-determine soil erosion in transport (G) G = self.mmf.dynamic(self, pcr, Precip, Runoff) #-sediment transport if self.SedTransFLAG == 1: #-read dynamic sediment transport processes mmf self.sediment_transport.dynamic_mmf( self, pcr, Runoff, np, G) #-update current date self.curdate = self.curdate + self.datetime.timedelta(days=1)
def netcdf2PCRobjCloneWithoutTime(ncFile,varName, cloneMapFileName = None,\ LatitudeLongitude = True,\ specificFillValue = None): # # EHS (19 APR 2013): To convert netCDF (tss) file to PCR file. # --- with clone checking # Only works if cells are 'square'. # Only works if cellsizeClone <= cellsizeInput # Get netCDF file and variable name: if ncFile in filecache.keys(): f = filecache[ncFile] #~ print "Cached: ", ncFile else: f = nc.Dataset(ncFile) filecache[ncFile] = f #~ print "New: ", ncFile #print ncFile #f = nc.Dataset(ncFile) varName = str(varName) if LatitudeLongitude == True: try: f.variables['lat'] = f.variables['latitude'] f.variables['lon'] = f.variables['longitude'] except: pass sameClone = True # check whether clone and input maps have the same attributes: if cloneMapFileName != None: # get the attributes of cloneMap attributeClone = getMapAttributesALL(cloneMapFileName) cellsizeClone = attributeClone['cellsize'] rowsClone = attributeClone['rows'] colsClone = attributeClone['cols'] xULClone = attributeClone['xUL'] yULClone = attributeClone['yUL'] # get the attributes of input (netCDF) cellsizeInput = f.variables['lat'][0]- f.variables['lat'][1] cellsizeInput = float(cellsizeInput) rowsInput = len(f.variables['lat']) colsInput = len(f.variables['lon']) xULInput = f.variables['lon'][0]-0.5*cellsizeInput yULInput = f.variables['lat'][0]+0.5*cellsizeInput # check whether both maps have the same attributes if cellsizeClone != cellsizeInput: sameClone = False if rowsClone != rowsInput: sameClone = False if colsClone != colsInput: sameClone = False if xULClone != xULInput: sameClone = False if yULClone != yULInput: sameClone = False cropData = f.variables[varName][:,:] # still original data factor = 1 # needed in regridData2FinerGrid if sameClone == False: # crop to cloneMap: minX = min(abs(f.variables['lon'][:] - (xULClone + 0.5*cellsizeInput))) # ; print(minX) xIdxSta = int(np.where(abs(f.variables['lon'][:] - (xULClone + 0.5*cellsizeInput)) == minX)[0]) xIdxEnd = int(math.ceil(xIdxSta + colsClone /(cellsizeInput/cellsizeClone))) minY = min(abs(f.variables['lat'][:] - (yULClone - 0.5*cellsizeInput))) # ; print(minY) yIdxSta = int(np.where(abs(f.variables['lat'][:] - (yULClone - 0.5*cellsizeInput)) == minY)[0]) yIdxEnd = int(math.ceil(yIdxSta + rowsClone /(cellsizeInput/cellsizeClone))) cropData = f.variables[varName][yIdxSta:yIdxEnd,xIdxSta:xIdxEnd] factor = int(round(float(cellsizeInput)/float(cellsizeClone))) # convert to PCR object and close f if specificFillValue != None: outPCR = pcr.numpy2pcr(pcr.Scalar, \ regridData2FinerGrid(factor,cropData,MV), \ float(specificFillValue)) else: outPCR = pcr.numpy2pcr(pcr.Scalar, \ regridData2FinerGrid(factor,cropData,MV), \ float(f.variables[varName]._FillValue)) #~ # debug: #~ pcr.report(outPCR,"tmp.map") #~ print(varName) #~ os.system('aguila tmp.map') #f.close(); f = None ; cropData = None # PCRaster object return (outPCR)
def joinMaps(inputTuple): '''Merges maps starting from an input tuple that specifies the output map name, the number of rows\ and the number rows, columns, ULL X and Y coordinates, cell length and the missing value identifer and a list of input maps''' outputFileName= inputTuple[0] nrRows= inputTuple[1] nrCols= inputTuple[2] xMin= inputTuple[3] yMax= inputTuple[4] cellLength= inputTuple[5] MV= inputTuple[6] fileNames= inputTuple[7] cloneFileName= inputTuple[8] #-echo to screen/logger msg = 'combining files for %s' % outputFileName, logger.info(msg) #-get extent xMax= xMin+nrCols*cellLength yMin= yMax-nrRows*cellLength xCoordinates= xMin+np.arange(nrCols+1)*cellLength yCoordinates= yMin+np.arange(nrRows+1)*cellLength yCoordinates= np.flipud(yCoordinates) msg = 'between %.2f, %.2f and %.2f, %.2f' % (xMin,yMin,xMax,yMax) logger.info(msg) #~ #-set output array #~ variableArray= np.ones((nrRows,nrCols))*MV #-set initial output aaray to zero variableArray= np.zeros((nrRows,nrCols))*MV #-iterate over maps for fileName in fileNames: print fileName attributeClone= getMapAttributesALL(fileName) cellLengthClone= attributeClone['cellsize'] rowsClone= attributeClone['rows'] colsClone= attributeClone['cols'] xULClone= attributeClone['xUL'] yULClone= attributeClone['yUL'] # check whether both maps have the same attributes and process process, nd= checkResolution(cellLength,cellLengthClone) if process: #-get coordinates and locations sampleXMin= xULClone sampleXMax= xULClone+colsClone*cellLengthClone sampleYMin= yULClone-rowsClone*cellLengthClone sampleYMax= yULClone sampleXCoordinates= sampleXMin+np.arange(colsClone+1)*cellLengthClone sampleYCoordinates= sampleYMin+np.arange(rowsClone+1)*cellLengthClone sampleYCoordinates= np.flipud(sampleYCoordinates) sampleXMin= getMax(xMin,sampleXMin) sampleXMax= getMin(xMax,sampleXMax) sampleYMin= getMax(yMin,sampleYMin) sampleYMax= getMin(yMax,sampleYMax) sampleRow0= getPosition(sampleYMin,sampleYCoordinates,nd) sampleRow1= getPosition(sampleYMax,sampleYCoordinates,nd) sampleCol0= getPosition(sampleXMin,sampleXCoordinates,nd) sampleCol1= getPosition(sampleXMax,sampleXCoordinates,nd) sampleRow0, sampleRow1= checkRowPosition(sampleRow0,sampleRow1) variableRow0= getPosition(sampleYMin,yCoordinates,nd) variableRow1= getPosition(sampleYMax,yCoordinates,nd) variableCol0= getPosition(sampleXMin,xCoordinates,nd) variableCol1= getPosition(sampleXMax,xCoordinates,nd) variableRow0,variableRow1= checkRowPosition(variableRow0,variableRow1) #-read sample array setclone(fileName) sampleArray= pcr2numpy(readmap(fileName),MV) print sampleArray sampleNrRows, sampleNrCols= sampleArray.shape # -create mask #~ mask= (variableArray[variableRow0:variableRow1,variableCol0:variableCol1] == MV) &\ #~ (sampleArray[sampleRow0:sampleRow1,sampleCol0:sampleCol1] <> MV) mask= (variableArray[variableRow0:variableRow1,variableCol0:variableCol1] <> MV) &\ (sampleArray[sampleRow0:sampleRow1,sampleCol0:sampleCol1] <> MV) #-add values msg = ' adding values in %d, %d rows, columns from (x, y) %.3f, %.3f and %.3f, %.3f to position (row, col) %d, %d and %d, %d' %\ (sampleNrRows, sampleNrCols,sampleXMin,sampleYMin,sampleXMax,sampleYMax,variableRow0,variableCol0,variableRow1,variableCol1) logger.info(msg) #~ variableArray[variableRow0:variableRow1,variableCol0:variableCol1][mask]= \ #~ sampleArray[sampleRow0:sampleRow1,sampleCol0:sampleCol1][mask] variableArray[variableRow0:variableRow1,variableCol0:variableCol1][mask] += sampleArray[sampleRow0:sampleRow1,sampleCol0:sampleCol1][mask] else: msg = '%s does not match resolution and is not processed' % fileName logger.warning(msg) #-report output map setclone(cloneFileName) report(numpy2pcr(Scalar,variableArray,MV),outputFileName)
def complexreservoir( waterlevel, ReserVoirLocs, LinkedReserVoirLocs, ResArea, ResThreshold, ResStorFunc, ResOutflowFunc, sh, hq, res_b, res_e, inflow, precip, pet, ReservoirComplexAreas, JDOY, timestepsecs=86400, ): mv = -999.0 inflow = pcr.ifthen(pcr.boolean(ReserVoirLocs), inflow) prec_av = pcr.ifthen(pcr.boolean(ReserVoirLocs), pcr.areaaverage(precip, ReservoirComplexAreas)) pet_av = pcr.ifthen(pcr.boolean(ReserVoirLocs), pcr.areaaverage(pet, ReservoirComplexAreas)) np_reslocs = pcr.pcr2numpy(ReserVoirLocs, 0.0) np_linkedreslocs = pcr.pcr2numpy(LinkedReserVoirLocs, 0.0) _outflow = [] nr_loop = np.max([int(timestepsecs / 21600), 1]) for n in range(0, nr_loop): np_waterlevel = pcr.pcr2numpy(waterlevel, np.nan) np_waterlevel_lower = np_waterlevel.copy() for val in np.unique(np_linkedreslocs): if val > 0: np_waterlevel_lower[np_linkedreslocs == val] = np_waterlevel[ np.where(np_reslocs == val)] diff_wl = np_waterlevel - np_waterlevel_lower diff_wl[np.isnan(diff_wl)] = mv np_waterlevel_lower[np.isnan(np_waterlevel_lower)] = mv pcr_diff_wl = pcr.numpy2pcr(pcr.Scalar, diff_wl, mv) pcr_wl_lower = pcr.numpy2pcr(pcr.Scalar, np_waterlevel_lower, mv) storage_start = pcr.ifthenelse( ResStorFunc == 1, ResArea * waterlevel, lookupResFunc(ReserVoirLocs, waterlevel, sh, "0-1"), ) outflow = pcr.ifthenelse( ResOutflowFunc == 1, lookupResRegMatr(ReserVoirLocs, waterlevel, hq, JDOY), pcr.ifthenelse( pcr_diff_wl >= 0, pcr.max(res_b * (waterlevel - ResThreshold)**res_e, 0), pcr.min(-1 * res_b * (pcr_wl_lower - ResThreshold)**res_e, 0), ), ) np_outflow = pcr.pcr2numpy(outflow, np.nan) np_outflow_linked = np_reslocs * 0.0 with np.errstate(invalid="ignore"): if np_outflow[np_outflow < 0] is not None: np_outflow_linked[np.in1d( np_reslocs, np_linkedreslocs[np_outflow < 0]).reshape( np_linkedreslocs.shape)] = np_outflow[np_outflow < 0] outflow_linked = pcr.numpy2pcr(pcr.Scalar, np_outflow_linked, 0.0) fl_nr_loop = float(nr_loop) storage = ( storage_start + (inflow * timestepsecs / fl_nr_loop) + (prec_av / fl_nr_loop / 1000.0) * ResArea - (pet_av / fl_nr_loop / 1000.0) * ResArea - (pcr.cover(outflow, 0.0) * timestepsecs / fl_nr_loop) + (pcr.cover(outflow_linked, 0.0) * timestepsecs / fl_nr_loop)) waterlevel = pcr.ifthenelse( ResStorFunc == 1, waterlevel + (storage - storage_start) / ResArea, lookupResFunc(ReserVoirLocs, storage, sh, "1-0"), ) np_outflow_nz = np_outflow * 0.0 with np.errstate(invalid="ignore"): np_outflow_nz[np_outflow > 0] = np_outflow[np_outflow > 0] _outflow.append(np_outflow_nz) outflow_av_temp = np.average(_outflow, 0) outflow_av_temp[np.isnan(outflow_av_temp)] = mv outflow_av = pcr.numpy2pcr(pcr.Scalar, outflow_av_temp, mv) return waterlevel, outflow_av, prec_av, pet_av, storage
def test_numpy2pcr(self): nrRows, nrCols, cellSize = 3, 2, 1.0 west, north = 0.0, 0.0 pcraster.setclone(nrRows, nrCols, cellSize, west, north) # Values in array must fit the value scale of the raster exactly. This # will be checked. # Valid boolean values are: 0, 1, missing_value. # Valid ldd values are: 1, 2, 3, 4, 5, 6, 7, 8, 9, missing_value. # Valid nominal values are: [-2^31 + 1, 2^31], missing_value. # Valid ordinal values are: [-2^31 + 1, 2^31], missing_value. # Valid scalar values are: All 32 bit float values. # Valid directional values are: All 32 bit float values. # bool_min = 0 # bool_max = 1 int8_min = numpy.iinfo(numpy.int8).min int8_max = numpy.iinfo(numpy.int8).max int32_min = numpy.iinfo(numpy.int32).min int32_max = numpy.iinfo(numpy.int32).max int64_min = numpy.iinfo(numpy.int64).min int64_max = numpy.iinfo(numpy.int64).max # bool -> Boolean (uint8) raster = pcraster.numpy2pcr(pcraster.Boolean, numpy.array([ [1, 1], [0, 5], [1, 1]], numpy.bool), 5) self.assertEqual(pcraster.cellvalue(raster, 1, 1), (True, True)) self.assertEqual(pcraster.cellvalue(raster, 2, 1), (False, True)) # It is not possible to create a bool array with other values than # 0 and 1. Passing 5 as missing value has no effect. self.assertEqual(pcraster.cellvalue(raster, 2, 2), (True, True)) # int8 -> Boolean (uint8) raster = pcraster.numpy2pcr(pcraster.Boolean, numpy.array([ [1, 1], [0, 5], [1, 1]], numpy.int8), 5) self.assertEqual(pcraster.cellvalue(raster, 1, 1), (True, True)) self.assertEqual(pcraster.cellvalue(raster, 2, 1), (False, True)) self.assertEqual(pcraster.cellvalue(raster, 2, 2)[1], False) with self.assertRaises(Exception) as context_manager: raster = pcraster.numpy2pcr(pcraster.Boolean, numpy.array([ [1, 1], [9, 5], [1, 1]], numpy.int8), 5) self.assertEqual(str(context_manager.exception), "Incorrect value 9 at input array [1][0] for Boolean map") # int8 -> Ldd (uint8) raster = pcraster.numpy2pcr(pcraster.Ldd, numpy.array([ [1, 2], [5, 15], [8, 9]], numpy.int8), 15) self.assertEqual(pcraster.cellvalue(raster, 1, 1), (1, True)) self.assertEqual(pcraster.cellvalue(raster, 2, 1), (5, True)) self.assertEqual(pcraster.cellvalue(raster, 2, 2)[1], False) with self.assertRaises(Exception) as context_manager: raster = pcraster.numpy2pcr(pcraster.Ldd, numpy.array([ [ 1, 2], [10, 15], [ 8, 9]], numpy.int8), 15) self.assertEqual(str(context_manager.exception), "Incorrect value 10 at input array [1][0] for LDD map") # int8 -> Nominal (int32) # All valid int8 values are valid nominal values. raster = pcraster.numpy2pcr(pcraster.Nominal, numpy.array([ [int8_min, 2], [ 5, 15], [ 8, int8_max]], numpy.int8), 15) self.assertEqual(pcraster.cellvalue(raster, 1, 1), (int8_min, True)) self.assertEqual(pcraster.cellvalue(raster, 2, 1), ( 5, True)) self.assertEqual(pcraster.cellvalue(raster, 2, 2)[1], False) self.assertEqual(pcraster.cellvalue(raster, 3, 2), (int8_max, True)) # int8 -> Ordinal (int32) # All valid int8 values are valid ordinal values. raster = pcraster.numpy2pcr(pcraster.Ordinal, numpy.array([ [int8_min, 2], [ 5, 15], [ 8, int8_max]], numpy.int8), 15) self.assertEqual(pcraster.cellvalue(raster, 1, 1), (int8_min, True)) self.assertEqual(pcraster.cellvalue(raster, 2, 1), ( 5, True)) self.assertEqual(pcraster.cellvalue(raster, 2, 2)[1], False) self.assertEqual(pcraster.cellvalue(raster, 3, 2), (int8_max, True)) # int8 -> Scalar (float32) # All valid int8 values are valid scalar values. raster = pcraster.numpy2pcr(pcraster.Scalar, numpy.array([ [int8_min, 2], [ 5, 15], [ 8, int8_max]], numpy.int8), 15) self.assertEqual(pcraster.cellvalue(raster, 1, 1), (int8_min, True)) self.assertEqual(pcraster.cellvalue(raster, 2, 1), ( 5, True)) self.assertEqual(pcraster.cellvalue(raster, 2, 2)[1], False) self.assertEqual(pcraster.cellvalue(raster, 3, 2), (int8_max, True)) # int8 -> Directional (float32) # All valid int8 values are valid directional values. raster = pcraster.numpy2pcr(pcraster.Directional, numpy.array([ [int8_min, 2], [ 5, 15], [ 8, int8_max]], numpy.int8), 15) self.assertEqual(pcraster.cellvalue(raster, 1, 1), (int8_min, True)) self.assertEqual(pcraster.cellvalue(raster, 2, 1), ( 5, True)) self.assertEqual(pcraster.cellvalue(raster, 2, 2)[1], False) self.assertEqual(pcraster.cellvalue(raster, 3, 2), (int8_max, True)) # int16 TODO # int32 TODO # int64 -> Boolean (uint8) raster = pcraster.numpy2pcr(pcraster.Boolean, numpy.array([ [1, 1], [0, 5], [1, 1]], numpy.int64), 5) self.assertEqual(pcraster.cellvalue(raster, 1, 1), (True, True)) self.assertEqual(pcraster.cellvalue(raster, 2, 1), (False, True)) self.assertEqual(pcraster.cellvalue(raster, 2, 2)[1], False) with self.assertRaises(Exception) as context_manager: raster = pcraster.numpy2pcr(pcraster.Boolean, numpy.array([ [1, 1], [9, 5], [1, 1]], numpy.int64), 5) self.assertEqual(str(context_manager.exception), "Incorrect value 9 at input array [1][0] for Boolean map") # int64 -> Ldd (uint8) raster = pcraster.numpy2pcr(pcraster.Ldd, numpy.array([ [1, 2], [5, 15], [8, 9]], numpy.int64), 15) self.assertEqual(pcraster.cellvalue(raster, 1, 1), (1, True)) self.assertEqual(pcraster.cellvalue(raster, 2, 1), (5, True)) self.assertEqual(pcraster.cellvalue(raster, 2, 2)[1], False) with self.assertRaises(Exception) as context_manager: raster = pcraster.numpy2pcr(pcraster.Ldd, numpy.array([ [ 1, 2], [10, 15], [ 8, 9]], numpy.int64), 15) self.assertEqual(str(context_manager.exception), "Incorrect value 10 at input array [1][0] for LDD map") # int64 -> Nominal (int32) raster = pcraster.numpy2pcr(pcraster.Nominal, numpy.array([ [int32_min+1, 2], [ 5, 15], [ 8, int32_max]], numpy.int64), 15) self.assertEqual(pcraster.cellvalue(raster, 1, 1), (int32_min + 1, True)) self.assertEqual(pcraster.cellvalue(raster, 2, 1), ( 5, True)) self.assertEqual(pcraster.cellvalue(raster, 2, 2)[1], False) self.assertEqual(pcraster.cellvalue(raster, 3, 2), ( int32_max, True)) with self.assertRaises(Exception) as context_manager: raster = pcraster.numpy2pcr(pcraster.Nominal, numpy.array([ [int32_min+1, 2], [ int32_min, 15], [ 8, int32_max]], numpy.int64), 15) self.assertEqual(str(context_manager.exception), "Incorrect value -2147483648 at input array [1][0] for Nominal map") # uint8 TODO # uint16 TODO # uint32 TODO # uint64 TODO # float16 TODO # float32 TODO # float64 TODO # complex64: Not supported. with self.assertRaises(Exception) as context_manager: raster = pcraster.numpy2pcr(pcraster.Nominal, numpy.array([ [-2, -1], [ 0, 15], [ 1, 2]], numpy.complex64), 15) self.assertEqual(str(context_manager.exception), "Unsupported array type")
def main(): ### Read input arguments ##### parser = OptionParser() usage = "usage: %prog [options]" parser = OptionParser(usage=usage) parser.add_option('-q', '--quiet', dest='verbose', default=True, action='store_false', help='do not print status messages to stdout') parser.add_option('-i', '--ini', dest='inifile', default='hand_contour_inun.ini', nargs=1, help='ini configuration file') parser.add_option('-f', '--flood_map', nargs=1, dest='flood_map', help='Flood map file (NetCDF point time series file') parser.add_option('-v', '--flood_variable', nargs=1, dest='flood_variable', default='water_level', help='variable name of flood water level') parser.add_option('-b', '--bankfull_map', dest='bankfull_map', default='', help='Map containing bank full level (is subtracted from flood map, in NetCDF)') parser.add_option('-c', '--catchment', dest='catchment_strahler', default=7, type='int', help='Strahler order threshold >= are selected as catchment boundaries') parser.add_option('-s', '--hand_strahler', dest='hand_strahler', default=7, type='int', help='Strahler order threshold >= selected as riverine') parser.add_option('-d', '--destination', dest='dest_path', default='inun', help='Destination path') (options, args) = parser.parse_args() if not os.path.exists(options.inifile): print 'path to ini file cannot be found' sys.exit(1) options.dest_path = os.path.abspath(options.dest_path) if not(os.path.isdir(options.dest_path)): os.makedirs(options.dest_path) # set up the logger flood_name = os.path.split(options.flood_map)[1].split('.')[0] case_name = 'inun_{:s}_hand_{:02d}_catch_{:02d}'.format(flood_name, options.hand_strahler, options.catchment_strahler) logfilename = os.path.join(options.dest_path, 'hand_contour_inun.log') logger, ch = inun_lib.setlogger(logfilename, 'HAND_INUN', options.verbose) logger.info('$Id: $') logger.info('Flood map: {:s}'.format(options.flood_map)) logger.info('Bank full map: {:s}'.format(options.bankfull_map)) logger.info('Destination path: {:s}'.format(options.dest_path)) # read out ini file ### READ CONFIG FILE # open config-file config = inun_lib.open_conf(options.inifile) # read settings options.dem_file = inun_lib.configget(config, 'maps', 'dem_file', True) options.ldd_file = inun_lib.configget(config, 'maps', 'ldd_file', True) options.stream_file = inun_lib.configget(config, 'maps', 'stream_file', True) options.riv_length_file = inun_lib.configget(config, 'maps', 'riv_length_file', True) options.riv_width_file = inun_lib.configget(config, 'maps', 'riv_width_file', True) options.file_format = inun_lib.configget(config, 'maps', 'file_format', 0, datatype='int') options.x_tile = inun_lib.configget(config, 'tiling', 'x_tile', 10000, datatype='int') options.y_tile = inun_lib.configget(config, 'tiling', 'y_tile', 10000, datatype='int') options.x_overlap = inun_lib.configget(config, 'tiling', 'x_overlap', 1000, datatype='int') options.y_overlap = inun_lib.configget(config, 'tiling', 'y_overlap', 1000, datatype='int') options.iterations = inun_lib.configget(config, 'inundation', 'iterations', 20, datatype='int') options.initial_level = inun_lib.configget(config, 'inundation', 'initial_level', 32., datatype='float') options.area_multiplier = inun_lib.configget(config, 'inundation', 'area_multiplier', 1., datatype='float') logger.info('DEM file: {:s}'.format(options.dem_file)) logger.info('LDD file: {:s}'.format(options.ldd_file)) logger.info('Columns per tile: {:d}'.format(options.x_tile)) logger.info('Rows per tile: {:d}'.format(options.y_tile)) logger.info('Columns overlap: {:d}'.format(options.x_overlap)) logger.info('Rows overlap: {:d}'.format(options.y_overlap)) metadata_global = {} # add metadata from the section [metadata] meta_keys = config.options('metadata_global') for key in meta_keys: metadata_global[key] = config.get('metadata_global', key) # add a number of metadata variables that are mandatory metadata_global['config_file'] = os.path.abspath(options.inifile) metadata_var = {} metadata_var['units'] = 'm' metadata_var['standard_name'] = 'water_surface_height_above_reference_datum' metadata_var['long_name'] = 'Coastal flooding' metadata_var['comment'] = 'water_surface_reference_datum_altitude is given in file {:s}'.format(options.dem_file) if not os.path.exists(options.dem_file): logger.error('path to dem file {:s} cannot be found'.format(options.dem_file)) sys.exit(1) if not os.path.exists(options.ldd_file): logger.error('path to ldd file {:s} cannot be found'.format(options.ldd_file)) sys.exit(1) # Read extent from a GDAL compatible file try: extent = inun_lib.get_gdal_extent(options.dem_file) except: msg = 'Input file {:s} not a gdal compatible file'.format(options.dem_file) inun_lib.close_with_error(logger, ch, msg) sys.exit(1) try: x, y = inun_lib.get_gdal_axes(options.dem_file, logging=logger) srs = inun_lib.get_gdal_projection(options.dem_file, logging=logger) except: msg = 'Input file {:s} not a gdal compatible file'.format(options.dem_file) inun_lib.close_with_error(logger, ch, msg) sys.exit(1) # read history from flood file if options.file_format == 0: a = nc.Dataset(options.flood_map, 'r') metadata_global['history'] = 'Created by: $Id: $, boundary conditions from {:s},\nhistory: {:s}'.format(os.path.abspath(options.flood_map), a.history) a.close() else: metadata_global['history'] = 'Created by: $Id: $, boundary conditions from {:s},\nhistory: {:s}'.format(os.path.abspath(options.flood_map), 'PCRaster file, no history') # first write subcatch maps and hand maps ############### TODO ###### # setup a HAND file dem_name = os.path.split(options.dem_file)[1].split('.')[0] hand_file = os.path.join(options.dest_path, '{:s}_hand_strahler_{:02d}.tif'.format(dem_name, options.hand_strahler)) if not(os.path.isfile(hand_file)): # hand file does not exist yet! Generate it, otherwise skip! logger.info('HAND file {:s} setting up...please wait...'.format(hand_file)) hand_file_tmp = os.path.join(options.dest_path, '{:s}_hand_strahler_{:02d}.tif.tmp'.format(dem_name, options.hand_strahler)) ds_hand = inun_lib.prepare_gdal(hand_file_tmp, x, y, logging=logger, srs=srs) band_hand = ds_hand.GetRasterBand(1) # Open terrain data for reading ds_dem, rasterband_dem = inun_lib.get_gdal_rasterband(options.dem_file) ds_ldd, rasterband_ldd = inun_lib.get_gdal_rasterband(options.ldd_file) ds_stream, rasterband_stream = inun_lib.get_gdal_rasterband(options.stream_file) n = 0 for x_loop in range(0, len(x), options.x_tile): x_start = np.maximum(x_loop, 0) x_end = np.minimum(x_loop + options.x_tile, len(x)) # determine actual overlap for cutting for y_loop in range(0, len(y), options.y_tile): x_overlap_min = x_start - np.maximum(x_start - options.x_overlap, 0) x_overlap_max = np.minimum(x_end + options.x_overlap, len(x)) - x_end n += 1 # print('tile {:001d}:'.format(n)) y_start = np.maximum(y_loop, 0) y_end = np.minimum(y_loop + options.y_tile, len(y)) y_overlap_min = y_start - np.maximum(y_start - options.y_overlap, 0) y_overlap_max = np.minimum(y_end + options.y_overlap, len(y)) - y_end # cut out DEM logger.debug('Computing HAND for xmin: {:d} xmax: {:d} ymin {:d} ymax {:d}'.format(x_start, x_end,y_start, y_end)) terrain = rasterband_dem.ReadAsArray(x_start - x_overlap_min, y_start - y_overlap_min, (x_end + x_overlap_max) - (x_start - x_overlap_min), (y_end + y_overlap_max) - (y_start - y_overlap_min) ) drainage = rasterband_ldd.ReadAsArray(x_start - x_overlap_min, y_start - y_overlap_min, (x_end + x_overlap_max) - (x_start - x_overlap_min), (y_end + y_overlap_max) - (y_start - y_overlap_min) ) stream = rasterband_stream.ReadAsArray(x_start - x_overlap_min, y_start - y_overlap_min, (x_end + x_overlap_max) - (x_start - x_overlap_min), (y_end + y_overlap_max) - (y_start - y_overlap_min) ) # write to temporary file terrain_temp_file = os.path.join(options.dest_path, 'terrain_temp.map') drainage_temp_file = os.path.join(options.dest_path, 'drainage_temp.map') stream_temp_file = os.path.join(options.dest_path, 'stream_temp.map') if rasterband_dem.GetNoDataValue() is not None: inun_lib.gdal_writemap(terrain_temp_file, 'PCRaster', np.arange(0, terrain.shape[1]), np.arange(0, terrain.shape[0]), terrain, rasterband_dem.GetNoDataValue(), gdal_type=gdal.GDT_Float32, logging=logger) else: # in case no nodata value is found logger.warning('No nodata value found in {:s}. assuming -9999'.format(options.dem_file)) inun_lib.gdal_writemap(terrain_temp_file, 'PCRaster', np.arange(0, terrain.shape[1]), np.arange(0, terrain.shape[0]), terrain, -9999., gdal_type=gdal.GDT_Float32, logging=logger) inun_lib.gdal_writemap(drainage_temp_file, 'PCRaster', np.arange(0, terrain.shape[1]), np.arange(0, terrain.shape[0]), drainage, rasterband_ldd.GetNoDataValue(), gdal_type=gdal.GDT_Int32, logging=logger) inun_lib.gdal_writemap(stream_temp_file, 'PCRaster', np.arange(0, terrain.shape[1]), np.arange(0, terrain.shape[0]), stream, rasterband_ldd.GetNoDataValue(), gdal_type=gdal.GDT_Int32, logging=logger) # read as pcr objects pcr.setclone(terrain_temp_file) terrain_pcr = pcr.readmap(terrain_temp_file) drainage_pcr = pcr.lddrepair(pcr.ldd(pcr.readmap(drainage_temp_file))) # convert to ldd type map stream_pcr = pcr.scalar(pcr.readmap(stream_temp_file)) # convert to ldd type map # compute streams stream_ge, subcatch = inun_lib.subcatch_stream(drainage_pcr, stream_pcr, options.hand_strahler) # generate streams basin = pcr.boolean(subcatch) hand_pcr, dist_pcr = inun_lib.derive_HAND(terrain_pcr, drainage_pcr, 3000, rivers=pcr.boolean(stream_ge), basin=basin) # convert to numpy hand = pcr.pcr2numpy(hand_pcr, -9999.) # cut relevant part if y_overlap_max == 0: y_overlap_max = -hand.shape[0] if x_overlap_max == 0: x_overlap_max = -hand.shape[1] hand_cut = hand[0+y_overlap_min:-y_overlap_max, 0+x_overlap_min:-x_overlap_max] band_hand.WriteArray(hand_cut, x_start, y_start) os.unlink(terrain_temp_file) os.unlink(drainage_temp_file) band_hand.FlushCache() ds_dem = None ds_ldd = None ds_stream = None band_hand.SetNoDataValue(-9999.) ds_hand = None logger.info('Finalizing {:s}'.format(hand_file)) # rename temporary file to final hand file os.rename(hand_file_tmp, hand_file) else: logger.info('HAND file {:s} already exists...skipping...'.format(hand_file)) ##################################################################################### # HAND file has now been prepared, moving to flood mapping part # ##################################################################################### # load the staticmaps needed to estimate volumes across all xax, yax, riv_length, fill_value = inun_lib.gdal_readmap(options.riv_length_file, 'GTiff') riv_length = np.ma.masked_where(riv_length==fill_value, riv_length) xax, yax, riv_width, fill_value = inun_lib.gdal_readmap(options.riv_width_file, 'GTiff') riv_width[riv_width == fill_value] = 0 x_res = np.abs((xax[-1]-xax[0])/(len(xax)-1)) y_res = np.abs((yax[-1]-yax[0])/(len(yax)-1)) flood_folder = os.path.join(options.dest_path, case_name) flood_vol_map = os.path.join(flood_folder, '{:s}_vol.tif'.format(os.path.split(options.flood_map)[1].split('.')[0])) if not(os.path.isdir(flood_folder)): os.makedirs(flood_folder) inun_file_tmp = os.path.join(flood_folder, '{:s}.tif.tmp'.format(case_name)) inun_file = os.path.join(flood_folder, '{:s}.tif'.format(case_name)) hand_temp_file = os.path.join(flood_folder, 'hand_temp.map') drainage_temp_file = os.path.join(flood_folder, 'drainage_temp.map') stream_temp_file = os.path.join(flood_folder, 'stream_temp.map') flood_vol_temp_file = os.path.join(flood_folder, 'flood_warp_temp.tif') # load the data with river levels and compute the volumes if options.file_format == 0: # assume we need the maximum value in a NetCDF time series grid a = nc.Dataset(options.flood_map, 'r') xax = a.variables['x'][:] yax = a.variables['y'][:] flood_series = a.variables[options.flood_variable][:] flood_data = flood_series.max(axis=0) if np.ma.is_masked(flood_data): flood = flood_data.data flood[flood_data.mask] = 0 if yax[-1] > yax[0]: yax = np.flipud(yax) flood = np.flipud(flood) a.close() elif options.file_format == 1: xax, yax, flood, flood_fill_value = inun_lib.gdal_readmap(options.flood_map, 'PCRaster') flood[flood==flood_fill_value] = 0. #res_x = x[1]-x[0] #res_y = y[1]-y[0] # load the bankfull depths if options.bankfull_map == '': bankfull = np.zeros(flood.shape) else: if options.file_format == 0: a = nc.Dataset(options.bankfull_map, 'r') xax = a.variables['x'][:] yax = a.variables['y'][:] bankfull = a.variables[options.flood_variable][0, :, :] if yax[-1] > yax[0]: yax = np.flipud(yax) bankfull = np.flipud(bankful) a.close() elif options.file_format == 1: xax, yax, bankfull, bankfull_fill_value = inun_lib.gdal_readmap(options.bankfull_map, 'PCRaster') # flood = bankfull*2 # res_x = 2000 # res_y = 2000 # subtract the bankfull water level to get flood levels (above bankfull) flood_vol = np.maximum(flood-bankfull, 0) flood_vol_m = riv_length*riv_width*flood_vol/(x_res * y_res) # volume expressed in meters water disc (1e6 is the surface area of one wflow grid cell) flood_vol_m_data = flood_vol_m.data flood_vol_m_data[flood_vol_m.mask] = -999. print('Saving water layer map to {:s}'.format(flood_vol_map)) # write to a tiff file inun_lib.gdal_writemap(flood_vol_map, 'GTiff', xax, yax, np.maximum(flood_vol_m_data, 0), -999.) ds_hand, rasterband_hand = inun_lib.get_gdal_rasterband(hand_file) ds_ldd, rasterband_ldd = inun_lib.get_gdal_rasterband(options.ldd_file) ds_stream, rasterband_stream = inun_lib.get_gdal_rasterband(options.stream_file) logger.info('Preparing flood map in {:s} ...please wait...'.format(inun_file)) ds_inun = inun_lib.prepare_gdal(inun_file_tmp, x, y, logging=logger, srs=srs) band_inun = ds_inun.GetRasterBand(1) # loop over all the tiles n = 0 for x_loop in range(0, len(x), options.x_tile): x_start = np.maximum(x_loop, 0) x_end = np.minimum(x_loop + options.x_tile, len(x)) # determine actual overlap for cutting for y_loop in range(0, len(y), options.y_tile): x_overlap_min = x_start - np.maximum(x_start - options.x_overlap, 0) x_overlap_max = np.minimum(x_end + options.x_overlap, len(x)) - x_end n += 1 # print('tile {:001d}:'.format(n)) y_start = np.maximum(y_loop, 0) y_end = np.minimum(y_loop + options.y_tile, len(y)) y_overlap_min = y_start - np.maximum(y_start - options.y_overlap, 0) y_overlap_max = np.minimum(y_end + options.y_overlap, len(y)) - y_end x_tile_ax = x[x_start - x_overlap_min:x_end + x_overlap_max] y_tile_ax = y[y_start - y_overlap_min:y_end + y_overlap_max] # cut out DEM logger.debug('handling xmin: {:d} xmax: {:d} ymin {:d} ymax {:d}'.format(x_start, x_end, y_start, y_end)) hand = rasterband_hand.ReadAsArray(x_start - x_overlap_min, y_start - y_overlap_min, (x_end + x_overlap_max) - (x_start - x_overlap_min), (y_end + y_overlap_max) - (y_start - y_overlap_min) ) drainage = rasterband_ldd.ReadAsArray(x_start - x_overlap_min, y_start - y_overlap_min, (x_end + x_overlap_max) - (x_start - x_overlap_min), (y_end + y_overlap_max) - (y_start - y_overlap_min) ) stream = rasterband_stream.ReadAsArray(x_start - x_overlap_min, y_start - y_overlap_min, (x_end + x_overlap_max) - (x_start - x_overlap_min), (y_end + y_overlap_max) - (y_start - y_overlap_min) ) print('len x-ax: {:d} len y-ax {:d} x-shape {:d} y-shape {:d}'.format(len(x_tile_ax), len(y_tile_ax), hand.shape[1], hand.shape[0])) inun_lib.gdal_writemap(hand_temp_file, 'PCRaster', x_tile_ax, y_tile_ax, hand, rasterband_hand.GetNoDataValue(), gdal_type=gdal.GDT_Float32, logging=logger) inun_lib.gdal_writemap(drainage_temp_file, 'PCRaster', x_tile_ax, y_tile_ax, drainage, rasterband_ldd.GetNoDataValue(), gdal_type=gdal.GDT_Int32, logging=logger) inun_lib.gdal_writemap(stream_temp_file, 'PCRaster', x_tile_ax, y_tile_ax, stream, rasterband_stream.GetNoDataValue(), gdal_type=gdal.GDT_Int32, logging=logger) # read as pcr objects pcr.setclone(hand_temp_file) hand_pcr = pcr.readmap(hand_temp_file) drainage_pcr = pcr.lddrepair(pcr.ldd(pcr.readmap(drainage_temp_file))) # convert to ldd type map stream_pcr = pcr.scalar(pcr.readmap(drainage_temp_file)) # convert to ldd type map # prepare a subcatchment map stream_ge, subcatch = inun_lib.subcatch_stream(drainage_pcr, stream_pcr, options.catchment_strahler) # generate subcatchments drainage_surf = pcr.ifthen(stream_ge > 0, pcr.accuflux(drainage_pcr, 1)) # proxy of drainage surface inaccurate at tile edges # compute weights for spreadzone (1/drainage_surf) subcatch = pcr.spreadzone(subcatch, 0, 0) # TODO check weighting scheme, perhaps not necessary # weight = 1./pcr.scalar(pcr.spreadzone(pcr.cover(pcr.ordinal(drainage_surf), 0), 0, 0)) # subcatch_fill = pcr.scalar(pcr.spreadzone(subcatch, 0, weight)) # # cover subcatch with subcatch_fill # pcr.report(weight, 'weight_{:02d}.map'.format(n)) # pcr.report(subcatch, 'subcatch_{:02d}.map'.format(n)) # pcr.report(pcr.nominal(subcatch_fill), 'subcatch_fill_{:02d}.map'.format(n)) inun_lib.gdal_warp(flood_vol_map, hand_temp_file, flood_vol_temp_file, gdal_interp=gdalconst.GRA_NearestNeighbour) # , x_tile_ax, y_tile_ax, flood_meter, fill_value = inun_lib.gdal_readmap(flood_vol_temp_file, 'GTiff') # convert meter depth to volume [m3] flood_vol = pcr.numpy2pcr(pcr.Scalar, flood_meter, fill_value)*((x_tile_ax[1] - x_tile_ax[0]) * (y_tile_ax[0] - y_tile_ax[1])) # resolution of SRTM *1166400000. ## now we have some nice volume. Now we need to redistribute! inundation_pcr = inun_lib.volume_spread(drainage_pcr, hand_pcr, subcatch, flood_vol, volume_thres=0., iterations=options.iterations, area_multiplier=options.area_multiplier) # 1166400000. inundation = pcr.pcr2numpy(inundation_pcr, -9999.) # cut relevant part if y_overlap_max == 0: y_overlap_max = -inundation.shape[0] if x_overlap_max == 0: x_overlap_max = -inundation.shape[1] inundation_cut = inundation[0+y_overlap_min:-y_overlap_max, 0+x_overlap_min:-x_overlap_max] # inundation_cut band_inun.WriteArray(inundation_cut, x_start, y_start) band_inun.FlushCache() # clean up os.unlink(flood_vol_temp_file) os.unlink(drainage_temp_file) os.unlink(hand_temp_file) # if n == 35: # band_inun.SetNoDataValue(-9999.) # ds_inun = None # sys.exit(0) os.unlink(flood_vol_map) logger.info('Finalizing {:s}'.format(inun_file)) # add the metadata to the file and band band_inun.SetNoDataValue(-9999.) ds_inun.SetMetadata(metadata_global) band_inun.SetMetadata(metadata_var) ds_inun = None ds_hand = None ds_ldd = None # rename temporary file to final hand file if os.path.isfile(inun_file): # remove an old result if available os.unlink(inun_file) os.rename(inun_file_tmp, inun_file) logger.info('Done! Thank you for using hand_contour_inun.py') logger, ch = inun_lib.closeLogger(logger, ch) del logger, ch sys.exit(0)
def loadsetclone(name): """ Load 'MaskMap' and set as clone :param name: name of the key in Settings.xml containing path and name of mask map as string :return: map: mask map (False=include in modelling; True=exclude from modelling) as pcraster """ settings = LisSettings.instance() binding = settings.binding flags = settings.flags filename = os.path.normpath(binding[name]) if not os.path.exists(filename): raise LisfloodError('File not existing: {}'.format(filename)) coord = filename.split() # returns a list of all the words in the string if len(coord) == 5: # changed order of x, y i- in setclone y is first in Lisflood # settings x is first # setclone row col cellsize xupleft yupleft try: setclone(int(coord[1]), int(coord[0]), float(coord[2]), float(coord[3]), float(coord[4])) # CM: pcraster except: rem = "[" + str(coord[0]) + " " + str(coord[1]) + " " + str( coord[2]) + " " + str(coord[3]) + " " + str(coord[4]) + "]" msg = "Maskmap: " + rem + \ " are not valid coordinates (col row cellsize xupleft yupleft)" raise LisfloodError(msg) mapnp = np.ones((int(coord[1]), int(coord[0]))) map_out = numpy2pcr(Boolean, mapnp, -9999) elif len(coord) == 1: # read information on clone map from map (pcraster or netcdf) try: # try to read a pcraster map iterSetClonePCR(filename) map_out = pcraster.boolean(iterReadPCRasterMap(filename)) flagmap = True mapnp = pcr2numpy(map_out, np.nan) except Exception as e: # FIXME manage exceptions and print type of error # print(str(e)) # print(type(e)) # try to read a netcdf file filename = os.path.splitext(binding[name])[0] + '.nc' nf1 = iterOpenNetcdf(filename, "", "r") value = listitems( nf1.variables)[-1][0] # get the last variable name if 'x' in nf1.variables: x1 = nf1.variables['x'][0] x2 = nf1.variables['x'][1] y1 = nf1.variables['y'][0] else: x1 = nf1.variables['lon'][0] x2 = nf1.variables['lon'][1] y1 = nf1.variables['lat'][0] cell_size = round(np.abs(x2 - x1), 4) nr_rows, nr_cols = nf1.variables[ value].shape # just use shape to know rows and cols... x = x1 - cell_size / 2 y = y1 + cell_size / 2 mapnp = np.array(nf1.variables[value][0:nr_rows, 0:nr_cols]) nf1.close() # setclone row col cellsize xupleft yupleft setclone(nr_rows, nr_cols, cell_size, x, y) map_out = numpy2pcr(Boolean, mapnp, 0) flagmap = True if flags['checkfiles']: checkmap(name, filename, map_out, flagmap, 0) else: raise LisfloodError( "Maskmap: {} is not a valid mask map nor valid coordinates".format( name)) _ = MaskAttrs(uuid.uuid4()) # init maskattrs # put in the ldd map # if there is no ldd at a cell, this cell should be excluded from modelling ldd = loadmap('Ldd', pcr=True) # convert ldd to numpy maskldd = pcr2numpy(ldd, np.nan) # convert numpy map to 8bit maskarea = np.bool8(mapnp) # compute mask (pixels in maskldd AND maskarea) mask = np.logical_not(np.logical_and(maskldd, maskarea)) _ = MaskInfo(mask, map_out) # MaskInfo init here if flags['nancheck']: nanCheckMap(ldd, binding['Ldd'], 'Ldd') return map_out
def main(): #-initialization # MVs MV= -999. # minimum catchment size to process catchmentSizeLimit= 0.0 # period of interest, start and end year startYear= 1961 endYear= 2010 # maps cloneMapFileName= '/data/hydroworld/PCRGLOBWB20/input30min/global/Global_CloneMap_30min.map' lddFileName= '/data/hydroworld/PCRGLOBWB20/input30min/routing/lddsound_30min.map' cellAreaFileName= '/data/hydroworld/PCRGLOBWB20/input30min/routing/cellarea30min.map' # set clone pcr.setclone(cloneMapFileName) # output outputPath= '/scratch/rens/reservedrecharge' percentileMapFileName= os.path.join(outputPath,'q%03d_cumsec.map') textFileName= os.path.join(outputPath,'groundwater_environmentalflow_%d.txt') fractionReservedRechargeMapFileName= os.path.join(outputPath,'fraction_reserved_recharge%d.map') fractionMinimumReservedRechargeMapFileName= os.path.join(outputPath,'minimum_fraction_reserved_recharge%d.map') # input inputPath= '/nfsarchive/edwin-emergency-backup-DO-NOT-DELETE/rapid/edwin/05min_runs_results/2015_04_27/non_natural_2015_04_27/global/netcdf/' # define data to be read from netCDF files ncData= {} variableName= 'totalRunoff' ncData[variableName]= {} ncData[variableName]['fileName']= os.path.join(inputPath,'totalRunoff_monthTot_output.nc') ncData[variableName]['fileRoot']= os.path.join(outputPath,'qloc') ncData[variableName]['annualAverage']= pcr.scalar(0) variableName= 'gwRecharge' ncData[variableName]= {} ncData[variableName]['fileName']= os.path.join(inputPath,'gwRecharge_monthTot_output.nc') ncData[variableName]['fileRoot']= os.path.join(outputPath,'gwrec') ncData[variableName]['annualAverage']= pcr.scalar(0) variableName= 'discharge' ncData[variableName]= {} ncData[variableName]['fileName']= os.path.join(inputPath,'totalRunoff_monthTot_output.nc') ncData[variableName]['fileRoot']= os.path.join(outputPath,'qc') ncData[variableName]['annualAverage']= pcr.scalar(0) ncData[variableName]['mapStack']= np.array([]) # percents and environmental flow condition set as percentile percents= range(10,110,10) environmentalFlowPercent= 10 if environmentalFlowPercent not in percents: percents.append(environmentalFlowPercent) percents.sort() #-start # obtain attributes pcr.setclone(cloneMapFileName) cloneSpatialAttributes= spatialAttributes(cloneMapFileName) years= range(startYear,endYear+1) # output path if not os.path.isdir(outputPath): os.makedirs(outputPath) os.chdir(outputPath) # compute catchments ldd= pcr.readmap(lddFileName) cellArea= pcr.readmap(cellAreaFileName) catchments= pcr.catchment(ldd,pcr.pit(ldd)) fractionWater= pcr.scalar(0.0) # temporary! lakeMask= pcr.boolean(0) # temporary! pcr.report(catchments,os.path.join(outputPath,'catchments.map')) maximumCatchmentID= int(pcr.cellvalue(pcr.mapmaximum(pcr.scalar(catchments)),1)[0]) # iterate over years weight= float(len(years))**-1 for year in years: #-echo year print ' - processing year %d' % year #-process data startDate= datetime.datetime(year,1,1) endDate= datetime.datetime(year,12,31) timeSteps= endDate.toordinal()-startDate.toordinal()+1 dynamicIncrement= 1 for variableName in ncData.keys(): print ' extracting %s' % variableName, ncFileIn= ncData[variableName]['fileName'] #-process data pcrDataSet= pcrObject(variableName, ncData[variableName]['fileRoot'],\ ncFileIn,cloneSpatialAttributes, pcrVALUESCALE= pcr.Scalar, resamplingAllowed= True,\ dynamic= True, dynamicStart= startDate, dynamicEnd= endDate, dynamicIncrement= dynamicIncrement, ncDynamicDimension= 'time') pcrDataSet.initializeFileInfo() pcrDataSet.processFileInfo() for fileInfo in pcrDataSet.fileProcessInfo.values()[0]: tempFileName= fileInfo[1] variableField= pcr.readmap(tempFileName) variableField= pcr.ifthen(pcr.defined(ldd),pcr.cover(variableField,0)) if variableName == 'discharge': dayNumber= int(os.path.splitext(tempFileName)[1].strip('.')) date= datetime.date(year,1,1)+datetime.timedelta(dayNumber-1) numberDays= calendar.monthrange(year,date.month)[1] variableField= pcr.max(0,pcr.catchmenttotal(variableField*cellArea,ldd)/(numberDays*24*3600)) ncData[variableName]['annualAverage']+= weight*variableField if 'mapStack' in ncData[variableName].keys(): tempArray= pcr2numpy(variableField,MV) mask= tempArray != MV if ncData[variableName]['mapStack'].size != 0: ncData[variableName]['mapStack']= np.vstack((ncData[variableName]['mapStack'],tempArray[mask])) else: ncData[variableName]['mapStack']= tempArray[mask] coordinates= np.zeros((ncData[variableName]['mapStack'].size,2)) pcr.setglobaloption('unitcell') tempArray= pcr2numpy(pcr.ycoordinate(pcr.boolean(1))+0.5,MV) coordinates[:,0]= tempArray[mask] tempArray= pcr2numpy(pcr.xcoordinate(pcr.boolean(1))+0.5,MV) coordinates[:,1]= tempArray[mask] os.remove(tempFileName) # delete object pcrDataSet= None del pcrDataSet # close line on screen print # report annual averages key= 'annualAverage' ncData['discharge'][key]/= 12 for variableName in ncData.keys(): ncData[variableName][key]= pcr.max(0,ncData[variableName][key]) pcr.report(ncData[variableName][key],\ os.path.join(outputPath,'%s_%s.map' % (variableName,key))) # remove aux.xml for tempFileName in os.listdir(outputPath): if 'aux.xml' in tempFileName: os.remove(tempFileName) # sort data print 'sorting discharge data' variableName= 'discharge' key= 'mapStack' indices= np.zeros((ncData[variableName][key].shape),np.uint) for iCnt in xrange(ncData[variableName][key].shape[1]): indices[:,iCnt]= ncData[variableName][key][:,iCnt].argsort(kind= 'mergesort') ncData[variableName][key][:,iCnt]= ncData[variableName][key][:,iCnt][indices[:,iCnt]] # extract values for percentiles print 'returning maps' for percent in percents: percentile= 0.01*percent index0= min(ncData[variableName][key].shape[0]-1,int(percentile*ncData[variableName][key].shape[0])) index1= min(ncData[variableName][key].shape[0]-1,int(percentile*ncData[variableName][key].shape[0])+1) x0= float(index0)/ncData[variableName][key].shape[0] x1= float(index1)/ncData[variableName][key].shape[0] if x0 <> x1: y= ncData[variableName][key][index0,:]+(percentile-x0)*\ (ncData[variableName][key][index1,:]-ncData[variableName][key][index0,:])/(x1-x0) else: y= ncData[variableName][key][index0,:] # convert a slice of the stack into an array tempArray= np.ones((cloneSpatialAttributes.numberRows,cloneSpatialAttributes.numberCols))*MV for iCnt in xrange(coordinates.shape[0]): row= coordinates[iCnt,0]-1 col= coordinates[iCnt,1]-1 tempArray[row,col]= y[iCnt] variableField= numpy2pcr(pcr.Scalar,tempArray,MV) pcr.report(variableField,percentileMapFileName % percent) if percent == environmentalFlowPercent: ncData[variableName]['environmentalFlow']= variableField tempArray= None; variableField= None del tempArray, variableField # process environmental flow # initialize map of reserved recharge fraction fractionReservedRechargeMap= pcr.ifthen(ncData[variableName]['environmentalFlow'] < 0,pcr.scalar(0)) fractionMinimumReservedRechargeMap= pcr.ifthen(ncData[variableName]['environmentalFlow'] < 0,pcr.scalar(0)) textFile= open(textFileName % environmentalFlowPercent,'w') hStr= 'Environmental flow analysis per basin, resulting in a map of renewable, exploitable recharge, for the %d%s quantile of discharge\n' % (environmentalFlowPercent,'%') hStr+= 'Returns Q_%d/R, the fraction of reserved recharge needed to sustain fully the environental flow requirement defined as the %d percentile,\n' % (environmentalFlowPercent, environmentalFlowPercent) hStr+= 'and Q*_%d/R, a reduced fraction that takes the availability of surface water into account\n' % environmentalFlowPercent textFile.write(hStr) print hStr # create header to display on screen and write to file # reported are: 1: ID, 2: Area, 3: average discharge, 4: environmental flow, 5: average recharge, # 6: Q_%d/Q, 7: Q_%d/R_Avg, 8: R_Avg/Q_Avg, 9: Q*_%d/R_Avg hStr= '%6s,%15s,%15s,%15s,%15s,%15s,%15s,%15s,%15s\n' % \ ('ID','Area [km2]','Q_Avg [m3]','Q_%d [m3]' % environmentalFlowPercent ,'R_Avg [m3]','Q_%d/Q_Avg [-]' % environmentalFlowPercent,\ 'Q_%d/Q_Avg [-]' % environmentalFlowPercent,'R_Avg/Q_Avg [-]','Q*_%d/Q_Avg [-]' % environmentalFlowPercent) textFile.write(hStr) print hStr for catchment in xrange(1,maximumCatchmentID+1): # create catchment mask and check whether it does not coincide with a lake catchmentMask= catchments == catchment catchmentSize= pcr.cellvalue(pcr.maptotal(pcr.ifthen(catchmentMask,cellArea*1.e-6)),1)[0] #~ ##~ if pcr.cellvalue(pcr.maptotal(pcr.ifthen(catchmentMask,pcr.scalar(lakeMask))),1) <> \ #~ ##~ pcr.cellvalue(pcr.maptotal(pcr.ifthen(catchmentMask,pcr.scalar(catchmentMask))),1)[0] and \ #~ ##~ catchmentSize > catchmentSizeLimit: key= 'annualAverage' variableName= 'discharge' if bool(pcr.cellvalue(pcr.maptotal(pcr.ifthen((ldd == 5) & catchmentMask,\ pcr.scalar(ncData[variableName][key] > 0))),1)[0]) and catchmentSize >= catchmentSizeLimit: # valid catchment, process # all volumes are in m3 per year key= 'annualAverage' catchmentAverageDischarge= pcr.cellvalue(pcr.mapmaximum(pcr.ifthen(catchmentMask & (ldd == 5),\ ncData[variableName][key])),1)[0]*365.25*3600*24 variableName= 'gwRecharge' catchmentRecharge= pcr.cellvalue(pcr.maptotal(pcr.ifthen(catchmentMask,ncData[variableName][key]*\ (1.-fractionWater)*cellArea)),1)[0] variableName= 'totalRunoff' catchmentRunoff= pcr.cellvalue(pcr.maptotal(pcr.ifthen(catchmentMask,ncData[variableName][key]*\ cellArea)),1)[0] key= 'environmentalFlow' variableName= 'discharge' catchmentEnvironmentalFlow= pcr.cellvalue(pcr.mapmaximum(pcr.ifthen(catchmentMask & (ldd == 5),\ ncData[variableName][key])),1)[0]*365.25*3600*24 catchmentRunoff= max(catchmentRunoff,catchmentEnvironmentalFlow) if catchmentAverageDischarge > 0.: fractionEnvironmentalFlow= catchmentEnvironmentalFlow/catchmentAverageDischarge fractionGroundWaterContribution= catchmentRecharge/catchmentAverageDischarge else: fractionEnvironmentalFlow= 0. fractionGroundWaterContribution= 0. if catchmentRecharge > 0: fractionReservedRecharge= min(1,catchmentEnvironmentalFlow/catchmentRecharge) else: fractionReservedRecharge= 1.0 fractionMinimumReservedRecharge= (fractionReservedRecharge+fractionGroundWaterContribution-\ fractionReservedRecharge*fractionGroundWaterContribution)*fractionReservedRecharge #~ # echo to screen, and write to file and map wStr= '%6s,%15.1f,%15.6g,%15.6g,%15.6g,%15.6f,%15.6f,%15.6f,%15.6f\n' % \ (catchment,catchmentSize,catchmentAverageDischarge,catchmentEnvironmentalFlow,catchmentRecharge,\ fractionEnvironmentalFlow,fractionReservedRecharge,fractionGroundWaterContribution,fractionMinimumReservedRecharge) print wStr textFile.write(wStr) # update maps fractionReservedRechargeMap= pcr.ifthenelse(catchmentMask,\ pcr.scalar(fractionReservedRecharge),fractionReservedRechargeMap) fractionMinimumReservedRechargeMap= pcr.ifthenelse(catchmentMask,\ pcr.scalar(fractionMinimumReservedRecharge),fractionMinimumReservedRechargeMap) #-report map and close text file pcr.report(fractionReservedRechargeMap,fractionReservedRechargeMapFileName % environmentalFlowPercent) pcr.report(fractionMinimumReservedRechargeMap,fractionMinimumReservedRechargeMapFileName % environmentalFlowPercent) # close text file textFile.close() # finished print 'all done!'
def netcdf2PCRobjClone(ncFile,varName,dateInput,\ useDoy = None, cloneMapFileName = None,\ LatitudeLongitude = True,\ specificFillValue = None): # # EHS (19 APR 2013): To convert netCDF (tss) file to PCR file. # --- with clone checking # Only works if cells are 'square'. # Only works if cellsizeClone <= cellsizeInput # Get netCDF file and variable name: print ncFile if ncFile in filecache.keys(): f = filecache[ncFile] print "Cached: ", ncFile else: f = nc.Dataset(ncFile) filecache[ncFile] = f print "New: ", ncFile varName = str(varName) if LatitudeLongitude == True: try: f.variables['lat'] = f.variables['latitude'] f.variables['lon'] = f.variables['longitude'] except: pass if varName == "evapotranspiration": try: f.variables['evapotranspiration'] = f.variables['referencePotET'] except: pass # date date = dateInput if useDoy == "Yes": print('Finding the date based on the given climatology doy index (1 to 366, or index 0 to 365)') idx = int(dateInput) - 1 elif useDoy == "month": # PS: WE NEED THIS ONE FOR NETCDF FILES that contain only 12 monthly values (e.g. cropCoefficientWaterNC). print('Finding the date based on the given climatology month index (1 to 12, or index 0 to 11)') # make sure that date is in the correct format if isinstance(date, str) == True: date = \ datetime.datetime.strptime(str(date),'%Y-%m-%d') idx = int(date.month) - 1 else: # make sure that date is in the correct format if isinstance(date, str) == True: date = \ datetime.datetime.strptime(str(date),'%Y-%m-%d') date = datetime.datetime(date.year,date.month,date.day) if useDoy == "yearly": date = datetime.datetime(date.year,int(1),int(1)) if useDoy == "monthly": date = datetime.datetime(date.year,date.month,int(1)) if useDoy == "yearly" or useDoy == "monthly" or useDoy == "daily_seasonal": # if the desired year is not available, use the first year or the last year that is available first_year_in_nc_file = findFirstYearInNCTime(f.variables['time']) last_year_in_nc_file = findLastYearInNCTime(f.variables['time']) # if date.year < first_year_in_nc_file: date = datetime.datetime(first_year_in_nc_file,date.month,date.day) msg = "\n" msg += "WARNING related to the netcdf file: "+str(ncFile)+" ; variable: "+str(varName)+" !!!!!!"+"\n" msg += "The date "+str(dateInput)+" is NOT available. " msg += "The date "+str(date.year)+"-"+str(date.month)+"-"+str(date.day)+" is used." msg += "\n" print(msg) if date.year > last_year_in_nc_file: date = datetime.datetime(last_year_in_nc_file,date.month,date.day) msg = "\n" msg += "WARNING related to the netcdf file: "+str(ncFile)+" ; variable: "+str(varName)+" !!!!!!"+"\n" msg += "The date "+str(dateInput)+" is NOT available. " msg += "The date "+str(date.year)+"-"+str(date.month)+"-"+str(date.day)+" is used." msg += "\n" print(msg) try: idx = nc.date2index(date, f.variables['time'], calendar = f.variables['time'].calendar, \ select ='exact') msg = "The date "+str(date.year)+"-"+str(date.month)+"-"+str(date.day)+" is available. The 'exact' option is used while selecting netcdf time." print(msg) except: msg = "The date "+str(date.year)+"-"+str(date.month)+"-"+str(date.day)+" is NOT available. The 'exact' option CANNOT be used while selecting netcdf time." print(msg) try: idx = nc.date2index(date, f.variables['time'], calendar = f.variables['time'].calendar, \ select = 'before') msg = "\n" msg += "WARNING related to the netcdf file: "+str(ncFile)+" ; variable: "+str(varName)+" !!!!!!"+"\n" msg += "The date "+str(date.year)+"-"+str(date.month)+"-"+str(date.day)+" is NOT available. The 'before' option is used while selecting netcdf time." msg += "\n" except: idx = nc.date2index(date, f.variables['time'], calendar = f.variables['time'].calendar, \ select = 'after') msg = "\n" msg += "WARNING related to the netcdf file: "+str(ncFile)+" ; variable: "+str(varName)+" !!!!!!"+"\n" msg += "The date "+str(date.year)+"-"+str(date.month)+"-"+str(date.day)+" is NOT available. The 'after' option is used while selecting netcdf time." msg += "\n" print(msg) idx = int(idx) sameClone = True # check whether clone and input maps have the same attributes: if cloneMapFileName != None: # get the attributes of cloneMap attributeClone = getMapAttributesALL(cloneMapFileName) cellsizeClone = attributeClone['cellsize'] rowsClone = attributeClone['rows'] colsClone = attributeClone['cols'] xULClone = attributeClone['xUL'] yULClone = attributeClone['yUL'] # get the attributes of input (netCDF) cellsizeInput = f.variables['lat'][0]- f.variables['lat'][1] cellsizeInput = float(cellsizeInput) rowsInput = len(f.variables['lat']) colsInput = len(f.variables['lon']) xULInput = f.variables['lon'][0]-0.5*cellsizeInput yULInput = f.variables['lat'][0]+0.5*cellsizeInput # check whether both maps have the same attributes if cellsizeClone != cellsizeInput: sameClone = False if rowsClone != rowsInput: sameClone = False if colsClone != colsInput: sameClone = False if xULClone != xULInput: sameClone = False if yULClone != yULInput: sameClone = False cropData = f.variables[varName][int(idx),:,:] # still original data factor = 1 # needed in regridData2FinerGrid if sameClone == False: print('Crop to the clone map with lower left corner (x,y): '+str(xULClone)+' , '+str(yULClone)) # crop to cloneMap: #~ xIdxSta = int(np.where(f.variables['lon'][:] == xULClone + 0.5*cellsizeInput)[0]) minX = min(abs(f.variables['lon'][:] - (xULClone + 0.5*cellsizeInput))) # ; print(minX) xIdxSta = int(np.where(abs(f.variables['lon'][:] - (xULClone + 0.5*cellsizeInput)) == minX)[0]) xIdxEnd = int(math.ceil(xIdxSta + colsClone /(cellsizeInput/cellsizeClone))) #~ yIdxSta = int(np.where(f.variables['lat'][:] == yULClone - 0.5*cellsizeInput)[0]) minY = min(abs(f.variables['lat'][:] - (yULClone - 0.5*cellsizeInput))) # ; print(minY) yIdxSta = int(np.where(abs(f.variables['lat'][:] - (yULClone - 0.5*cellsizeInput)) == minY)[0]) yIdxEnd = int(math.ceil(yIdxSta + rowsClone /(cellsizeInput/cellsizeClone))) cropData = f.variables[varName][idx,yIdxSta:yIdxEnd,xIdxSta:xIdxEnd] factor = int(float(cellsizeInput)/float(cellsizeClone)) # convert to PCR object and close f if specificFillValue != None: outPCR = pcr.numpy2pcr(pcr.Scalar, \ regridData2FinerGrid(factor,cropData,MV), \ float(specificFillValue)) else: outPCR = pcr.numpy2pcr(pcr.Scalar, \ regridData2FinerGrid(factor,cropData,MV), \ float(f.variables[varName]._FillValue)) #f.close(); f = None ; cropData = None # PCRaster object return (outPCR)
def loadmap(name, pcr=False, lddflag=False, timestampflag='exact', averageyearflag=False): """ Load a static map either value or pcraster map or netcdf (single or stack) Load a static map either value or pcraster map or netcdf (single or stack) If a netCDF stack is loaded, map is read according to timestepInit date (i.e. model time step). If timestepInit is a step number, step number is converted to date (referred to CalendarDayStart in settings.xml). Then date is used to read time step from netCDF file. if timestampflag = 'closest' and loadmap is reading a NetCDF stack, the timestep with the closest timestamp will be loaded if the exact one is not available. :param name: name of key in Settings.xml input file containing path and name of the map file (as string) :param pcr: flag for output maps in pcraster format :param lddflag: flag for local drain direction map (CM??) :param timestampflag: look for exact time stamp in netcdf file ('exact') or for the closest (left) time stamp available ('closest') :param averageyearflag: if True, use "average year" netcdf file over the entire model simulation period :return: map or mapC :except: pcr: maps must have the same size of clone.map netCDF: time step timestepInit must be included into the stack """ # name of the key in Settimgs.xml file containing path and name of the map file settings = LisSettings.instance() binding = settings.binding flags = settings.flags value = binding[name] # path and name of the map file filename = value load = False pcrmap = False # try reading in PCRaster map format try: # try reading constant value mapC = float(value) flagmap = False load = True if pcr: map = mapC except ValueError: try: # try reading pcraster map exploiting the iterAccess class map = iterReadPCRasterMap(value) flagmap = True load = True pcrmap = True except: load = False if load and pcrmap: #map is loaded and it is in pcraster format try: # test if map is same size as clone map, if not it will make an error test = pcraster.scalar(map) + pcraster.scalar(map) except: raise LisfloodError( "{} might be of a different size than clone size".format( value)) # if failed before try reading from netCDF map format if not load: # read a netcdf (single one not a stack) filename = os.path.splitext(value)[0] + '.nc' # get mapextend of netcdf map and calculate the cutting cut0, cut1, cut2, cut3 = mapattrNetCDF(filename) # load netcdf map but only the rectangle needed nf1 = iterOpenNetcdf(filename, "", 'r') value = listitems(nf1.variables)[-1][0] # get the last variable name (it must be the variable to be read by Lisflood) if not settings.timestep_init: # if timestep_init is missing, read netcdf as single static map mapnp = nf1.variables[value][cut2:cut3, cut0:cut1] else: if 'time' in nf1.variables: # read a netcdf (stack) - state files # get information from netCDF stack t_steps = nf1.variables[ 'time'][:] # get values for timesteps ([ 0., 24., 48., 72., 96.]) t_unit = nf1.variables[ 'time'].units # get unit (u'hours since 2015-01-01 06:00:00') t_cal = get_calendar_type(nf1) # get year from time unit in case average year is used if averageyearflag: # get date of the first step in netCDF file containing average year values first_date = num2date(t_steps[0], t_unit, t_cal) # get year of the first step in netCDF file containing average year values t_ref_year = first_date.year # select timestep to use for reading from netCDF stack based on timestep_init (state file time step) timestepI = calendar(settings.timestep_init, binding['calendar_type']) if isinstance(timestepI, datetime.datetime): #reading dates in XML settings file # get step id number in netCDF stack for timestepInit date if averageyearflag: #if using an average year don't care about the year in timestepIDate and change it to the netCDF first time step year try: timestepI = timestepI.replace(year=t_ref_year) except: timestepI = timestepI.replace(day=28) timestepI = timestepI.replace(year=t_ref_year) timestepI = date2num(timestepI, nf1.variables['time'].units) else: # reading step numbers in XML file # timestepI = int(timestepI) -1 begin = calendar(binding['CalendarDayStart']) DtSec = float(binding['DtSec']) DtDay = DtSec / 86400. # Time step, expressed as fraction of day (same as self.var.DtSec and self.var.DtDay) # get date for step number timestepI (referred to CalendarDayStart) timestepIDate = begin + datetime.timedelta( days=(timestepI - 1) * DtDay) # get step id number in netCDF stack for step timestepInit # timestepInit refers to CalenradDayStart # timestepI now refers to first date in netCDF stack if averageyearflag: #using an average year, don't care about the year in timestepIDate and change it to the netCDF time unit year try: timestepIDate = timestepIDate.replace( year=t_ref_year) except: #if simulation year is leap and average year is not, switch 29/2 with 28/2 timestepIDate = timestepIDate.replace(day=28) timestepIDate = timestepIDate.replace( year=t_ref_year) timestepI = date2num(timestepIDate, units=t_unit, calendar=t_cal) if not (timestepI in nf1.variables['time'][:]): if timestampflag == 'exact': #look for exact time stamp when loading data msg = "time step " + str(int( timestepI) + 1) + " is not stored in " + filename raise LisfloodError(msg) elif timestampflag == 'closest': #get the closest value timestepInew = takeClosest(t_steps, timestepI) #set timestepI to the closest available time step in netCDF file timestepI = timestepInew itime = np.where(nf1.variables['time'][:] == timestepI)[0][0] mapnp = nf1.variables[value][itime, cut2:cut3, cut0:cut1] else: # read a netcdf (single one) mapnp = nf1.variables[value][cut2:cut3, cut0:cut1] # masking try: maskinfo = MaskInfo.instance() mapnp.mask = maskinfo.info.mask except KeyError as e: pass nf1.close() # if a map should be pcraster if pcr: # check if integer map (like outlets, lakes etc checkint = str(mapnp.dtype) if checkint == "int16" or checkint == "int32": mapnp[mapnp.mask] = -9999 map = numpy2pcr(Nominal, mapnp, -9999) elif checkint == "int8": mapnp[mapnp < 0] = -9999 map = numpy2pcr(Nominal, mapnp, -9999) else: mapnp[np.isnan(mapnp)] = -9999 map = numpy2pcr(Scalar, mapnp, -9999) # if the map is a ldd if lddflag: map = pcraster.ldd(pcraster.nominal(map)) else: mapC = compressArray(mapnp, pcr=False, name=filename) flagmap = True # pcraster map but it has to be an array if pcrmap and not pcr: mapC = compressArray(map, name=filename) if flags['checkfiles']: print(name, filename) if flagmap == False: checkmap(name, filename, mapC, flagmap, 0) elif pcr: checkmap(name, filename, map, flagmap, 0) else: print(name, mapC.size) if mapC.size > 0: map = decompress(mapC) checkmap(name, filename, map, flagmap, 0) if pcr: if flags['nancheck'] and name != 'Ldd': nanCheckMap(map, filename, name) return map elif isinstance(mapC, np.ndarray): return mapC.astype(float) else: if flags['nancheck'] and name != 'Ldd': nanCheckMap(mapC, filename, name) return mapC
def netcdf2PCRobjClone(ncFile,varName,dateInput,\ useDoy = None, cloneMapFileName = None,\ LatitudeLongitude = True,\ specificFillValue = None): # # EHS (19 APR 2013): To convert netCDF (tss) file to PCR file. # --- with clone checking # Only works if cells are 'square'. # Only works if cellsizeClone <= cellsizeInput # Get netCDF file and variable name: #~ print ncFile if ncFile in filecache.keys(): f = filecache[ncFile] #~ print "Cached: ", ncFile else: f = nc.Dataset(ncFile) filecache[ncFile] = f #~ print "New: ", ncFile varName = str(varName) if varName == "Automatic": for key in f.variables.keys(): if key not in ['latitude', 'longitude', 'lat', 'lon', 'time']: varName = key print varName if LatitudeLongitude == True: try: f.variables['lat'] = f.variables['latitude'] f.variables['lon'] = f.variables['longitude'] except: pass if varName == "evapotranspiration": try: f.variables['evapotranspiration'] = f.variables['referencePotET'] except: pass # date date = dateInput if useDoy == "Yes": idx = dateInput - 1 else: if isinstance(date, str) == True: date = \ datetime.datetime.strptime(str(date),'%Y-%m-%d') date = datetime.datetime(date.year,date.month,date.day) # time index (in the netCDF file) if useDoy == "month": idx = int(date.month) - 1 else: if useDoy == "yearly":\ date = datetime.datetime(date.year,int(1),int(1)) if useDoy == "monthly":\ date = datetime.datetime(date.year,date.month,int(1)) nctime = f.variables['time'] # A netCDF time variable object. idx = nc.date2index(date, nctime, calendar=nctime.calendar, \ select='exact') idx = int(idx) sameClone = True # check whether clone and input maps have the same attributes: if cloneMapFileName != None: # get the attributes of cloneMap attributeClone = getMapAttributesALL(cloneMapFileName) cellsizeClone = attributeClone['cellsize'] rowsClone = attributeClone['rows'] colsClone = attributeClone['cols'] xULClone = attributeClone['xUL'] yULClone = attributeClone['yUL'] # get the attributes of input (netCDF) cellsizeInput = f.variables['lat'][0]- f.variables['lat'][1] cellsizeInput = float(cellsizeInput) rowsInput = len(f.variables['lat']) colsInput = len(f.variables['lon']) xULInput = f.variables['lon'][0]-0.5*cellsizeInput yULInput = f.variables['lat'][0]+0.5*cellsizeInput # check whether both maps have the same attributes if cellsizeClone != cellsizeInput: sameClone = False if rowsClone != rowsInput: sameClone = False if colsClone != colsInput: sameClone = False if xULClone != xULInput: sameClone = False if yULClone != yULInput: sameClone = False cropData = f.variables[varName][int(idx),:,:] # still original data factor = 1 # needed in regridData2FinerGrid if sameClone == False: # crop to cloneMap: #~ xIdxSta = int(np.where(f.variables['lon'][:] == xULClone + 0.5*cellsizeInput)[0]) minX = min(abs(f.variables['lon'][:] - (xULClone + 0.5*cellsizeInput))) # ; print(minX) xIdxSta = int(np.where(abs(f.variables['lon'][:] - (xULClone + 0.5*cellsizeInput)) == minX)[0]) xIdxEnd = int(math.ceil(xIdxSta + colsClone /(cellsizeInput/cellsizeClone))) #~ yIdxSta = int(np.where(f.variables['lat'][:] == yULClone - 0.5*cellsizeInput)[0]) minY = min(abs(f.variables['lat'][:] - (yULClone - 0.5*cellsizeInput))) # ; print(minY) yIdxSta = int(np.where(abs(f.variables['lat'][:] - (yULClone - 0.5*cellsizeInput)) == minY)[0]) yIdxEnd = int(math.ceil(yIdxSta + rowsClone /(cellsizeInput/cellsizeClone))) cropData = f.variables[varName][idx,yIdxSta:yIdxEnd,xIdxSta:xIdxEnd] factor = int(round(float(cellsizeInput)/float(cellsizeClone))) # convert to PCR object and close f if specificFillValue != None: outPCR = pcr.numpy2pcr(pcr.Scalar, \ regridData2FinerGrid(factor,cropData,MV), \ float(specificFillValue)) else: outPCR = pcr.numpy2pcr(pcr.Scalar, \ regridData2FinerGrid(factor,cropData,MV), \ float(f.variables[varName]._FillValue)) #f.close(); f = None ; cropData = None # PCRaster object return (outPCR)
def netcdf2PCRobjClone(ncFile,varName,dateInput,\ useDoy = None, cloneMapFileName = None,\ LatitudeLongitude = False,\ specificFillValue = None): # # EHS (19 APR 2013): To convert netCDF (tss) file to PCR file. # --- with clone checking # Only works if cells are 'square'. # Only works if cellsizeClone <= cellsizeInput # Get netCDF file and variable name: if ncFile in filecache.keys(): f = filecache[ncFile] print "Cached: ", ncFile else: f = nc.Dataset(ncFile) filecache[ncFile] = f print "New: ", ncFile varName = str(varName) if LatitudeLongitude == True: try: f.variables['lat'] = f.variables['latitude'] f.variables['lon'] = f.variables['longitude'] except: pass if varName == "evapotranspiration": try: f.variables['evapotranspiration'] = f.variables['referencePotET'] except: pass # date date = dateInput if useDoy == "Yes": idx = dateInput - 1 else: if isinstance(date, str) == True: date = \ datetime.datetime.strptime(str(date),'%Y-%m-%d') date = datetime.datetime(date.year, date.month, date.day) # time index (in the netCDF file) if useDoy == "month": idx = int(date.month) - 1 else: if useDoy == "yearly": \ date = datetime.datetime(date.year,int(1),int(1)) if useDoy == "monthly": \ date = datetime.datetime(date.year,date.month,int(1)) nctime = f.variables['time'] # A netCDF time variable object. idx = nc.date2index(date, nctime, calendar=nctime.calendar, \ select='exact') idx = int(idx) sameClone = True # check whether clone and input maps have the same attributes: if cloneMapFileName != None: # get the attributes of cloneMap attributeClone = getMapAttributesALL(cloneMapFileName) cellsizeClone = attributeClone['cellsize'] rowsClone = attributeClone['rows'] colsClone = attributeClone['cols'] xULClone = attributeClone['xUL'] yULClone = attributeClone['yUL'] # get the attributes of input (netCDF) cellsizeInput = f.variables['lat'][0] - f.variables['lat'][1] cellsizeInput = float(cellsizeInput) rowsInput = len(f.variables['lat']) colsInput = len(f.variables['lon']) xULInput = f.variables['lon'][0] - 0.5 * cellsizeInput yULInput = f.variables['lat'][0] + 0.5 * cellsizeInput # check whether both maps have the same attributes if cellsizeClone != cellsizeInput: sameClone = False if rowsClone != rowsInput: sameClone = False if colsClone != colsInput: sameClone = False if xULClone != xULInput: sameClone = False if yULClone != yULInput: sameClone = False cropData = f.variables[varName][int(idx), :, :] # still original data factor = 1 # needed in regridData2FinerGrid if sameClone == False: # crop to cloneMap: #~ xIdxSta = int(np.where(f.variables['lon'][:] == xULClone + 0.5*cellsizeInput)[0]) minX = min( abs(f.variables['lon'][:] - (xULClone + 0.5 * cellsizeInput))) # ; print(minX) xIdxSta = int( np.where( abs(f.variables['lon'][:] - (xULClone + 0.5 * cellsizeInput)) == minX)[0]) xIdxEnd = int( math.ceil(xIdxSta + colsClone / (cellsizeInput / cellsizeClone))) #~ yIdxSta = int(np.where(f.variables['lat'][:] == yULClone - 0.5*cellsizeInput)[0]) minY = min( abs(f.variables['lat'][:] - (yULClone - 0.5 * cellsizeInput))) # ; print(minY) yIdxSta = int( np.where( abs(f.variables['lat'][:] - (yULClone - 0.5 * cellsizeInput)) == minY)[0]) yIdxEnd = int( math.ceil(yIdxSta + rowsClone / (cellsizeInput / cellsizeClone))) cropData = f.variables[varName][idx, yIdxSta:yIdxEnd, xIdxSta:xIdxEnd] factor = int(float(cellsizeInput) / float(cellsizeClone)) # convert to PCR object and close f if specificFillValue != None: outPCR = pcr.numpy2pcr(pcr.Scalar, \ regridData2FinerGrid(factor,cropData,MV), \ float(specificFillValue)) else: outPCR = pcr.numpy2pcr(pcr.Scalar, \ regridData2FinerGrid(factor,cropData,MV), \ float(f.variables[varName]._FillValue)) #f.close(); f = None cropData = None # PCRaster object return (outPCR)
def netcdf2PCRobjCloneWind(ncFile,varName,dateInput,useDoy = None, cloneMapFileName=None): # EHS (02 SEP 2013): This is a special function made by Niko Wanders (for his DA framework). # EHS (19 APR 2013): To convert netCDF (tss) file to PCR file. # --- with clone checking # Only works if cells are 'square'. # Only works if cellsizeClone <= cellsizeInput # Get netCDF file and variable name: f = nc.Dataset(ncFile) varName = str(varName) # date date = dateInput if useDoy == "Yes": idx = dateInput - 1 else: if isinstance(date, str) == True: date = \ datetime.datetime.strptime(str(date),'%Y-%m-%d') date = datetime.datetime(date.year,date.month,date.day, 0, 0) # time index (in the netCDF file) nctime = f.variables['time'] # A netCDF time variable object. idx = nc.date2index(date, nctime, select="exact") idx = int(idx) sameClone = True # check whether clone and input maps have the same attributes: if cloneMapFileName != None: # get the attributes of cloneMap attributeClone = getMapAttributesALL(cloneMapFileName) cellsizeClone = attributeClone['cellsize'] rowsClone = attributeClone['rows'] colsClone = attributeClone['cols'] xULClone = attributeClone['xUL'] yULClone = attributeClone['yUL'] # get the attributes of input (netCDF) cellsizeInput = f.variables['lat'][0]- f.variables['lat'][1] cellsizeInput = float(cellsizeInput) rowsInput = len(f.variables['lat']) colsInput = len(f.variables['lon']) xULInput = f.variables['lon'][0]-0.5*cellsizeInput yULInput = f.variables['lat'][0]+0.5*cellsizeInput # check whether both maps have the same attributes if cellsizeClone != cellsizeInput: sameClone = False if rowsClone != rowsInput: sameClone = False if colsClone != colsInput: sameClone = False if xULClone != xULInput: sameClone = False if yULClone != yULInput: sameClone = False cropData = f.variables[varName][int(idx),:,:] # still original data factor = 1 # needed in regridData2FinerGrid if sameClone == False: # crop to cloneMap: xIdxSta = int(np.where(f.variables['lon'][:] == xULClone + 0.5*cellsizeInput)[0]) xIdxEnd = int(math.ceil(xIdxSta + colsClone /(cellsizeInput/cellsizeClone))) yIdxSta = int(np.where(f.variables['lat'][:] == yULClone - 0.5*cellsizeInput)[0]) yIdxEnd = int(math.ceil(yIdxSta + rowsClone /(cellsizeInput/cellsizeClone))) cropData = f.variables[varName][idx,yIdxSta:yIdxEnd,xIdxSta:xIdxEnd] factor = int(float(cellsizeInput)/float(cellsizeClone)) # convert to PCR object and close f outPCR = pcr.numpy2pcr(pcr.Scalar, \ regridData2FinerGrid(factor,cropData,MV), \ float(f.variables[varName]._FillValue)) f.close(); f = None ; cropData = None # PCRaster object return (outPCR)
def netcdf2PCRobjCloneWithoutTime(ncFile,varName, cloneMapFileName = None,\ LatitudeLongitude = False,\ specificFillValue = None): # # EHS (19 APR 2013): To convert netCDF (tss) file to PCR file. # --- with clone checking # Only works if cells are 'square'. # Only works if cellsizeClone <= cellsizeInput # Get netCDF file and variable name: if ncFile in filecache.keys(): f = filecache[ncFile] print "Cached: ", ncFile else: f = nc.Dataset(ncFile) filecache[ncFile] = f print "New: ", ncFile #print ncFile #f = nc.Dataset(ncFile) varName = str(varName) if LatitudeLongitude == True: try: f.variables['lat'] = f.variables['latitude'] f.variables['lon'] = f.variables['longitude'] except: pass sameClone = True # check whether clone and input maps have the same attributes: if cloneMapFileName != None: # get the attributes of cloneMap attributeClone = getMapAttributesALL(cloneMapFileName) cellsizeClone = attributeClone['cellsize'] rowsClone = attributeClone['rows'] colsClone = attributeClone['cols'] xULClone = attributeClone['xUL'] yULClone = attributeClone['yUL'] # get the attributes of input (netCDF) cellsizeInput = f.variables['lat'][0] - f.variables['lat'][1] cellsizeInput = float(cellsizeInput) rowsInput = len(f.variables['lat']) colsInput = len(f.variables['lon']) xULInput = f.variables['lon'][0] - 0.5 * cellsizeInput yULInput = f.variables['lat'][0] + 0.5 * cellsizeInput # check whether both maps have the same attributes if cellsizeClone != cellsizeInput: sameClone = False if rowsClone != rowsInput: sameClone = False if colsClone != colsInput: sameClone = False if xULClone != xULInput: sameClone = False if yULClone != yULInput: sameClone = False cropData = f.variables[varName][:, :] # still original data factor = 1 # needed in regridData2FinerGrid if sameClone == False: # crop to cloneMap: minX = min( abs(f.variables['lon'][:] - (xULClone + 0.5 * cellsizeInput))) # ; print(minX) xIdxSta = int( np.where( abs(f.variables['lon'][:] - (xULClone + 0.5 * cellsizeInput)) == minX)[0]) xIdxEnd = int( math.ceil(xIdxSta + colsClone / (cellsizeInput / cellsizeClone))) minY = min( abs(f.variables['lat'][:] - (yULClone - 0.5 * cellsizeInput))) # ; print(minY) yIdxSta = int( np.where( abs(f.variables['lat'][:] - (yULClone - 0.5 * cellsizeInput)) == minY)[0]) yIdxEnd = int( math.ceil(yIdxSta + rowsClone / (cellsizeInput / cellsizeClone))) cropData = f.variables[varName][yIdxSta:yIdxEnd, xIdxSta:xIdxEnd] factor = int(float(cellsizeInput) / float(cellsizeClone)) # convert to PCR object and close f if specificFillValue != None: outPCR = pcr.numpy2pcr(pcr.Scalar, \ regridData2FinerGrid(factor,cropData,MV), \ float(specificFillValue)) else: outPCR = pcr.numpy2pcr(pcr.Scalar, \ regridData2FinerGrid(factor,cropData,MV), \ float(f.variables[varName]._FillValue)) #~ # debug: #~ pcr.report(outPCR,"tmp.map") #~ print(varName) #~ os.system('aguila tmp.map') #f.close(); f = None cropData = None # PCRaster object return (outPCR)
def netcdf2PCRobjCloneWind(ncFile, varName, dateInput, useDoy=None, cloneMapFileName=None): # EHS (02 SEP 2013): This is a special function made by Niko Wanders (for his DA framework). # EHS (19 APR 2013): To convert netCDF (tss) file to PCR file. # --- with clone checking # Only works if cells are 'square'. # Only works if cellsizeClone <= cellsizeInput # Get netCDF file and variable name: f = nc.Dataset(ncFile) varName = str(varName) # date date = dateInput if useDoy == "Yes": idx = dateInput - 1 else: if isinstance(date, str) == True: date = \ datetime.datetime.strptime(str(date),'%Y-%m-%d') date = datetime.datetime(date.year, date.month, date.day, 0, 0) # time index (in the netCDF file) nctime = f.variables['time'] # A netCDF time variable object. idx = nc.date2index(date, nctime, select="exact") idx = int(idx) sameClone = True # check whether clone and input maps have the same attributes: if cloneMapFileName != None: # get the attributes of cloneMap attributeClone = getMapAttributesALL(cloneMapFileName) cellsizeClone = attributeClone['cellsize'] rowsClone = attributeClone['rows'] colsClone = attributeClone['cols'] xULClone = attributeClone['xUL'] yULClone = attributeClone['yUL'] # get the attributes of input (netCDF) cellsizeInput = f.variables['lat'][0] - f.variables['lat'][1] cellsizeInput = float(cellsizeInput) rowsInput = len(f.variables['lat']) colsInput = len(f.variables['lon']) xULInput = f.variables['lon'][0] - 0.5 * cellsizeInput yULInput = f.variables['lat'][0] + 0.5 * cellsizeInput # check whether both maps have the same attributes if cellsizeClone != cellsizeInput: sameClone = False if rowsClone != rowsInput: sameClone = False if colsClone != colsInput: sameClone = False if xULClone != xULInput: sameClone = False if yULClone != yULInput: sameClone = False cropData = f.variables[varName][int(idx), :, :] # still original data factor = 1 # needed in regridData2FinerGrid if sameClone == False: # crop to cloneMap: xIdxSta = int( np.where(f.variables['lon'][:] == xULClone + 0.5 * cellsizeInput)[0]) xIdxEnd = int( math.ceil(xIdxSta + colsClone / (cellsizeInput / cellsizeClone))) yIdxSta = int( np.where(f.variables['lat'][:] == yULClone - 0.5 * cellsizeInput)[0]) yIdxEnd = int( math.ceil(yIdxSta + rowsClone / (cellsizeInput / cellsizeClone))) cropData = f.variables[varName][idx, yIdxSta:yIdxEnd, xIdxSta:xIdxEnd] factor = int(float(cellsizeInput) / float(cellsizeClone)) # convert to PCR object and close f outPCR = pcr.numpy2pcr(pcr.Scalar, \ regridData2FinerGrid(factor,cropData,MV), \ float(f.variables[varName]._FillValue)) f.close() f = None cropData = None # PCRaster object return (outPCR)
def regridMapFile2FinerGrid(rescaleFac, coarse): if rescaleFac == 1: return coarse return pcr.numpy2pcr( pcr.Scalar, regridData2FinerGrid(rescaleFac, pcr.pcr2numpy(coarse, MV), MV), MV)
def get_return_period_gumbel(p_zero_in_pcraster, loc_in_pcraster, scale_in_pcraster, flvol_in_pcraster, max_return_period = np.longdouble(1e9), max_return_period_that_can_be_assigned = 1000.): """ Transforms a unique, or array of flood volumes into the belonging return periods, according to gumbel parameters (belonging to non-zero part of the distribution) and a zero probability Inputs: p_zero: probability that flood volume is zero loc: Gumbel location parameter (of non-zero part of distribution) scale: Gumbel scale parameter (of non-zero part of distribution) flvol: Flood volume that will be transformed to return period max_return_period: maximum return period considered. This maximum is needed to prevent that floating point precision becomes a problem (default: 1e9) This function is copied from: https://repos.deltares.nl/repos/Hydrology/trunk/GLOFRIS/src/rp_bias_corr.py """ np.seterr(divide='ignore') np.seterr(invalid='ignore') # convert all pcraster maps to numpy arrays p_zero = np.longdouble(pcr.pcr2numpy(p_zero_in_pcraster, vos.MV)) loc = np.longdouble(pcr.pcr2numpy(loc_in_pcraster , vos.MV)) scale = np.longdouble(pcr.pcr2numpy(scale_in_pcraster , vos.MV)) flvol = np.longdouble(pcr.pcr2numpy(flvol_in_pcraster , vos.MV)) # maximum values for the given max_return_period max_p = 1.0-1.0/max_return_period max_p_residual = np.minimum(np.maximum((max_p-p_zero)/(1.0-p_zero), 0.0), 1.0) max_p_residual[p_zero >= max_p] = 0.0 max_reduced_variate = -np.log(-np.log((max_p_residual))) #~ print np.nanmin(max_p_residual) #~ print np.nanmax(max_p_residual) #~ print np.amin(max_p_residual) #~ print np.amax(max_p_residual) #~ #~ print np.nanmin(max_reduced_variate) #~ print np.nanmax(max_reduced_variate) #~ print np.amin(max_reduced_variate) #~ print np.amax(max_reduced_variate) # compute the gumbel reduced variate belonging to the Gumbel distribution (excluding any zero-values): reduced_variate = (flvol-loc)/scale # make sure that the reduced variate does not exceed the one reduced_variate = np.longdouble(np.minimum((flvol-loc)/scale, max_reduced_variate)) #~ print np.nanmin(reduced_variate) #~ print np.nanmax(reduced_variate) #~ print np.amin(reduced_variate) #~ print np.amax(reduced_variate) # transform the reduced variate into a probability (residual after removing the zero volume probability) p_residual = np.minimum(np.maximum(np.exp(-np.exp(-np.longdouble(reduced_variate))), np.longdouble(0.0)), np.longdouble(1.0)) #~ p_residual = np.minimum(np.maximum(np.exp(-np.exp(-np.longdouble(reduced_variate))), 0.0), 1.0) #~ print np.nanmin(p_residual) #~ print np.nanmax(p_residual) #~ print np.amin(p_residual) #~ print np.amax(p_residual) # transform from non-zero only distribution to zero-included distribution p = np.minimum(np.maximum(p_residual*(1.0 - p_zero) + p_zero, p_zero), max_p) # never larger than max_p # p = np.maximum(0.0, p) #~ print "" #~ print "p" #~ print np.nanmin(p) #~ print np.nanmax(p) #~ #~ print np.amin(p) #~ print np.amax(p) #~ print "p" #~ print "" # transform into a return period return_period = 1.0/(1.0-p) # assign maximum return period for p_zero = 1.0 (value is always zero) return_period[p_zero == 1.0000] = max_return_period # limit return period to maximum return period that can be assigned return_period[return_period > max_return_period_that_can_be_assigned] = max_return_period_that_can_be_assigned # cell with mv will be still mv return_period[p_zero == vos.MV] = vos.MV #~ # test values (calculated in the original Hessel's script, not needed) #~ test_p = p == 1 #~ diff_p = 1.0 - p print np.nanmin(return_period) print np.nanmax(return_period) print np.amin(return_period) print np.amax(return_period) print np.nanmin(return_period[p_zero != vos.MV]) print np.nanmax(return_period[p_zero != vos.MV]) print np.amin(return_period[p_zero != vos.MV]) print np.amax(return_period[p_zero != vos.MV]) #~ pcr.report(pcr.numpy2pcr(pcr.Scalar, np.float64(return_period), vos.MV), "return_period.map") #~ cmd = "aguila " + "return_period.map" #~ os.system(cmd) return pcr.numpy2pcr(pcr.Scalar, np.float64(return_period), vos.MV)
def joinMaps(inputTuple): '''Merges maps starting from an input tuple that specifies the output map name, the number of rows\ and the number rows, columns, ULL X and Y coordinates, cell length and the missing value identifer and a list of input maps''' outputFileName = inputTuple[0] nrRows = inputTuple[1] nrCols = inputTuple[2] xMin = inputTuple[3] yMax = inputTuple[4] cellLength = inputTuple[5] MV = inputTuple[6] fileNames = inputTuple[7] cloneFileName = inputTuple[8] #-echo to screen print 'combining files for %s' % outputFileName, #-get extent xMax = xMin + nrCols * cellLength yMin = yMax - nrRows * cellLength xCoordinates = xMin + np.arange(nrCols + 1) * cellLength yCoordinates = yMin + np.arange(nrRows + 1) * cellLength yCoordinates = np.flipud(yCoordinates) print 'between %.2f, %.2f and %.2f, %.2f' % (xMin, yMin, xMax, yMax) #-set output array variableArray = np.ones((nrRows, nrCols)) * MV #-iterate over maps for fileName in fileNames: print fileName attributeClone = getMapAttributesALL(fileName) cellLengthClone = attributeClone['cellsize'] rowsClone = attributeClone['rows'] colsClone = attributeClone['cols'] xULClone = attributeClone['xUL'] yULClone = attributeClone['yUL'] # check whether both maps have the same attributes and process process, nd = checkResolution(cellLength, cellLengthClone) if process: #-get coordinates and locations sampleXMin = xULClone sampleXMax = xULClone + colsClone * cellLengthClone sampleYMin = yULClone - rowsClone * cellLengthClone sampleYMax = yULClone sampleXCoordinates = sampleXMin + np.arange(colsClone + 1) * cellLengthClone sampleYCoordinates = sampleYMin + np.arange(rowsClone + 1) * cellLengthClone sampleYCoordinates = np.flipud(sampleYCoordinates) sampleXMin = getMax(xMin, sampleXMin) sampleXMax = getMin(xMax, sampleXMax) sampleYMin = getMax(yMin, sampleYMin) sampleYMax = getMin(yMax, sampleYMax) sampleRow0 = getPosition(sampleYMin, sampleYCoordinates, nd) sampleRow1 = getPosition(sampleYMax, sampleYCoordinates, nd) sampleCol0 = getPosition(sampleXMin, sampleXCoordinates, nd) sampleCol1 = getPosition(sampleXMax, sampleXCoordinates, nd) sampleRow0, sampleRow1 = checkRowPosition(sampleRow0, sampleRow1) variableRow0 = getPosition(sampleYMin, yCoordinates, nd) variableRow1 = getPosition(sampleYMax, yCoordinates, nd) variableCol0 = getPosition(sampleXMin, xCoordinates, nd) variableCol1 = getPosition(sampleXMax, xCoordinates, nd) variableRow0, variableRow1 = checkRowPosition( variableRow0, variableRow1) #-read sample array setclone(fileName) sampleArray = pcr2numpy(readmap(fileName), MV) sampleNrRows, sampleNrCols = sampleArray.shape #-create mask mask= (variableArray[variableRow0:variableRow1,variableCol0:variableCol1] == MV) &\ (sampleArray[sampleRow0:sampleRow1,sampleCol0:sampleCol1] <> MV) #-add values print ' adding values in %d, %d rows, columns from (x, y) %.3f, %.3f and %.3f, %.3f to position (row, col) %d, %d and %d, %d' %\ (sampleNrRows, sampleNrCols,sampleXMin,sampleYMin,sampleXMax,sampleYMax,variableRow0,variableCol0,variableRow1,variableCol1) variableArray[variableRow0:variableRow1,variableCol0:variableCol1][mask]= \ sampleArray[sampleRow0:sampleRow1,sampleCol0:sampleCol1][mask] else: print '%s does not match resolution and is not processed' % fileName #-report output map setclone(cloneFileName) report(numpy2pcr(Scalar, variableArray, MV), outputFileName)