def test_spatial_nearest_neighbor_radial(self): # build some test points x_in = [0, 4, 2, 0, 4] y_in = [0, 0, 2, 4, 4] x_out = [0, 2, 4] y_out = [1, 0, 3] pts_in = geometry.build_point_geometries(x_in, y_in) pts_out = geometry.build_point_geometries(x_out, y_out) # instantiate the interpolation class interpolation = SpatialInterpolation.NearestNeighborRadial # test multiple matches (large search distance) interpolation.set_param("max_distance", 10) mapped = interpolation.transform(pts_in, pts_out) self.assertTrue('None' not in mapped) self.assertTrue(len(mapped) == 3) # test no matches (very small search distance) interpolation.set_param("max_distance", 0) mapped = interpolation.transform(pts_in, pts_out) self.assertTrue('None' not in mapped) self.assertTrue(len(mapped) == 0) # test partial matches (small search distance) interpolation.set_param("max_distance", 1) mapped = interpolation.transform(pts_in, pts_out) self.assertTrue('None' not in mapped) self.assertTrue(len(mapped) == 2)
def test_spatial_nearest_neighbor_radial(self): # build some test points x_in = [0,4,2,0,4] y_in = [0,0,2,4,4] x_out = [0,2,4] y_out = [1,0,3] pts_in = geometry.build_point_geometries(x_in, y_in) pts_out = geometry.build_point_geometries(x_out, y_out) # instantiate the interpolation class interpolation = SpatialInterpolation.NearestNeighborRadial # test multiple matches (large search distance) interpolation.set_param("max_distance", 10) mapped = interpolation.transform(pts_in, pts_out) self.assertTrue('None' not in mapped) self.assertTrue(len(mapped) == 3) # test no matches (very small search distance) interpolation.set_param("max_distance", 0) mapped = interpolation.transform(pts_in, pts_out) self.assertTrue('None' not in mapped) self.assertTrue(len(mapped) == 0) # test partial matches (small search distance) interpolation.set_param("max_distance", 1) mapped = interpolation.transform(pts_in, pts_out) self.assertTrue('None' not in mapped) self.assertTrue(len(mapped) == 2)
def build_nodes(inp): geoms = {} lines = None with open(inp, "r") as f: lines = f.readlines() # first read all the node coordinates nodes = {} node_order = [] cidx = find(lines, lambda x: "COORDINATES" in x) for line in lines[cidx + 3 :]: if line.strip() == "": break vals = re.split(" +", line.strip()) if vals[0] in nodes: nodes[vals[0]].append((float(vals[1]), float(vals[2]))) else: nodes[vals[0]] = [(float(vals[1]), float(vals[2]))] node_order.append(vals[0]) for name, coords in nodes.iteritems(): ptx = [x for x, y in coords] pty = [y for x, y in coords] g = geom_utilites.build_point_geometries(ptx, pty) geoms[name] = {"geometry": g} return geoms
def setup_model(self, geomcount, valuecount): # create an exchange item unit = mdl.create_unit('cubic meters per second') variable = mdl.create_variable('streamflow') # create exchange item item = stdlib.ExchangeItem(name='Test', desc='Test Exchange Item', unit=unit, variable=variable) # set exchange item geometries xcoords = [i for i in range(geomcount)] ycoords = [i*1.5 for i in range(geomcount)] geoms = geometry.build_point_geometries(xcoords, ycoords) item.addGeometries2(geoms) # set exchange item values start_time = dt.now() end_time = start_time+timedelta(minutes=valuecount - 1) time_step = 60 # in seconds item.initializeDatesValues(start_datetime=start_time, end_datetime=end_time, timestep_in_seconds=time_step) values = numpy.random.rand(valuecount, geomcount) # for i in range(len(dates)): item.setValuesBySlice(values) #, time_index_slice=(0,num_timesteps,1)) return item, geoms
def calc_ti_geometries(self): elog.info("TOPMODEL: Building Geometry Objects") tigeoms = [] satgeoms = [] with open(self.topo_input, "r") as sr: lines = sr.readlines() ncols = int(lines[0].split(" ")[-1].strip()) nrows = int(lines[1].split(" ")[-1].strip()) lowerx = float(lines[2].split(" ")[-1].strip()) lowery = float(lines[3].split(" ")[-1].strip()) cellsize = float(lines[4].split(" ")[-1].strip()) nodata = float(lines[5].split(" ")[-1].strip()) # read ti data data = np.genfromtxt(self.topo_input, delimiter=" ", skip_header=6) # build X and Y coordinate arrays xi = np.linspace(lowerx, lowerx + ncols * cellsize, ncols) yi = np.linspace(lowery + nrows * cellsize, lowery, nrows) x, y = np.meshgrid(xi, yi) # generate 2d arrays from xi, yi x = x.ravel() # convert to 1-d y = y.ravel() # convert to 1-d data = data.ravel() # convert to 1-d # remove all nodata points from x, y arrays nonzero = np.where(data != nodata) x = x[nonzero] y = y[nonzero] tigeoms = geometry.build_point_geometries(x, y) self.ti_geoms = tigeoms
def test_create_exchange_item(self): item = ExchangeItem('e1', 'Test', 'Test Exchange Item') # -- Create Unit --# unit = mdl.create_unit('cubic meters per second') # -- Create Variable --# variable = mdl.create_variable('streamflow') # set data dates = [datetime.datetime(2014, 1, 1, 12, 0, 0) + datetime.timedelta(days=i) for i in range(0, 100)] vals = [random.random() for i in range(0,100)] x = [random.random() for i in range(0,150)] y = [random.random() for i in range(0,150)] geoms = geometry.build_point_geometries(x,y) item.addGeometries2(geoms) item.setValues2(vals, dates) self.assertTrue(item.name() == 'Test') self.assertTrue(item.description() == 'Test Exchange Item') self.assertTrue(len(item.getDates2()) == 100) # assert the correct number of values self.assertTrue(item.getEarliestTime2() == datetime.datetime(2014, 1, 1, 12, 0, 0)) # asser the correct start time self.assertTrue(item.getLatestTime2() == datetime.datetime(2014, 4, 10, 12, 0, 0)) # assert the correct end time self.assertTrue(len(item.getValues2()) == 100) # assert the correct number of values self.assertTrue([] not in item.getValues2()) # assert no empty values default_srs = osr.SpatialReference() default_srs.ImportFromEPSG(4269) self.assertTrue(item.srs().ExportToWkt() == default_srs.ExportToWkt())
def calc_ti_geometries(self): elog.info('TOPMODEL: Building Geometry Objects') tigeoms = [] satgeoms = [] with open(self.topo_input, 'r') as sr: lines = sr.readlines() ncols = int(lines[0].split(' ')[-1].strip()) nrows = int(lines[1].split(' ')[-1].strip()) lowerx = float(lines[2].split(' ')[-1].strip()) lowery = float(lines[3].split(' ')[-1].strip()) cellsize = float(lines[4].split(' ')[-1].strip()) nodata = float(lines[5].split(' ')[-1].strip()) # read ti data data = np.genfromtxt(self.topo_input, delimiter=' ', skip_header=6) # build X and Y coordinate arrays xi = np.linspace(lowerx, lowerx + ncols * cellsize, ncols) yi = np.linspace(lowery + nrows * cellsize, lowery, nrows) x, y = np.meshgrid(xi, yi) # generate 2d arrays from xi, yi x = x.ravel() # convert to 1-d y = y.ravel() # convert to 1-d data = data.ravel() # convert to 1-d # remove all nodata points from x, y arrays nonzero = np.where(data != nodata) x = x[nonzero] y = y[nonzero] tigeoms = geometry.build_point_geometries(x, y) self.ti_geoms = tigeoms
def build_nodes(inp): geoms = {} lines = None with open(inp,'r') as f: lines = f.readlines() # first read all the node coordinates nodes = {} node_order = [] cidx = find(lines, lambda x: 'COORDINATES' in x) for line in lines[cidx+3:]: if line.strip() == '': break vals = re.split(' +',line.strip()) if vals[0] in nodes: nodes[vals[0]].append((float(vals[1]), float(vals[2]))) else: nodes[vals[0]] = [(float(vals[1]), float(vals[2]))] node_order.append(vals[0]) for name,coords in nodes.iteritems(): ptx = [x for x,y in coords] pty = [y for x,y in coords] g = geom_utilites.build_point_geometries(ptx, pty) geoms[name] = {'geometry':g} return geoms
def insert_simulation(db, user, geomcount, valuecount): print '%d geometries, %d values per geometry' % (geomcount, valuecount) print 'Total number of data values: %d' % (valuecount * geomcount) # create an exchange item unit = mdl.create_unit('cubic meters per second') variable = mdl.create_variable('streamflow') # create exchange item item = stdlib.ExchangeItem(name='Test', desc='Test Exchange Item', unit=unit, variable=variable) # set exchange item geometries xcoords = numpy.random.rand(geomcount) ycoords = numpy.random.rand(geomcount) points = geometry.build_point_geometries(xcoords, ycoords) item.addGeometries2(points) # set exchange item values start_time = dt.now() # set start time to 'now' end_time = start_time + timedelta(days=valuecount-1) # create endtime base on the number of values current_time = start_time # initial time dates = [] # list to hold dates values = [] # list to hold values for each date # populate dates list while current_time <= end_time: # add date dates.append(current_time) # add some random values for each geometry values.append([random.random() for pt in points] ) # increment time by 1 day current_time += timedelta(days=1) # set dates and values in the exchange item item.setValues2(values, dates) # create the simulation st = dt(2014, 3, 1, 12, 0, 0) et = dt(2014, 3, 1, 23, 0, 0) description = 'Some model descipription' name = 'test simulation' # turn off verbosity for simulation insert os.environ['LOGGING_SHOWINFO'] = '0' os.environ['LOGGING_SHOWDEBUG'] = '0' db.create_simulation('My Simulation', user[0], None, item, st, et, 1, 'days', description, name) os.environ['LOGGING_SHOWINFO'] = '1' # query simulations simulations = db.read.getAllSimulations() simulation = db.read.getSimulationByName('My Simulation') assert simulation is not None
def test_add_geoms2(self): # add a single geom pt = geometry.build_point_geometry(10, 11) x, y, z = pt.GetPoint() self.assertTrue(isinstance(pt, Geometry2)) ret = self.item.addGeometry(pt) self.assertTrue(ret) g = self.item.getGeometries2(-1) self.assertTrue(isinstance(g, Geometry2)) x, y, z = g.GetPoint() self.assertTrue(x == 10) self.assertTrue(y == 11) self.assertTrue(z == 0) # add an invalid geometry ret = self.item.addGeometry((0, 1, 1)) self.assertFalse(ret) # add many of geometries xs = range(100, 110) ys = range(110, 120) pts = geometry.build_point_geometries(xs, ys) geoms = [] for pt in pts: geoms.append(pt) ret = self.item.addGeometries2(geoms) self.assertTrue(ret) g = self.item.getGeometries2(-1) self.assertTrue(isinstance(g, Geometry2)) x, y, z = g.GetPoint() self.assertTrue(x == 109) self.assertTrue(y == 119) self.assertTrue(z == 0) # add a mix of valid and invalid geometries geoms = [] for i in range(100, 110): if i % 2 == 0: pt = geometry.build_point_geometry(i, i + 1) geoms.append(pt) else: geoms.append((i, i + 1)) ret = self.item.addGeometries2(geoms) self.assertFalse(ret) g = self.item.getGeometries2(-1) self.assertTrue(isinstance(g, Geometry2)) x, y, z = g.GetPoint() self.assertTrue(x == 108) self.assertTrue(y == 109) self.assertTrue(z == 0)
def test_add_geoms2(self): # add a single geom pt = geometry.build_point_geometry(10, 11) x,y,z = pt.GetPoint() self.assertTrue(isinstance(pt, Geometry2)) ret = self.item.addGeometry(pt) self.assertTrue(ret) g = self.item.getGeometries2(-1) self.assertTrue(isinstance(g, Geometry2)) x,y,z = g.GetPoint() self.assertTrue(x == 10) self.assertTrue(y == 11) self.assertTrue(z == 0) # add an invalid geometry ret = self.item.addGeometry((0,1,1)) self.assertFalse(ret) # add many of geometries xs = range(100,110) ys = range(110,120) pts = geometry.build_point_geometries(xs, ys) geoms = [] for pt in pts: geoms.append(pt) ret = self.item.addGeometries2(geoms) self.assertTrue(ret) g = self.item.getGeometries2(-1) self.assertTrue(isinstance(g, Geometry2)) x,y,z = g.GetPoint() self.assertTrue(x == 109) self.assertTrue(y == 119) self.assertTrue(z == 0) # add a mix of valid and invalid geometries geoms = [] for i in range(100,110): if i % 2 == 0: pt = geometry.build_point_geometry(i, i + 1) geoms.append(pt) else: geoms.append((i,i+1)) ret = self.item.addGeometries2(geoms) self.assertFalse(ret) g = self.item.getGeometries2(-1) self.assertTrue(isinstance(g, Geometry2)) x,y,z = g.GetPoint() self.assertTrue(x == 108) self.assertTrue(y == 109) self.assertTrue(z == 0)
def setUp(self): # -- Create Unit --# unit = mdl.create_unit('cubic meters per second') # -- Create Variable --# variable = mdl.create_variable('streamflow') # create exchange item self.item = ExchangeItem(name='Test', desc='Test Exchange Item', unit=unit, variable=variable) coords = [(1,2),(2,3),(3,4),(4,5),(5,6),(6,7),(7,8),(8,9),(9,10)] x,y = zip(*coords) geoms = geometry.build_point_geometries(x, y) self.item.addGeometries2(geoms)
def build_geometries(self, xcoords, ycoords): """ builds point geometries consistent with the native UEB ordering :param xcoords: list of x coordinates :param xyoords: list of xycoordinates :return: a list of geometries """ # build a meshgrid x, y = numpy.meshgrid(xcoords, ycoords) # x any y coords are paired using Fortran ordering to be consistent with the way activeCells are ordered. This # is necessary to ensure that calculation looping is maintained in the run function. x = numpy.ravel(x, 'F') y = numpy.ravel(y, 'F') # build point geometries return geometry.build_point_geometries(x, y)
def testSetValuesByTime(self): item = ExchangeItem('e1', 'Test', 'Test Exchange Item') # set start and end dates st = datetime.datetime(2014, 1, 1, 12, 0, 0) et = datetime.datetime(2014, 2, 1, 12, 0, 0) timestep_in_seconds = 3600 # build geometries x = [random.random() for i in range(0, 150)] y = [random.random() for i in range(0, 150)] geoms = geometry.build_point_geometries(x, y) item.addGeometries2(geoms) values = range(30) # set values for geometry=0, beginning at the start time success = item.setValuesBySlice(values, time_index_slice=(0, 30, 1), geometry_index_slice=(0, 1, 1)) self.assertFalse(success) # initialize the dates and values containers item.initializeDatesValues(st, et, timestep_in_seconds) success = item.setValuesBySlice(values, time_index_slice=(0, 30, 1), geometry_index_slice=(0, 1, 1)) self.assertTrue(success) # get the data geoms = item.getGeometries2() times = item.getDates2() vals = item.getValues2() # assert that data is set properly self.assertTrue(len(geoms) == 150) self.assertTrue(times[0][1] == st) self.assertTrue(times[-1][1] == et) self.assertTrue( (times[-1][1] - times[-2][1]).total_seconds() == float(timestep_in_seconds)) self.assertTrue(vals.shape[0] == len(times)) self.assertTrue(vals.shape[1] == len(geoms))
def setUp(self): # -- Create Unit --# unit = mdl.create_unit('cubic meters per second') # -- Create Variable --# variable = mdl.create_variable('streamflow') # create exchange item self.item = ExchangeItem(name='Test', desc='Test Exchange Item', unit=unit, variable=variable) coords = [(1, 2), (2, 3), (3, 4), (4, 5), (5, 6), (6, 7), (7, 8), (8, 9), (9, 10)] x, y = zip(*coords) geoms = geometry.build_point_geometries(x, y) self.item.addGeometries2(geoms)
def setUp(self): self.item = ExchangeItem('e1', 'Test', 'Test Exchange Item') # set start and end dates self.st = datetime.datetime(2014, 1, 1, 12, 0, 0) self.et = datetime.datetime(2014, 2, 1, 12, 0, 0) self.timestep_in_seconds = 3600 # build geometries x = [random.random() for i in range(0,150)] y = [random.random() for i in range(0,150)] geoms = geometry.build_point_geometries(x,y) self.item.addGeometries2(geoms) values = range(30) # initialize the dates and values containers self.item.initializeDatesValues(self.st, self.et, self.timestep_in_seconds) success = self.item.setValuesBySlice(values, time_index_slice=(0,30,1), geometry_index_slice=(0, 1, 1)) self.assertTrue(success)
def testSetValuesByTime(self): item = ExchangeItem('e1', 'Test', 'Test Exchange Item') # set start and end dates st = datetime.datetime(2014, 1, 1, 12, 0, 0) et = datetime.datetime(2014, 2, 1, 12, 0, 0) timestep_in_seconds = 3600 # build geometries x = [random.random() for i in range(0,150)] y = [random.random() for i in range(0,150)] geoms = geometry.build_point_geometries(x,y) item.addGeometries2(geoms) values = range(30) # set values for geometry=0, beginning at the start time success = item.setValuesBySlice(values, time_index_slice=(0,30,1), geometry_index_slice=(0, 1, 1)) self.assertFalse(success) # initialize the dates and values containers item.initializeDatesValues(st, et, timestep_in_seconds) success = item.setValuesBySlice(values, time_index_slice=(0,30,1), geometry_index_slice=(0, 1, 1)) self.assertTrue(success) # get the data geoms = item.getGeometries2() times = item.getDates2() vals = item.getValues2() # assert that data is set properly self.assertTrue(len(geoms) == 150) self.assertTrue(times[0][1] == st) self.assertTrue(times[-1][1] == et) self.assertTrue((times[-1][1] - times[-2][1]).total_seconds() == float(timestep_in_seconds)) self.assertTrue(vals.shape[0] == len(times)) self.assertTrue(vals.shape[1] == len(geoms))
def setUp(self): self.item = ExchangeItem('e1', 'Test', 'Test Exchange Item') # set start and end dates self.st = datetime.datetime(2014, 1, 1, 12, 0, 0) self.et = datetime.datetime(2014, 2, 1, 12, 0, 0) self.timestep_in_seconds = 3600 # build geometries x = [random.random() for i in range(0, 150)] y = [random.random() for i in range(0, 150)] geoms = geometry.build_point_geometries(x, y) self.item.addGeometries2(geoms) values = range(30) # initialize the dates and values containers self.item.initializeDatesValues(self.st, self.et, self.timestep_in_seconds) success = self.item.setValuesBySlice(values, time_index_slice=(0, 30, 1), geometry_index_slice=(0, 1, 1)) self.assertTrue(success)
def calc_ti_geometries(self): sPrint('Reading Topographic Indices') with open(self.topo_input, 'r') as sr: lines = sr.readlines() ncols = int(lines[0].split(' ')[-1].strip()) nrows = int(lines[1].split(' ')[-1].strip()) lowerx = float(lines[2].split(' ')[-1].strip()) lowery = float(lines[3].split(' ')[-1].strip()) cellsize = float(lines[4].split(' ')[-1].strip()) nodata = float(lines[5].split(' ')[-1].strip()) # read ti data data = np.genfromtxt(self.topo_input, delimiter=' ', skip_header=6) # build X and Y coordinate arrays xi = np.linspace(lowerx, lowerx + ncols * cellsize, ncols) yi = np.linspace(lowery + nrows * cellsize, lowery, nrows) x, y = np.meshgrid(xi, yi) # generate 2d arrays from xi, yi x = x.ravel() # convert to 1-d y = y.ravel() # convert to 1-d data = data.ravel() # convert to 1-d # remove all nodata points from x, y arrays nonzero = np.where(data != nodata) x = x[nonzero] y = y[nonzero] with open(self.fac_input, 'r') as fac: lines = fac.readlines() fac_ncols = int(lines[0].split(' ')[-1].strip()) fac_nrows = int(lines[1].split(' ')[-1].strip()) fac_lowerx = float(lines[2].split(' ')[-1].strip()) fac_lowery = float(lines[3].split(' ')[-1].strip()) fac_cellsize = float(lines[4].split(' ')[-1].strip()) fac_nodata = float(lines[5].split(' ')[-1].strip()) # read ti data fac_data = np.genfromtxt(self.fac_input, delimiter=' ', skip_header=6) # build X and Y coordinate arrays xi = np.linspace(fac_lowerx, fac_lowerx + fac_ncols * fac_cellsize, fac_ncols) yi = np.linspace(fac_lowery + fac_nrows * fac_cellsize, fac_lowery, fac_nrows) x, y = np.meshgrid(xi, yi) # generate 2d arrays from xi, yi x = x.ravel() # convert to 1-d y = y.ravel() # convert to 1-d fac_data = fac_data.ravel() # convert to 1-d # isolate the rivers based on threshold threshold = np.max(fac_data) # todo, user input rivers = np.where((fac_data != fac_nodata) & (fac_data >= threshold)) # select only the x and y values that are considered rivers x = x[rivers] y = y[rivers] self.rivers_idxs = rivers sPrint('Building Geometry Objects') tigeoms = geometry.build_point_geometries(x, y) self.ti_geoms = tigeoms
def __init__(self, args): super(Wrapper, self).__init__() handle = netCDF4.Dataset(args['ncpath'], 'r') variables = handle.variables.keys() tdim = args['tdim'] xdim = args['xdim'] ydim = args['ydim'] tunit = {args['tunit']: 1} if isinstance(args['starttime'], datetime.datetime): st = args['starttime'] else: st = parser.parse(args['starttime']) # make sure the variables provided exist in the nc file assert tdim in variables, 'time variable name not specified. Cannot continue' assert xdim in variables, 'x dimension variable name not specified. Cannot continue' assert ydim in variables, 'y dimension variable name not specified. Cannot continue' # get data for these variables timesteps = handle.variables[tdim][:] times = [] for ts in timesteps: # update the time unit value tunit[args['tunit']] = ts # unpack the tunit dictionary to create a timedelta object dt = datetime.timedelta(**tunit) times.append(st + dt) variables.remove(tdim) x = handle.variables[xdim][:] variables.remove(xdim) y = handle.variables[ydim][:] variables.remove(ydim) # create flattened lists of x,y coords from meshgrid xcoords, ycoords = numpy.meshgrid(x, y) xcoords = xcoords.flatten() ycoords = ycoords.flatten() # loop through the remaining variables and expose them as outputs # for var in variables: for v in range(len(variables)): var = variables[v] # create a unit unit = stdlib.Unit() unit.UnitName(handle.variables[var].units if 'units' in dir(handle.variables[var]) else 'N/A') unit.UnitTypeCV("N/A") unit.UnitAbbreviation("N/A") # create a variable variable = stdlib.Variable() variable.VariableNameCV(handle.variables[var].name) variable.VariableDefinition("N/A") # create geometries geoms = geometry.build_point_geometries( xcoords, ycoords) # This takes way to long # create exchange item oei = stdlib.ExchangeItem( name=variable.VariableNameCV(), desc='Autogenerated variable parsed from %s' % args['ncpath'], geometry=geoms, unit=unit, variable=variable, type=stdlib.ExchangeItemType.OUTPUT) # flatten each timestep of the data values = [v.flatten() for v in handle.variables[var][:]] # set these data. Skip occurrences with mismatched values and times arrays if len(values) == len(times): success = oei.setValues2(values, times) # only expose the exchange item if data was set properly if success: # save the oei self.outputs(oei) # set metadata name = args['ncpath'].split('/')[-1] self.name(name) self.description('NetCDF data component, ' + name) self.simulation_start(times[0]) self.simulation_end(times[-1]) self.status(stdlib.Status.READY)
def __init__(self, config_params): super(ueb, self).__init__(config_params) # build inputs and outputs io = mdl.build_exchange_items_from_config(config_params) # set input and output exchange items self.inputs(value=io[stdlib.ExchangeItemType.INPUT]) self.outputs(value=io[stdlib.ExchangeItemType.OUTPUT]) # grab the C library path and the control file path lib = config_params['lib'] conFile = config_params['control'] # load the UEB C library self.__uebLib = cdll.LoadLibrary(join(os.path.dirname(__file__), lib)) # save the current directory for saving output data self.curdir = os.path.dirname(os.path.abspath(conFile)) # the base directory for the control file is used to convert relative paths into absolute paths self.base_dir = os.path.dirname(conFile) # get param, sitevar, input, output, and watershed files with open(conFile, 'r') as f: # lines = f.readlines() lines = f.read().splitlines() # this will auto strip the \n \r C_paramFile = os.path.join(self.base_dir, lines[1]) C_sitevarFile = os.path.join(self.base_dir, lines[2]) C_inputconFile = os.path.join(self.base_dir, lines[3]) C_outputconFile = os.path.join(self.base_dir, lines[4]) C_watershedFile = os.path.join(self.base_dir, lines[5]) C_wsvarName = lines[6].split(' ')[0] C_wsycorName = lines[6].split(' ')[1] C_wsxcorName = lines[6].split(' ')[2] C_aggoutputconFile = os.path.join(self.base_dir, lines[7]) C_aggoutputFile = os.path.join(self.base_dir, lines[8]) ModelStartDate = [ int(float(l)) for l in lines[9].split(' ') if l != '' ] ModelEndDate = [ int(float(l)) for l in lines[10].split(' ') if l != '' ] ModelDt = float(lines[11]) ModelUTCOffset = float(lines[12]) inpDailyorSubdaily = bool(lines[13] == True) self.outtStride, outyStep, outxStep = [ int(s) for s in lines[14].split(' ') ] C_wsxcorArray = c_float() C_wsycorArray = c_float() self.C_wsArray = pointer(pointer(c_int32())) self.C_dimlen1 = c_int() self.C_dimlen2 = c_int() totalgrid = 0 self.C_wsfillVal = c_int(-9999) npar = c_int(32) tinitTime = c_int(0) self.C_parvalArray = pointer(c_float(0)) numOut = 70 # hack: number of outputs? self.C_pOut = pointer(pointOutput()) C_aggOut = pointer(aggOutput()) self.C_ncOut = pointer(ncOutput()) self.C_npout = c_int(0) self.C_nncout = c_int(0) C_naggout = c_int(0) C_nZones = c_int(0) C_tNameout = c_char_p("time") tunits = (c_char * 256)() C_tUnitsout = pointer(tunits) C_tlong_name = c_char_p("time") C_tcalendar = c_char_p("standard") self.t_out = pointer(c_float(0)) C_out_fillVal = c_float(-9999.0) self.C_outDimord = c_int(0) C_aggoutDimord = c_int(1) self.outvarindx = c_int(17) # aggoutvarindx = c_int(17) # size = c_int() # rank = c_int() # irank = c_int() # jrank = c_int() # startTimeT = c_double(0.0) # TotalTime = c_double(0.0) # totalmodelrunTime = c_double(0.0) # TsReadTime = c_double(0.0) # TSStartTime = c_double() # ComputeStartTime = c_double() # ComputeTime = c_double(0.0) # OutWriteTime = c_double() self.uebVars = (c_char_p * 70)( "Year", "Month", "Day", "dHour", "atff", "HRI", "Eacl", "Ema", "conZen", "Ta", "P", "V", "RH", "Qsi", "Qli", "Qnet", "Us", "SWE", "tausn", "Pr", "Ps", "Alb", "QHs", "QEs", "Es", "SWIT", "QMs", "Q", "FM", "Tave", "TSURFs", "cump", "cumes", "cumMr", "Qnet", "smelt", "refDepth", "totalRefDepth", "cf", "Taufb", "Taufd", "Qsib", "Qsid", "Taub", "Taud", "Qsns", "Qsnc", "Qlns", "Qlnc", "Vz", "Rkinsc", "Rkinc", "Inmax", "intc", "ieff", "Ur", "Wc", "Tc", "Tac", "QHc", "QEc", "Ec", "Qpc", "Qmc", "Mc", "FMc", "SWIGM", "SWISM", "SWIR", "errMB") C_zName = c_char_p("Outletlocations") C_tcorvar = pointer((c_float * 13)()) self.C_tsvarArray = pointer((c_float * 13)()) C_tsvarArrayTemp = pointer((c_float * 5)()) #todo: [#] == pointer, * == pointer self.C_tsvarArray = pointer((POINTER(c_float) * 13)()) C_ntimesteps = pointer((c_int * 5)()) # create pointer to instance of sitevar struct array self.C_strsvArray = pointer((sitevar * 32)()) # create pointer to instance of inpforcvar struct array self.C_strinpforcArray = pointer((inpforcvar * 13)()) # mask = c_float() # pcap_lookupnet(dev, ctypes.byref(net), ctypes.byref(mask), errbuf) # read watershed netcdf file self.__uebLib.readwsncFile(C_watershedFile, C_wsvarName, C_wsycorName, C_wsxcorName, byref(C_wsycorArray), byref(C_wsxcorArray), byref(self.C_wsArray), byref(self.C_dimlen1), byref(self.C_dimlen2), byref(self.C_wsfillVal)) wsArray1D = numpy.empty((self.C_dimlen1.value * self.C_dimlen2.value), dtype=numpy.float) for i in xrange(self.C_dimlen1.value): for j in xrange(self.C_dimlen2.value): wsArray1D[i * self.C_dimlen2.value + j] = self.C_wsArray[i][j] # zvalues is the unique set of wsArray1D zValues = list(set(wsArray1D)) # fillset = [wsfillVal] # zVal is the set of zValues that do not equal wsFillVal zVal = [ zValues[i] for i in xrange(len(zValues)) if zValues[i] != self.C_wsfillVal ] C_nZones = len(zVal) z_ycor = [0.0 for i in xrange(C_nZones)] z_xcor = [0.0 for i in xrange(C_nZones)] # read params (#194) self.__uebLib.readParams(C_paramFile, byref(self.C_parvalArray), npar) # read site variables (#200) self.__uebLib.readSiteVars(C_sitevarFile, byref(self.C_strsvArray)) # read 2d NetCDF Data for i in range(0, 32): a = self.C_strsvArray.contents[i] if a.svType == 1: # print "%d %s %s\n" % (i, a.svFile,a.svVarName) retvalue = self.__uebLib.read2DNC( os.path.join(self.base_dir, a.svFile), a.svVarName, byref(a.svArrayValues)) #//read input /forcing control file--all possible entries of input control have to be provided #readInputForcVars(inputconFile, strinpforcArray); # read input force variables (main.cpp, line 219) self.__uebLib.readInputForcVars(cast(C_inputconFile, c_char_p), self.C_strinpforcArray) # elog.info('UEB Start Date: %s' % sd.strftime("%m-%d-%Y %H:%M:%S")) # elog.info('UEB End Date: %s' % ed.strftime("%m-%d-%Y %H:%M:%S")) # calculate model time span as a julian date (main.cpp, line 220) modelSpan = jdutil.datetime_to_jd(datetime.datetime(*ModelEndDate)) - \ jdutil.datetime_to_jd(datetime.datetime(*ModelStartDate)) # setup the model start dates as UEB expects them ModelStartHour = ModelStartDate.pop( -1) # an integer representing the start hour (24 hour time) ModelEndHour = ModelEndDate.pop( -1) # an integer representing the end hour (24 hour time) # ModelStartDate is a 3 element array: [year, month, day] # ModelEndDate is a 3 element array: [year, month, day] # convert Simulation Time parameters into ctypes self.C_ModelStartDate = (c_int * len(ModelStartDate))(*ModelStartDate) self.C_ModelEndDate = (c_int * len(ModelEndDate))(*ModelEndDate) self.C_ModelDt = c_double(ModelDt) self.C_ModelUTCOffset = c_double(ModelUTCOffset) self.C_ModelStartHour = c_double(ModelStartHour) self.C_ModelEndHour = c_double(ModelEndHour) # calculate model time steps (main.cpp, line 222) self.numTimeStep = int(math.ceil(modelSpan * (24. / ModelDt))) + 1 # initialize C_tsvarArray values (this replaces __uebLib.readTextData) self.initialize_timeseries_variable_array(self.C_strinpforcArray, self.numTimeStep) # NOTE: C_strinpforcArray stores info about the forcing data files # # read forcing data (main.cpp, line 226) # if self.C_strsvArray.contents[16].svType != 3: # no accumulation zone (fixme: ???) # for it in xrange(13): # inftype = self.C_strinpforcArray.contents[it].infType # print 'infFile: ',self.C_strinpforcArray.contents[it].infFile # if inftype == 0: # # # read the files stored in C_strinpforcArray and populated C_tsvarArray # self.__uebLib.readTextData(os.path.join(self.base_dir, self.C_strinpforcArray.contents[it].infFile), byref(self.C_tsvarArray.contents[it]), byref(C_ntimesteps[0])) # # elif inftype == 2 or inftype == -1: # self.C_tsvarArray.contents[it] = (c_float * 2)() # C_ntimesteps.contents[0] = 2 # # copy the default value if a single value is the option # self.C_tsvarArray.contents[it][0] = self.C_strinpforcArray.contents[it].infType # self.C_tsvarArray.contents[it][1] = self.C_strinpforcArray.contents[it].infdefValue # :: this array is initialized to (numOut+1, numTimeStep+1) rather than (numOut, numTimeStep) # :: b/c otherwise the calculations from RunUEB are incorrect for the first row. # :: e.g. # :: 2009 2010 5 30.000 23.999979 # :: rather than: # :: 2009 10 1 0.000 0.569902 # :: # :: I thought this was b/c numpy.float32 (and 64) are smaller than c_float, however this change # :: below didn't fix the problem. # :: # create a numpy array for outputs self.outvarArray = numpy.zeros(shape=(numOut + 1, self.numTimeStep), dtype=numpy.float, order="C") # arrays_old = self.outvarArray.astype(numpy.float32) arrays = self.outvarArray.astype(c_float) rows, cols = self.outvarArray.shape arrays_as_list = list(arrays) #get ctypes handles ctypes_arrays = [ numpy.ctypeslib.as_ctypes(array) for array in arrays_as_list ] #Pack into pointer array self.C_outvarArray = (POINTER(c_float) * rows)(*ctypes_arrays) # x = numpy.zeros(shape=(numOut, self.numTimeStep), dtype=numpy.float, order="C") # _floatpp = numpy.ctypeslib.ndpointer(dtype=numpy.uintp, ndim=1, flags='C') # xpp = (x.__array_interface__['data'][0] + numpy.arange(x.shape[0])*x.strides[0]).astype(numpy.uintp) # self.C_outvarArray = pointer(((POINTER(c_float) * self.numTimeStep) * numOut)()) # a = (c_float * self.numTimeStep)() # outvarArray = pointer((a * numOut)()) # for i in xrange(numOut): # outvarArray[i] = pointer((c_float * self.numTimeStep)()) # total grid size to compute progess totalgrid = self.C_dimlen1.value * self.C_dimlen2.value # read output control file (main.cpp, line 251) # readOutputControl(outputconFile, aggoutputconFile, pOut, ncOut, aggOut, npout, nncout, naggout); self.__uebLib.readOutputControl(cast(C_outputconFile, c_char_p), cast(C_aggoutputconFile, c_char_p), byref(self.C_pOut), byref(self.C_ncOut), byref(C_aggOut), byref(self.C_npout), byref(self.C_nncout), byref(C_naggout)) # create output netcdf self.C_outtSteps = self.numTimeStep / self.outtStride self.t_out = numpy.empty(shape=(self.C_outtSteps), dtype=numpy.float, order="C") for i in xrange(self.C_outtSteps): self.t_out[i] = i * self.outtStride * ModelDt # initialize the output arrays aggoutvarArray = numpy.zeros( (C_nZones, C_naggout.value, self.C_outtSteps), dtype=numpy.float) totalAgg = numpy.empty((self.C_outtSteps, ), dtype=numpy.float) ZonesArr = numpy.zeros((C_nZones, ), dtype=numpy.int32) # main.cpp, line 290 # CREATE 3D NC OUTPUT FILES # convert self.t_out into a float pointer C_t_out = self.t_out.ctypes.data_as(POINTER(c_float)) for i in xrange(self.C_nncout.value): ''' for (int icout = 0; icout < nncout; icout++) retvalue = create3DNC_uebOutputs(ncOut[icout].outfName, (const char*)ncOut[icout].symbol, (const char*)ncOut[icout].units, tNameout, tUnitsout, tlong_name, tcalendar, outtSteps, outDimord, self.t_out, &out_fillVal, watershedFile, wsvarName, wsycorName, wsxcorName); ''' retvalue = self.__uebLib.create3DNC_uebOutputs( self.C_ncOut[i].outfName, cast(self.C_ncOut[i].symbol, c_char_p), cast(self.C_ncOut[i].units, c_char_p), C_tNameout, C_tUnitsout, C_tlong_name, C_tcalendar, self.C_outtSteps, self.C_outDimord, C_t_out, byref(C_out_fillVal), C_watershedFile, C_wsvarName, C_wsycorName, C_wsxcorName) # CREATE 3D NC AGGREGATE OUTPUT FILE # convert z_ycor and x_xcor from list into ctype C_z_xcor = numpy.asarray(z_xcor).ctypes.data_as(POINTER(c_float)) C_z_ycor = numpy.asarray(z_ycor).ctypes.data_as(POINTER(c_float)) retvalue = self.__uebLib.create3DNC_uebAggregatedOutputs( C_aggoutputFile, C_aggOut, C_naggout, C_tNameout, C_tUnitsout, C_tlong_name, C_tcalendar, self.C_outtSteps, C_aggoutDimord, C_t_out, byref(C_out_fillVal), C_watershedFile, C_wsvarName, C_wsycorName, C_wsxcorName, C_nZones, C_zName, C_z_ycor, C_z_xcor) # todo: create output element set # print 'Output Calculations available at: ' # for pid in xrange(self.C_npout.value): # print " Point(",self.C_pOut[pid].xcoord,", ",self.C_pOut[pid].ycoord,') ' # todo: This is where UEB grid points are defined!, expose these as input/output spatial objects # main.cpp, line 303 self.activeCells = [] # print 'Calculations will be performed at: ' for iy in xrange(self.C_dimlen1.value): for jx in xrange(self.C_dimlen2.value): if self.C_wsArray[iy][ jx] != self.C_wsfillVal.value and self.C_strsvArray.contents[ 16].svType != 3: # print " Point(",jx,", ",iy,') ' self.activeCells.append((iy, jx)) # build output exchange items xcoords = [] ycoords = [] for pid in xrange(self.C_npout.value): xcoords.append(self.C_pOut[pid].xcoord) ycoords.append(self.C_pOut[pid].ycoord) self.pts = geometry.build_point_geometries(xcoords, ycoords) self.__swe = self.outputs()['Snow Water Equivalent'] self.__swit = self.outputs()['Surface Water Input Total'] self.__swe.addGeometries2(self.pts) self.__swit.addGeometries2(self.pts) # build input exchange items ds = nc.Dataset(C_watershedFile) Xlist = ds.variables['x'] Ylist = ds.variables['y'] self.geoms = self.build_geometries(Xlist, Ylist) self.inputs()['Precipitation'].addGeometries2(self.geoms) self.inputs()['Temperature'].addGeometries2(self.geoms) # set start, end, and timestep parameters ts = datetime.timedelta(hours=ModelDt) self.time_step(ts.total_seconds()) sd = datetime.datetime(*ModelStartDate) ed = datetime.datetime(*ModelEndDate) self.simulation_start(sd) self.simulation_end(ed)
def insert_simulation(db, user, geomcount, valuecount): print '%d geometries, %d values per geometry' % (geomcount, valuecount) print 'Total number of data values: %d' % (valuecount * geomcount) # create an exchange item unit = mdl.create_unit('cubic meters per second') variable = mdl.create_variable('streamflow') # create exchange item item = stdlib.ExchangeItem(name='Test', desc='Test Exchange Item', unit=unit, variable=variable) # set exchange item geometries xcoords = numpy.random.rand(geomcount) ycoords = numpy.random.rand(geomcount) points = geometry.build_point_geometries(xcoords, ycoords) item.addGeometries2(points) # set exchange item values start_time = dt.now() # set start time to 'now' end_time = start_time + timedelta( days=valuecount - 1) # create endtime base on the number of values current_time = start_time # initial time dates = [] # list to hold dates values = [] # list to hold values for each date # populate dates list while current_time <= end_time: # add date dates.append(current_time) # add some random values for each geometry values.append([random.random() for pt in points]) # increment time by 1 day current_time += timedelta(days=1) # set dates and values in the exchange item item.setValues2(values, dates) # create the simulation st = dt(2014, 3, 1, 12, 0, 0) et = dt(2014, 3, 1, 23, 0, 0) description = 'Some model descipription' name = 'test simulation' # turn off verbosity for simulation insert os.environ['LOGGING_SHOWINFO'] = '0' os.environ['LOGGING_SHOWDEBUG'] = '0' db.create_simulation('My Simulation', user[0], None, item, st, et, 1, 'days', description, name) os.environ['LOGGING_SHOWINFO'] = '1' # query simulations simulations = db.read.getAllSimulations() simulation = db.read.getSimulationByName('My Simulation') assert simulation is not None
def __init__(self, config_params): super(ueb, self).__init__(config_params) # build inputs and outputs io = mdl.build_exchange_items_from_config(config_params) # set input and output exchange items self.inputs(value=io[stdlib.ExchangeItemType.INPUT]) self.outputs(value=io[stdlib.ExchangeItemType.OUTPUT]) # grab the C library path and the control file path lib = config_params['lib'] conFile = config_params['control'] # load the UEB C library self.__uebLib = cdll.LoadLibrary(join(os.path.dirname(__file__), lib)) # save the current directory for saving output data self.curdir = os.path.dirname(os.path.abspath(conFile)) # the base directory for the control file is used to convert relative paths into absolute paths self.base_dir = os.path.dirname(conFile) # get param, sitevar, input, output, and watershed files with open(conFile, 'r') as f: # lines = f.readlines() lines = f.read().splitlines() # this will auto strip the \n \r C_paramFile = os.path.join(self.base_dir, lines[1]) C_sitevarFile = os.path.join(self.base_dir, lines[2]) C_inputconFile = os.path.join(self.base_dir, lines[3]) C_outputconFile = os.path.join(self.base_dir, lines[4]) C_watershedFile = os.path.join(self.base_dir, lines[5]) C_wsvarName = lines[6].split(' ')[0] C_wsycorName = lines[6].split(' ')[1] C_wsxcorName = lines[6].split(' ')[2] C_aggoutputconFile = os.path.join(self.base_dir, lines[7]) C_aggoutputFile = os.path.join(self.base_dir, lines[8]) ModelStartDate = [int(float(l)) for l in lines[9].split(' ') if l != ''] ModelEndDate = [int(float(l)) for l in lines[10].split(' ') if l != ''] ModelDt = float(lines[11]) ModelUTCOffset = float(lines[12]) inpDailyorSubdaily = bool(lines[13]==True) self.outtStride, outyStep, outxStep = [int(s) for s in lines[14].split(' ')] C_wsxcorArray = c_float() C_wsycorArray = c_float() self.C_wsArray = pointer(pointer(c_int32())) self.C_dimlen1 = c_int() self.C_dimlen2 = c_int() totalgrid = 0 self.C_wsfillVal = c_int(-9999) npar = c_int(32) tinitTime = c_int(0) self.C_parvalArray = pointer(c_float(0)); numOut = 70 # hack: number of outputs? self.C_pOut = pointer(pointOutput()) C_aggOut = pointer(aggOutput()) self.C_ncOut = pointer(ncOutput()) self.C_npout = c_int(0) self.C_nncout = c_int(0) C_naggout = c_int(0) C_nZones = c_int(0) C_tNameout = c_char_p("time") tunits = (c_char*256)() C_tUnitsout = pointer(tunits) C_tlong_name = c_char_p("time") C_tcalendar = c_char_p("standard") self.t_out = pointer(c_float(0)) C_out_fillVal = c_float(-9999.0) self.C_outDimord = c_int(0) C_aggoutDimord = c_int(1) self.outvarindx = c_int(17) # aggoutvarindx = c_int(17) # size = c_int() # rank = c_int() # irank = c_int() # jrank = c_int() # startTimeT = c_double(0.0) # TotalTime = c_double(0.0) # totalmodelrunTime = c_double(0.0) # TsReadTime = c_double(0.0) # TSStartTime = c_double() # ComputeStartTime = c_double() # ComputeTime = c_double(0.0) # OutWriteTime = c_double() self.uebVars = (c_char_p * 70)("Year", "Month", "Day", "dHour", "atff", "HRI", "Eacl", "Ema", "conZen", "Ta", "P", "V", "RH", "Qsi", "Qli", "Qnet","Us", "SWE", "tausn", "Pr", "Ps", "Alb", "QHs", "QEs", "Es", "SWIT", "QMs", "Q", "FM", "Tave", "TSURFs", "cump", "cumes", "cumMr", "Qnet", "smelt", "refDepth", "totalRefDepth", "cf", "Taufb", "Taufd", "Qsib", "Qsid", "Taub", "Taud", "Qsns", "Qsnc", "Qlns", "Qlnc", "Vz", "Rkinsc", "Rkinc", "Inmax", "intc", "ieff", "Ur", "Wc", "Tc", "Tac", "QHc", "QEc", "Ec", "Qpc", "Qmc", "Mc", "FMc", "SWIGM", "SWISM", "SWIR", "errMB") C_zName = c_char_p("Outletlocations") C_tcorvar = pointer((c_float * 13)()) self.C_tsvarArray = pointer((c_float * 13)()) C_tsvarArrayTemp = pointer((c_float * 5)()) #todo: [#] == pointer, * == pointer self.C_tsvarArray = pointer((POINTER(c_float)*13)()) C_ntimesteps = pointer((c_int * 5)()) # create pointer to instance of sitevar struct array self.C_strsvArray = pointer((sitevar * 32)()) # create pointer to instance of inpforcvar struct array self.C_strinpforcArray = pointer((inpforcvar * 13)()) # mask = c_float() # pcap_lookupnet(dev, ctypes.byref(net), ctypes.byref(mask), errbuf) # read watershed netcdf file self.__uebLib.readwsncFile(C_watershedFile, C_wsvarName, C_wsycorName, C_wsxcorName, byref(C_wsycorArray), byref(C_wsxcorArray), byref(self.C_wsArray), byref(self.C_dimlen1), byref(self.C_dimlen2), byref(self.C_wsfillVal)) wsArray1D = numpy.empty((self.C_dimlen1.value*self.C_dimlen2.value),dtype=numpy.float) for i in xrange(self.C_dimlen1.value) : for j in xrange(self.C_dimlen2.value): wsArray1D[i*self.C_dimlen2.value + j] = self.C_wsArray[i][j]; # zvalues is the unique set of wsArray1D zValues = list(set(wsArray1D)) # fillset = [wsfillVal] # zVal is the set of zValues that do not equal wsFillVal zVal = [zValues[i] for i in xrange(len(zValues)) if zValues[i] != self.C_wsfillVal] C_nZones = len(zVal) z_ycor = [0.0 for i in xrange(C_nZones)] z_xcor = [0.0 for i in xrange(C_nZones)] # read params (#194) self.__uebLib.readParams(C_paramFile, byref(self.C_parvalArray), npar) # read site variables (#200) self.__uebLib.readSiteVars(C_sitevarFile, byref(self.C_strsvArray)) # read 2d NetCDF Data for i in range(0,32): a = self.C_strsvArray.contents[i] if a.svType == 1: # print "%d %s %s\n" % (i, a.svFile,a.svVarName) retvalue = self.__uebLib.read2DNC(os.path.join(self.base_dir, a.svFile), a.svVarName, byref(a.svArrayValues)) #//read input /forcing control file--all possible entries of input control have to be provided #readInputForcVars(inputconFile, strinpforcArray); # read input force variables (main.cpp, line 219) self.__uebLib.readInputForcVars(cast(C_inputconFile,c_char_p), self.C_strinpforcArray) # elog.info('UEB Start Date: %s' % sd.strftime("%m-%d-%Y %H:%M:%S")) # elog.info('UEB End Date: %s' % ed.strftime("%m-%d-%Y %H:%M:%S")) # calculate model time span as a julian date (main.cpp, line 220) modelSpan = jdutil.datetime_to_jd(datetime.datetime(*ModelEndDate)) - \ jdutil.datetime_to_jd(datetime.datetime(*ModelStartDate)) # setup the model start dates as UEB expects them ModelStartHour = ModelStartDate.pop(-1) # an integer representing the start hour (24 hour time) ModelEndHour = ModelEndDate.pop(-1) # an integer representing the end hour (24 hour time) # ModelStartDate is a 3 element array: [year, month, day] # ModelEndDate is a 3 element array: [year, month, day] # convert Simulation Time parameters into ctypes self.C_ModelStartDate = (c_int * len(ModelStartDate))(*ModelStartDate) self.C_ModelEndDate =(c_int * len(ModelEndDate))(*ModelEndDate) self.C_ModelDt = c_double(ModelDt) self.C_ModelUTCOffset = c_double(ModelUTCOffset) self.C_ModelStartHour = c_double(ModelStartHour) self.C_ModelEndHour = c_double(ModelEndHour) # calculate model time steps (main.cpp, line 222) self.numTimeStep = int(math.ceil(modelSpan*(24./ModelDt)) ) + 1 # initialize C_tsvarArray values (this replaces __uebLib.readTextData) self.initialize_timeseries_variable_array(self.C_strinpforcArray, self.numTimeStep) # NOTE: C_strinpforcArray stores info about the forcing data files # # read forcing data (main.cpp, line 226) # if self.C_strsvArray.contents[16].svType != 3: # no accumulation zone (fixme: ???) # for it in xrange(13): # inftype = self.C_strinpforcArray.contents[it].infType # print 'infFile: ',self.C_strinpforcArray.contents[it].infFile # if inftype == 0: # # # read the files stored in C_strinpforcArray and populated C_tsvarArray # self.__uebLib.readTextData(os.path.join(self.base_dir, self.C_strinpforcArray.contents[it].infFile), byref(self.C_tsvarArray.contents[it]), byref(C_ntimesteps[0])) # # elif inftype == 2 or inftype == -1: # self.C_tsvarArray.contents[it] = (c_float * 2)() # C_ntimesteps.contents[0] = 2 # # copy the default value if a single value is the option # self.C_tsvarArray.contents[it][0] = self.C_strinpforcArray.contents[it].infType # self.C_tsvarArray.contents[it][1] = self.C_strinpforcArray.contents[it].infdefValue # :: this array is initialized to (numOut+1, numTimeStep+1) rather than (numOut, numTimeStep) # :: b/c otherwise the calculations from RunUEB are incorrect for the first row. # :: e.g. # :: 2009 2010 5 30.000 23.999979 # :: rather than: # :: 2009 10 1 0.000 0.569902 # :: # :: I thought this was b/c numpy.float32 (and 64) are smaller than c_float, however this change # :: below didn't fix the problem. # :: # create a numpy array for outputs self.outvarArray = numpy.zeros(shape=(numOut+1, self.numTimeStep), dtype=numpy.float, order="C") # arrays_old = self.outvarArray.astype(numpy.float32) arrays = self.outvarArray.astype(c_float) rows, cols = self.outvarArray.shape arrays_as_list = list(arrays) #get ctypes handles ctypes_arrays = [numpy.ctypeslib.as_ctypes(array) for array in arrays_as_list] #Pack into pointer array self.C_outvarArray = (POINTER(c_float) * rows)(*ctypes_arrays) # x = numpy.zeros(shape=(numOut, self.numTimeStep), dtype=numpy.float, order="C") # _floatpp = numpy.ctypeslib.ndpointer(dtype=numpy.uintp, ndim=1, flags='C') # xpp = (x.__array_interface__['data'][0] + numpy.arange(x.shape[0])*x.strides[0]).astype(numpy.uintp) # self.C_outvarArray = pointer(((POINTER(c_float) * self.numTimeStep) * numOut)()) # a = (c_float * self.numTimeStep)() # outvarArray = pointer((a * numOut)()) # for i in xrange(numOut): # outvarArray[i] = pointer((c_float * self.numTimeStep)()) # total grid size to compute progess totalgrid = self.C_dimlen1.value*self.C_dimlen2.value # read output control file (main.cpp, line 251) # readOutputControl(outputconFile, aggoutputconFile, pOut, ncOut, aggOut, npout, nncout, naggout); self.__uebLib.readOutputControl(cast(C_outputconFile,c_char_p), cast(C_aggoutputconFile, c_char_p), byref(self.C_pOut), byref(self.C_ncOut), byref(C_aggOut), byref(self.C_npout), byref(self.C_nncout), byref(C_naggout)) # create output netcdf self.C_outtSteps = self.numTimeStep / self.outtStride self.t_out = numpy.empty(shape=(self.C_outtSteps), dtype=numpy.float, order="C") for i in xrange(self.C_outtSteps): self.t_out[i] = i*self.outtStride*ModelDt # initialize the output arrays aggoutvarArray = numpy.zeros((C_nZones,C_naggout.value, self.C_outtSteps), dtype=numpy.float) totalAgg = numpy.empty((self.C_outtSteps,), dtype=numpy.float) ZonesArr = numpy.zeros((C_nZones,), dtype=numpy.int32) # main.cpp, line 290 # CREATE 3D NC OUTPUT FILES # convert self.t_out into a float pointer C_t_out = self.t_out.ctypes.data_as(POINTER(c_float)) for i in xrange(self.C_nncout.value): ''' for (int icout = 0; icout < nncout; icout++) retvalue = create3DNC_uebOutputs(ncOut[icout].outfName, (const char*)ncOut[icout].symbol, (const char*)ncOut[icout].units, tNameout, tUnitsout, tlong_name, tcalendar, outtSteps, outDimord, self.t_out, &out_fillVal, watershedFile, wsvarName, wsycorName, wsxcorName); ''' retvalue = self.__uebLib.create3DNC_uebOutputs(self.C_ncOut[i].outfName, cast(self.C_ncOut[i].symbol, c_char_p), cast(self.C_ncOut[i].units, c_char_p), C_tNameout, C_tUnitsout, C_tlong_name, C_tcalendar, self.C_outtSteps, self.C_outDimord, C_t_out, byref(C_out_fillVal), C_watershedFile, C_wsvarName, C_wsycorName, C_wsxcorName); # CREATE 3D NC AGGREGATE OUTPUT FILE # convert z_ycor and x_xcor from list into ctype C_z_xcor = numpy.asarray(z_xcor).ctypes.data_as(POINTER(c_float)) C_z_ycor = numpy.asarray(z_ycor).ctypes.data_as(POINTER(c_float)) retvalue = self.__uebLib.create3DNC_uebAggregatedOutputs(C_aggoutputFile, C_aggOut, C_naggout, C_tNameout, C_tUnitsout, C_tlong_name, C_tcalendar, self.C_outtSteps, C_aggoutDimord, C_t_out, byref(C_out_fillVal), C_watershedFile, C_wsvarName, C_wsycorName, C_wsxcorName, C_nZones, C_zName, C_z_ycor, C_z_xcor); # todo: create output element set # print 'Output Calculations available at: ' # for pid in xrange(self.C_npout.value): # print " Point(",self.C_pOut[pid].xcoord,", ",self.C_pOut[pid].ycoord,') ' # todo: This is where UEB grid points are defined!, expose these as input/output spatial objects # main.cpp, line 303 self.activeCells = [] # print 'Calculations will be performed at: ' for iy in xrange(self.C_dimlen1.value): for jx in xrange(self.C_dimlen2.value): if self.C_wsArray[iy][jx] != self.C_wsfillVal.value and self.C_strsvArray.contents[16].svType != 3: # print " Point(",jx,", ",iy,') ' self.activeCells.append((iy, jx)) # build output exchange items xcoords = [] ycoords = [] for pid in xrange(self.C_npout.value): xcoords.append(self.C_pOut[pid].xcoord) ycoords.append(self.C_pOut[pid].ycoord) self.pts = geometry.build_point_geometries(xcoords, ycoords) self.__swe = self.outputs()['Snow Water Equivalent'] self.__swit = self.outputs()['Surface Water Input Total'] self.__swe.addGeometries2(self.pts) self.__swit.addGeometries2(self.pts) # build input exchange items ds = nc.Dataset(C_watershedFile) Xlist = ds.variables['x'] Ylist = ds.variables['y'] self.geoms = self.build_geometries(Xlist, Ylist) self.inputs()['Precipitation'].addGeometries2(self.geoms) self.inputs()['Temperature'].addGeometries2(self.geoms) # set start, end, and timestep parameters ts = datetime.timedelta(hours=ModelDt) self.time_step(ts.total_seconds()) sd = datetime.datetime(*ModelStartDate) ed = datetime.datetime(*ModelEndDate) self.simulation_start(sd) self.simulation_end(ed)
def __init__(self, args): super(Wrapper, self).__init__() handle = netCDF4.Dataset(args["ncpath"], "r") variables = handle.variables.keys() tdim = args["tdim"] xdim = args["xdim"] ydim = args["ydim"] tunit = {args["tunit"]: 1} if isinstance(args["starttime"], datetime.datetime): st = args["starttime"] else: st = parser.parse(args["starttime"]) # make sure the variables provided exist in the nc file assert tdim in variables, "time variable name not specified. Cannot continue" assert xdim in variables, "x dimension variable name not specified. Cannot continue" assert ydim in variables, "y dimension variable name not specified. Cannot continue" # get data for these variables timesteps = handle.variables[tdim][:] times = [] for ts in timesteps: # update the time unit value tunit[args["tunit"]] = ts # unpack the tunit dictionary to create a timedelta object dt = datetime.timedelta(**tunit) times.append(st + dt) variables.remove(tdim) x = handle.variables[xdim][:] variables.remove(xdim) y = handle.variables[ydim][:] variables.remove(ydim) # create flattened lists of x,y coords from meshgrid xcoords, ycoords = numpy.meshgrid(x, y) xcoords = xcoords.flatten() ycoords = ycoords.flatten() # loop through the remaining variables and expose them as outputs # for var in variables: for v in range(len(variables)): var = variables[v] # create a unit unit = stdlib.Unit() unit.UnitName(handle.variables[var].units if "units" in dir(handle.variables[var]) else "N/A") unit.UnitTypeCV("N/A") unit.UnitAbbreviation("N/A") # create a variable variable = stdlib.Variable() variable.VariableNameCV(handle.variables[var].name) variable.VariableDefinition("N/A") # create geometries geoms = geometry.build_point_geometries(xcoords, ycoords) # This takes way to long # create exchange item oei = stdlib.ExchangeItem( name=variable.VariableNameCV(), desc="Autogenerated variable parsed from %s" % args["ncpath"], geometry=geoms, unit=unit, variable=variable, type=stdlib.ExchangeItemType.OUTPUT, ) # flatten each timestep of the data values = [v.flatten() for v in handle.variables[var][:]] # set these data. Skip occurrences with mismatched values and times arrays if len(values) == len(times): success = oei.setValues2(values, times) # only expose the exchange item if data was set properly if success: # save the oei self.outputs(oei) # set metadata name = args["ncpath"].split("/")[-1] self.name(name) self.description("NetCDF data component, " + name) self.simulation_start(times[0]) self.simulation_end(times[-1]) self.status(stdlib.Status.READY)
def calc_ti_geometries(self): sPrint('Reading Topographic Indices') with open(self.topo_input, 'r') as sr: lines = sr.readlines() ncols = int(lines[0].split(' ')[-1].strip()) nrows = int(lines[1].split(' ')[-1].strip()) lowerx = float(lines[2].split(' ')[-1].strip()) lowery = float(lines[3].split(' ')[-1].strip()) cellsize = float(lines[4].split(' ')[-1].strip()) nodata = float(lines[5].split(' ')[-1].strip()) # read ti data data = np.genfromtxt(self.topo_input, delimiter=' ', skip_header=6) # build X and Y coordinate arrays xi = np.linspace(lowerx, lowerx+ncols*cellsize, ncols) yi = np.linspace(lowery+nrows*cellsize, lowery, nrows) x,y = np.meshgrid(xi,yi) # generate 2d arrays from xi, yi x = x.ravel() # convert to 1-d y = y.ravel() # convert to 1-d data = data.ravel() # convert to 1-d # remove all nodata points from x, y arrays nonzero = np.where(data != nodata) x = x[nonzero] y = y[nonzero] with open(self.fac_input, 'r') as fac: lines = fac.readlines() fac_ncols = int(lines[0].split(' ')[-1].strip()) fac_nrows = int(lines[1].split(' ')[-1].strip()) fac_lowerx = float(lines[2].split(' ')[-1].strip()) fac_lowery = float(lines[3].split(' ')[-1].strip()) fac_cellsize = float(lines[4].split(' ')[-1].strip()) fac_nodata = float(lines[5].split(' ')[-1].strip()) # read ti data fac_data = np.genfromtxt(self.fac_input, delimiter=' ', skip_header=6) # build X and Y coordinate arrays xi = np.linspace(fac_lowerx, fac_lowerx + fac_ncols * fac_cellsize, fac_ncols) yi = np.linspace(fac_lowery + fac_nrows * fac_cellsize, fac_lowery, fac_nrows) x,y = np.meshgrid(xi,yi) # generate 2d arrays from xi, yi x = x.ravel() # convert to 1-d y = y.ravel() # convert to 1-d fac_data = fac_data.ravel() # convert to 1-d # isolate the rivers based on threshold threshold = np.max(fac_data) # todo, user input rivers = np.where((fac_data != fac_nodata) & (fac_data >= threshold) ) # select only the x and y values that are considered rivers x = x[rivers] y = y[rivers] self.rivers_idxs = rivers sPrint('Building Geometry Objects') tigeoms = geometry.build_point_geometries(x,y) self.ti_geoms = tigeoms