def test_spatial_nearest_neighbor_radial(self):
# build some test points
x_in = [0, 4, 2, 0, 4]
y_in = [0, 0, 2, 4, 4]
x_out = [0, 2, 4]
y_out = [1, 0, 3]
pts_in = geometry.build_point_geometries(x_in, y_in)
pts_out = geometry.build_point_geometries(x_out, y_out)
# instantiate the interpolation class
interpolation = SpatialInterpolation.NearestNeighborRadial
# test multiple matches (large search distance)
interpolation.set_param("max_distance", 10)
mapped = interpolation.transform(pts_in, pts_out)
self.assertTrue('None' not in mapped)
self.assertTrue(len(mapped) == 3)
# test no matches (very small search distance)
interpolation.set_param("max_distance", 0)
mapped = interpolation.transform(pts_in, pts_out)
self.assertTrue('None' not in mapped)
self.assertTrue(len(mapped) == 0)
# test partial matches (small search distance)
interpolation.set_param("max_distance", 1)
mapped = interpolation.transform(pts_in, pts_out)
self.assertTrue('None' not in mapped)
self.assertTrue(len(mapped) == 2)
def test_spatial_nearest_neighbor_radial(self):
# build some test points
x_in = [0,4,2,0,4]
y_in = [0,0,2,4,4]
x_out = [0,2,4]
y_out = [1,0,3]
pts_in = geometry.build_point_geometries(x_in, y_in)
pts_out = geometry.build_point_geometries(x_out, y_out)
# instantiate the interpolation class
interpolation = SpatialInterpolation.NearestNeighborRadial
# test multiple matches (large search distance)
interpolation.set_param("max_distance", 10)
mapped = interpolation.transform(pts_in, pts_out)
self.assertTrue('None' not in mapped)
self.assertTrue(len(mapped) == 3)
# test no matches (very small search distance)
interpolation.set_param("max_distance", 0)
mapped = interpolation.transform(pts_in, pts_out)
self.assertTrue('None' not in mapped)
self.assertTrue(len(mapped) == 0)
# test partial matches (small search distance)
interpolation.set_param("max_distance", 1)
mapped = interpolation.transform(pts_in, pts_out)
self.assertTrue('None' not in mapped)
self.assertTrue(len(mapped) == 2)
def build_nodes(inp):
geoms = {}
lines = None
with open(inp, "r") as f:
lines = f.readlines()
# first read all the node coordinates
nodes = {}
node_order = []
cidx = find(lines, lambda x: "COORDINATES" in x)
for line in lines[cidx + 3 :]:
if line.strip() == "":
break
vals = re.split(" +", line.strip())
if vals[0] in nodes:
nodes[vals[0]].append((float(vals[1]), float(vals[2])))
else:
nodes[vals[0]] = [(float(vals[1]), float(vals[2]))]
node_order.append(vals[0])
for name, coords in nodes.iteritems():
ptx = [x for x, y in coords]
pty = [y for x, y in coords]
g = geom_utilites.build_point_geometries(ptx, pty)
geoms[name] = {"geometry": g}
return geoms
def setup_model(self, geomcount, valuecount):
# create an exchange item
unit = mdl.create_unit('cubic meters per second')
variable = mdl.create_variable('streamflow')
# create exchange item
item = stdlib.ExchangeItem(name='Test', desc='Test Exchange Item',
unit=unit, variable=variable)
# set exchange item geometries
xcoords = [i for i in range(geomcount)]
ycoords = [i*1.5 for i in range(geomcount)]
geoms = geometry.build_point_geometries(xcoords, ycoords)
item.addGeometries2(geoms)
# set exchange item values
start_time = dt.now()
end_time = start_time+timedelta(minutes=valuecount - 1)
time_step = 60 # in seconds
item.initializeDatesValues(start_datetime=start_time, end_datetime=end_time, timestep_in_seconds=time_step)
values = numpy.random.rand(valuecount, geomcount)
# for i in range(len(dates)):
item.setValuesBySlice(values) #, time_index_slice=(0,num_timesteps,1))
return item, geoms
def calc_ti_geometries(self):
elog.info("TOPMODEL: Building Geometry Objects")
tigeoms = []
satgeoms = []
with open(self.topo_input, "r") as sr:
lines = sr.readlines()
ncols = int(lines[0].split(" ")[-1].strip())
nrows = int(lines[1].split(" ")[-1].strip())
lowerx = float(lines[2].split(" ")[-1].strip())
lowery = float(lines[3].split(" ")[-1].strip())
cellsize = float(lines[4].split(" ")[-1].strip())
nodata = float(lines[5].split(" ")[-1].strip())
# read ti data
data = np.genfromtxt(self.topo_input, delimiter=" ", skip_header=6)
# build X and Y coordinate arrays
xi = np.linspace(lowerx, lowerx + ncols * cellsize, ncols)
yi = np.linspace(lowery + nrows * cellsize, lowery, nrows)
x, y = np.meshgrid(xi, yi) # generate 2d arrays from xi, yi
x = x.ravel() # convert to 1-d
y = y.ravel() # convert to 1-d
data = data.ravel() # convert to 1-d
# remove all nodata points from x, y arrays
nonzero = np.where(data != nodata)
x = x[nonzero]
y = y[nonzero]
tigeoms = geometry.build_point_geometries(x, y)
self.ti_geoms = tigeoms
def test_create_exchange_item(self):
item = ExchangeItem('e1', 'Test', 'Test Exchange Item')
# -- Create Unit --#
unit = mdl.create_unit('cubic meters per second')
# -- Create Variable --#
variable = mdl.create_variable('streamflow')
# set data
dates = [datetime.datetime(2014, 1, 1, 12, 0, 0) + datetime.timedelta(days=i) for i in range(0, 100)]
vals = [random.random() for i in range(0,100)]
x = [random.random() for i in range(0,150)]
y = [random.random() for i in range(0,150)]
geoms = geometry.build_point_geometries(x,y)
item.addGeometries2(geoms)
item.setValues2(vals, dates)
self.assertTrue(item.name() == 'Test')
self.assertTrue(item.description() == 'Test Exchange Item')
self.assertTrue(len(item.getDates2()) == 100) # assert the correct number of values
self.assertTrue(item.getEarliestTime2() == datetime.datetime(2014, 1, 1, 12, 0, 0)) # asser the correct start time
self.assertTrue(item.getLatestTime2() == datetime.datetime(2014, 4, 10, 12, 0, 0)) # assert the correct end time
self.assertTrue(len(item.getValues2()) == 100) # assert the correct number of values
self.assertTrue([] not in item.getValues2()) # assert no empty values
default_srs = osr.SpatialReference()
default_srs.ImportFromEPSG(4269)
self.assertTrue(item.srs().ExportToWkt() == default_srs.ExportToWkt())
def calc_ti_geometries(self):
elog.info('TOPMODEL: Building Geometry Objects')
tigeoms = []
satgeoms = []
with open(self.topo_input, 'r') as sr:
lines = sr.readlines()
ncols = int(lines[0].split(' ')[-1].strip())
nrows = int(lines[1].split(' ')[-1].strip())
lowerx = float(lines[2].split(' ')[-1].strip())
lowery = float(lines[3].split(' ')[-1].strip())
cellsize = float(lines[4].split(' ')[-1].strip())
nodata = float(lines[5].split(' ')[-1].strip())
# read ti data
data = np.genfromtxt(self.topo_input, delimiter=' ', skip_header=6)
# build X and Y coordinate arrays
xi = np.linspace(lowerx, lowerx + ncols * cellsize, ncols)
yi = np.linspace(lowery + nrows * cellsize, lowery, nrows)
x, y = np.meshgrid(xi, yi) # generate 2d arrays from xi, yi
x = x.ravel() # convert to 1-d
y = y.ravel() # convert to 1-d
data = data.ravel() # convert to 1-d
# remove all nodata points from x, y arrays
nonzero = np.where(data != nodata)
x = x[nonzero]
y = y[nonzero]
tigeoms = geometry.build_point_geometries(x, y)
self.ti_geoms = tigeoms
def build_nodes(inp):
geoms = {}
lines = None
with open(inp,'r') as f:
lines = f.readlines()
# first read all the node coordinates
nodes = {}
node_order = []
cidx = find(lines, lambda x: 'COORDINATES' in x)
for line in lines[cidx+3:]:
if line.strip() == '':
break
vals = re.split(' +',line.strip())
if vals[0] in nodes:
nodes[vals[0]].append((float(vals[1]), float(vals[2])))
else:
nodes[vals[0]] = [(float(vals[1]), float(vals[2]))]
node_order.append(vals[0])
for name,coords in nodes.iteritems():
ptx = [x for x,y in coords]
pty = [y for x,y in coords]
g = geom_utilites.build_point_geometries(ptx, pty)
geoms[name] = {'geometry':g}
return geoms
def insert_simulation(db, user, geomcount, valuecount):
print '%d geometries, %d values per geometry' % (geomcount, valuecount)
print 'Total number of data values: %d' % (valuecount * geomcount)
# create an exchange item
unit = mdl.create_unit('cubic meters per second')
variable = mdl.create_variable('streamflow')
# create exchange item
item = stdlib.ExchangeItem(name='Test', desc='Test Exchange Item', unit=unit, variable=variable)
# set exchange item geometries
xcoords = numpy.random.rand(geomcount)
ycoords = numpy.random.rand(geomcount)
points = geometry.build_point_geometries(xcoords, ycoords)
item.addGeometries2(points)
# set exchange item values
start_time = dt.now() # set start time to 'now'
end_time = start_time + timedelta(days=valuecount-1) # create endtime base on the number of values
current_time = start_time # initial time
dates = [] # list to hold dates
values = [] # list to hold values for each date
# populate dates list
while current_time <= end_time:
# add date
dates.append(current_time)
# add some random values for each geometry
values.append([random.random() for pt in points] )
# increment time by 1 day
current_time += timedelta(days=1)
# set dates and values in the exchange item
item.setValues2(values, dates)
# create the simulation
st = dt(2014, 3, 1, 12, 0, 0)
et = dt(2014, 3, 1, 23, 0, 0)
description = 'Some model descipription'
name = 'test simulation'
# turn off verbosity for simulation insert
os.environ['LOGGING_SHOWINFO'] = '0'
os.environ['LOGGING_SHOWDEBUG'] = '0'
db.create_simulation('My Simulation', user[0], None, item, st, et, 1, 'days', description, name)
os.environ['LOGGING_SHOWINFO'] = '1'
# query simulations
simulations = db.read.getAllSimulations()
simulation = db.read.getSimulationByName('My Simulation')
assert simulation is not None
def test_add_geoms2(self):
# add a single geom
pt = geometry.build_point_geometry(10, 11)
x, y, z = pt.GetPoint()
self.assertTrue(isinstance(pt, Geometry2))
ret = self.item.addGeometry(pt)
self.assertTrue(ret)
g = self.item.getGeometries2(-1)
self.assertTrue(isinstance(g, Geometry2))
x, y, z = g.GetPoint()
self.assertTrue(x == 10)
self.assertTrue(y == 11)
self.assertTrue(z == 0)
# add an invalid geometry
ret = self.item.addGeometry((0, 1, 1))
self.assertFalse(ret)
# add many of geometries
xs = range(100, 110)
ys = range(110, 120)
pts = geometry.build_point_geometries(xs, ys)
geoms = []
for pt in pts:
geoms.append(pt)
ret = self.item.addGeometries2(geoms)
self.assertTrue(ret)
g = self.item.getGeometries2(-1)
self.assertTrue(isinstance(g, Geometry2))
x, y, z = g.GetPoint()
self.assertTrue(x == 109)
self.assertTrue(y == 119)
self.assertTrue(z == 0)
# add a mix of valid and invalid geometries
geoms = []
for i in range(100, 110):
if i % 2 == 0:
pt = geometry.build_point_geometry(i, i + 1)
geoms.append(pt)
else:
geoms.append((i, i + 1))
ret = self.item.addGeometries2(geoms)
self.assertFalse(ret)
g = self.item.getGeometries2(-1)
self.assertTrue(isinstance(g, Geometry2))
x, y, z = g.GetPoint()
self.assertTrue(x == 108)
self.assertTrue(y == 109)
self.assertTrue(z == 0)
def test_add_geoms2(self):
# add a single geom
pt = geometry.build_point_geometry(10, 11)
x,y,z = pt.GetPoint()
self.assertTrue(isinstance(pt, Geometry2))
ret = self.item.addGeometry(pt)
self.assertTrue(ret)
g = self.item.getGeometries2(-1)
self.assertTrue(isinstance(g, Geometry2))
x,y,z = g.GetPoint()
self.assertTrue(x == 10)
self.assertTrue(y == 11)
self.assertTrue(z == 0)
# add an invalid geometry
ret = self.item.addGeometry((0,1,1))
self.assertFalse(ret)
# add many of geometries
xs = range(100,110)
ys = range(110,120)
pts = geometry.build_point_geometries(xs, ys)
geoms = []
for pt in pts:
geoms.append(pt)
ret = self.item.addGeometries2(geoms)
self.assertTrue(ret)
g = self.item.getGeometries2(-1)
self.assertTrue(isinstance(g, Geometry2))
x,y,z = g.GetPoint()
self.assertTrue(x == 109)
self.assertTrue(y == 119)
self.assertTrue(z == 0)
# add a mix of valid and invalid geometries
geoms = []
for i in range(100,110):
if i % 2 == 0:
pt = geometry.build_point_geometry(i, i + 1)
geoms.append(pt)
else:
geoms.append((i,i+1))
ret = self.item.addGeometries2(geoms)
self.assertFalse(ret)
g = self.item.getGeometries2(-1)
self.assertTrue(isinstance(g, Geometry2))
x,y,z = g.GetPoint()
self.assertTrue(x == 108)
self.assertTrue(y == 109)
self.assertTrue(z == 0)
def setUp(self):
# -- Create Unit --#
unit = mdl.create_unit('cubic meters per second')
# -- Create Variable --#
variable = mdl.create_variable('streamflow')
# create exchange item
self.item = ExchangeItem(name='Test', desc='Test Exchange Item', unit=unit, variable=variable)
coords = [(1,2),(2,3),(3,4),(4,5),(5,6),(6,7),(7,8),(8,9),(9,10)]
x,y = zip(*coords)
geoms = geometry.build_point_geometries(x, y)
self.item.addGeometries2(geoms)
def build_geometries(self, xcoords, ycoords):
"""
builds point geometries consistent with the native UEB ordering
:param xcoords: list of x coordinates
:param xyoords: list of xycoordinates
:return: a list of geometries
"""
# build a meshgrid
x, y = numpy.meshgrid(xcoords, ycoords)
# x any y coords are paired using Fortran ordering to be consistent with the way activeCells are ordered. This
# is necessary to ensure that calculation looping is maintained in the run function.
x = numpy.ravel(x, 'F')
y = numpy.ravel(y, 'F')
# build point geometries
return geometry.build_point_geometries(x, y)
def build_geometries(self, xcoords, ycoords):
"""
builds point geometries consistent with the native UEB ordering
:param xcoords: list of x coordinates
:param xyoords: list of xycoordinates
:return: a list of geometries
"""
# build a meshgrid
x, y = numpy.meshgrid(xcoords, ycoords)
# x any y coords are paired using Fortran ordering to be consistent with the way activeCells are ordered. This
# is necessary to ensure that calculation looping is maintained in the run function.
x = numpy.ravel(x, 'F')
y = numpy.ravel(y, 'F')
# build point geometries
return geometry.build_point_geometries(x, y)
def testSetValuesByTime(self):
item = ExchangeItem('e1', 'Test', 'Test Exchange Item')
# set start and end dates
st = datetime.datetime(2014, 1, 1, 12, 0, 0)
et = datetime.datetime(2014, 2, 1, 12, 0, 0)
timestep_in_seconds = 3600
# build geometries
x = [random.random() for i in range(0, 150)]
y = [random.random() for i in range(0, 150)]
geoms = geometry.build_point_geometries(x, y)
item.addGeometries2(geoms)
values = range(30)
# set values for geometry=0, beginning at the start time
success = item.setValuesBySlice(values,
time_index_slice=(0, 30, 1),
geometry_index_slice=(0, 1, 1))
self.assertFalse(success)
# initialize the dates and values containers
item.initializeDatesValues(st, et, timestep_in_seconds)
success = item.setValuesBySlice(values,
time_index_slice=(0, 30, 1),
geometry_index_slice=(0, 1, 1))
self.assertTrue(success)
# get the data
geoms = item.getGeometries2()
times = item.getDates2()
vals = item.getValues2()
# assert that data is set properly
self.assertTrue(len(geoms) == 150)
self.assertTrue(times[0][1] == st)
self.assertTrue(times[-1][1] == et)
self.assertTrue(
(times[-1][1] -
times[-2][1]).total_seconds() == float(timestep_in_seconds))
self.assertTrue(vals.shape[0] == len(times))
self.assertTrue(vals.shape[1] == len(geoms))
def setUp(self):
# -- Create Unit --#
unit = mdl.create_unit('cubic meters per second')
# -- Create Variable --#
variable = mdl.create_variable('streamflow')
# create exchange item
self.item = ExchangeItem(name='Test',
desc='Test Exchange Item',
unit=unit,
variable=variable)
coords = [(1, 2), (2, 3), (3, 4), (4, 5), (5, 6), (6, 7), (7, 8),
(8, 9), (9, 10)]
x, y = zip(*coords)
geoms = geometry.build_point_geometries(x, y)
self.item.addGeometries2(geoms)
def setUp(self):
self.item = ExchangeItem('e1', 'Test', 'Test Exchange Item')
# set start and end dates
self.st = datetime.datetime(2014, 1, 1, 12, 0, 0)
self.et = datetime.datetime(2014, 2, 1, 12, 0, 0)
self.timestep_in_seconds = 3600
# build geometries
x = [random.random() for i in range(0,150)]
y = [random.random() for i in range(0,150)]
geoms = geometry.build_point_geometries(x,y)
self.item.addGeometries2(geoms)
values = range(30)
# initialize the dates and values containers
self.item.initializeDatesValues(self.st, self.et, self.timestep_in_seconds)
success = self.item.setValuesBySlice(values, time_index_slice=(0,30,1), geometry_index_slice=(0, 1, 1))
self.assertTrue(success)
def testSetValuesByTime(self):
item = ExchangeItem('e1', 'Test', 'Test Exchange Item')
# set start and end dates
st = datetime.datetime(2014, 1, 1, 12, 0, 0)
et = datetime.datetime(2014, 2, 1, 12, 0, 0)
timestep_in_seconds = 3600
# build geometries
x = [random.random() for i in range(0,150)]
y = [random.random() for i in range(0,150)]
geoms = geometry.build_point_geometries(x,y)
item.addGeometries2(geoms)
values = range(30)
# set values for geometry=0, beginning at the start time
success = item.setValuesBySlice(values, time_index_slice=(0,30,1), geometry_index_slice=(0, 1, 1))
self.assertFalse(success)
# initialize the dates and values containers
item.initializeDatesValues(st, et, timestep_in_seconds)
success = item.setValuesBySlice(values, time_index_slice=(0,30,1), geometry_index_slice=(0, 1, 1))
self.assertTrue(success)
# get the data
geoms = item.getGeometries2()
times = item.getDates2()
vals = item.getValues2()
# assert that data is set properly
self.assertTrue(len(geoms) == 150)
self.assertTrue(times[0][1] == st)
self.assertTrue(times[-1][1] == et)
self.assertTrue((times[-1][1] - times[-2][1]).total_seconds() == float(timestep_in_seconds))
self.assertTrue(vals.shape[0] == len(times))
self.assertTrue(vals.shape[1] == len(geoms))
def setUp(self):
self.item = ExchangeItem('e1', 'Test', 'Test Exchange Item')
# set start and end dates
self.st = datetime.datetime(2014, 1, 1, 12, 0, 0)
self.et = datetime.datetime(2014, 2, 1, 12, 0, 0)
self.timestep_in_seconds = 3600
# build geometries
x = [random.random() for i in range(0, 150)]
y = [random.random() for i in range(0, 150)]
geoms = geometry.build_point_geometries(x, y)
self.item.addGeometries2(geoms)
values = range(30)
# initialize the dates and values containers
self.item.initializeDatesValues(self.st, self.et,
self.timestep_in_seconds)
success = self.item.setValuesBySlice(values,
time_index_slice=(0, 30, 1),
geometry_index_slice=(0, 1, 1))
self.assertTrue(success)
def calc_ti_geometries(self):
sPrint('Reading Topographic Indices')
with open(self.topo_input, 'r') as sr:
lines = sr.readlines()
ncols = int(lines[0].split(' ')[-1].strip())
nrows = int(lines[1].split(' ')[-1].strip())
lowerx = float(lines[2].split(' ')[-1].strip())
lowery = float(lines[3].split(' ')[-1].strip())
cellsize = float(lines[4].split(' ')[-1].strip())
nodata = float(lines[5].split(' ')[-1].strip())
# read ti data
data = np.genfromtxt(self.topo_input, delimiter=' ', skip_header=6)
# build X and Y coordinate arrays
xi = np.linspace(lowerx, lowerx + ncols * cellsize, ncols)
yi = np.linspace(lowery + nrows * cellsize, lowery, nrows)
x, y = np.meshgrid(xi, yi) # generate 2d arrays from xi, yi
x = x.ravel() # convert to 1-d
y = y.ravel() # convert to 1-d
data = data.ravel() # convert to 1-d
# remove all nodata points from x, y arrays
nonzero = np.where(data != nodata)
x = x[nonzero]
y = y[nonzero]
with open(self.fac_input, 'r') as fac:
lines = fac.readlines()
fac_ncols = int(lines[0].split(' ')[-1].strip())
fac_nrows = int(lines[1].split(' ')[-1].strip())
fac_lowerx = float(lines[2].split(' ')[-1].strip())
fac_lowery = float(lines[3].split(' ')[-1].strip())
fac_cellsize = float(lines[4].split(' ')[-1].strip())
fac_nodata = float(lines[5].split(' ')[-1].strip())
# read ti data
fac_data = np.genfromtxt(self.fac_input, delimiter=' ', skip_header=6)
# build X and Y coordinate arrays
xi = np.linspace(fac_lowerx, fac_lowerx + fac_ncols * fac_cellsize,
fac_ncols)
yi = np.linspace(fac_lowery + fac_nrows * fac_cellsize, fac_lowery,
fac_nrows)
x, y = np.meshgrid(xi, yi) # generate 2d arrays from xi, yi
x = x.ravel() # convert to 1-d
y = y.ravel() # convert to 1-d
fac_data = fac_data.ravel() # convert to 1-d
# isolate the rivers based on threshold
threshold = np.max(fac_data) # todo, user input
rivers = np.where((fac_data != fac_nodata) & (fac_data >= threshold))
# select only the x and y values that are considered rivers
x = x[rivers]
y = y[rivers]
self.rivers_idxs = rivers
sPrint('Building Geometry Objects')
tigeoms = geometry.build_point_geometries(x, y)
self.ti_geoms = tigeoms
def __init__(self, args):
super(Wrapper, self).__init__()
handle = netCDF4.Dataset(args['ncpath'], 'r')
variables = handle.variables.keys()
tdim = args['tdim']
xdim = args['xdim']
ydim = args['ydim']
tunit = {args['tunit']: 1}
if isinstance(args['starttime'], datetime.datetime):
st = args['starttime']
else:
st = parser.parse(args['starttime'])
# make sure the variables provided exist in the nc file
assert tdim in variables, 'time variable name not specified. Cannot continue'
assert xdim in variables, 'x dimension variable name not specified. Cannot continue'
assert ydim in variables, 'y dimension variable name not specified. Cannot continue'
# get data for these variables
timesteps = handle.variables[tdim][:]
times = []
for ts in timesteps:
# update the time unit value
tunit[args['tunit']] = ts
# unpack the tunit dictionary to create a timedelta object
dt = datetime.timedelta(**tunit)
times.append(st + dt)
variables.remove(tdim)
x = handle.variables[xdim][:]
variables.remove(xdim)
y = handle.variables[ydim][:]
variables.remove(ydim)
# create flattened lists of x,y coords from meshgrid
xcoords, ycoords = numpy.meshgrid(x, y)
xcoords = xcoords.flatten()
ycoords = ycoords.flatten()
# loop through the remaining variables and expose them as outputs
# for var in variables:
for v in range(len(variables)):
var = variables[v]
# create a unit
unit = stdlib.Unit()
unit.UnitName(handle.variables[var].units if 'units' in
dir(handle.variables[var]) else 'N/A')
unit.UnitTypeCV("N/A")
unit.UnitAbbreviation("N/A")
# create a variable
variable = stdlib.Variable()
variable.VariableNameCV(handle.variables[var].name)
variable.VariableDefinition("N/A")
# create geometries
geoms = geometry.build_point_geometries(
xcoords, ycoords) # This takes way to long
# create exchange item
oei = stdlib.ExchangeItem(
name=variable.VariableNameCV(),
desc='Autogenerated variable parsed from %s' % args['ncpath'],
geometry=geoms,
unit=unit,
variable=variable,
type=stdlib.ExchangeItemType.OUTPUT)
# flatten each timestep of the data
values = [v.flatten() for v in handle.variables[var][:]]
# set these data. Skip occurrences with mismatched values and times arrays
if len(values) == len(times):
success = oei.setValues2(values, times)
# only expose the exchange item if data was set properly
if success:
# save the oei
self.outputs(oei)
# set metadata
name = args['ncpath'].split('/')[-1]
self.name(name)
self.description('NetCDF data component, ' + name)
self.simulation_start(times[0])
self.simulation_end(times[-1])
self.status(stdlib.Status.READY)
def __init__(self, config_params):
super(ueb, self).__init__(config_params)
# build inputs and outputs
io = mdl.build_exchange_items_from_config(config_params)
# set input and output exchange items
self.inputs(value=io[stdlib.ExchangeItemType.INPUT])
self.outputs(value=io[stdlib.ExchangeItemType.OUTPUT])
# grab the C library path and the control file path
lib = config_params['lib']
conFile = config_params['control']
# load the UEB C library
self.__uebLib = cdll.LoadLibrary(join(os.path.dirname(__file__), lib))
# save the current directory for saving output data
self.curdir = os.path.dirname(os.path.abspath(conFile))
# the base directory for the control file is used to convert relative paths into absolute paths
self.base_dir = os.path.dirname(conFile)
# get param, sitevar, input, output, and watershed files
with open(conFile, 'r') as f:
# lines = f.readlines()
lines = f.read().splitlines() # this will auto strip the \n \r
C_paramFile = os.path.join(self.base_dir, lines[1])
C_sitevarFile = os.path.join(self.base_dir, lines[2])
C_inputconFile = os.path.join(self.base_dir, lines[3])
C_outputconFile = os.path.join(self.base_dir, lines[4])
C_watershedFile = os.path.join(self.base_dir, lines[5])
C_wsvarName = lines[6].split(' ')[0]
C_wsycorName = lines[6].split(' ')[1]
C_wsxcorName = lines[6].split(' ')[2]
C_aggoutputconFile = os.path.join(self.base_dir, lines[7])
C_aggoutputFile = os.path.join(self.base_dir, lines[8])
ModelStartDate = [
int(float(l)) for l in lines[9].split(' ') if l != ''
]
ModelEndDate = [
int(float(l)) for l in lines[10].split(' ') if l != ''
]
ModelDt = float(lines[11])
ModelUTCOffset = float(lines[12])
inpDailyorSubdaily = bool(lines[13] == True)
self.outtStride, outyStep, outxStep = [
int(s) for s in lines[14].split(' ')
]
C_wsxcorArray = c_float()
C_wsycorArray = c_float()
self.C_wsArray = pointer(pointer(c_int32()))
self.C_dimlen1 = c_int()
self.C_dimlen2 = c_int()
totalgrid = 0
self.C_wsfillVal = c_int(-9999)
npar = c_int(32)
tinitTime = c_int(0)
self.C_parvalArray = pointer(c_float(0))
numOut = 70 # hack: number of outputs?
self.C_pOut = pointer(pointOutput())
C_aggOut = pointer(aggOutput())
self.C_ncOut = pointer(ncOutput())
self.C_npout = c_int(0)
self.C_nncout = c_int(0)
C_naggout = c_int(0)
C_nZones = c_int(0)
C_tNameout = c_char_p("time")
tunits = (c_char * 256)()
C_tUnitsout = pointer(tunits)
C_tlong_name = c_char_p("time")
C_tcalendar = c_char_p("standard")
self.t_out = pointer(c_float(0))
C_out_fillVal = c_float(-9999.0)
self.C_outDimord = c_int(0)
C_aggoutDimord = c_int(1)
self.outvarindx = c_int(17)
# aggoutvarindx = c_int(17)
# size = c_int()
# rank = c_int()
# irank = c_int()
# jrank = c_int()
# startTimeT = c_double(0.0)
# TotalTime = c_double(0.0)
# totalmodelrunTime = c_double(0.0)
# TsReadTime = c_double(0.0)
# TSStartTime = c_double()
# ComputeStartTime = c_double()
# ComputeTime = c_double(0.0)
# OutWriteTime = c_double()
self.uebVars = (c_char_p * 70)(
"Year", "Month", "Day", "dHour", "atff", "HRI", "Eacl", "Ema",
"conZen", "Ta", "P", "V", "RH", "Qsi", "Qli", "Qnet", "Us", "SWE",
"tausn", "Pr", "Ps", "Alb", "QHs", "QEs", "Es", "SWIT", "QMs", "Q",
"FM", "Tave", "TSURFs", "cump", "cumes", "cumMr", "Qnet", "smelt",
"refDepth", "totalRefDepth", "cf", "Taufb", "Taufd", "Qsib",
"Qsid", "Taub", "Taud", "Qsns", "Qsnc", "Qlns", "Qlnc", "Vz",
"Rkinsc", "Rkinc", "Inmax", "intc", "ieff", "Ur", "Wc", "Tc",
"Tac", "QHc", "QEc", "Ec", "Qpc", "Qmc", "Mc", "FMc", "SWIGM",
"SWISM", "SWIR", "errMB")
C_zName = c_char_p("Outletlocations")
C_tcorvar = pointer((c_float * 13)())
self.C_tsvarArray = pointer((c_float * 13)())
C_tsvarArrayTemp = pointer((c_float * 5)())
#todo: [#] == pointer, * == pointer
self.C_tsvarArray = pointer((POINTER(c_float) * 13)())
C_ntimesteps = pointer((c_int * 5)())
# create pointer to instance of sitevar struct array
self.C_strsvArray = pointer((sitevar * 32)())
# create pointer to instance of inpforcvar struct array
self.C_strinpforcArray = pointer((inpforcvar * 13)())
# mask = c_float()
# pcap_lookupnet(dev, ctypes.byref(net), ctypes.byref(mask), errbuf)
# read watershed netcdf file
self.__uebLib.readwsncFile(C_watershedFile, C_wsvarName, C_wsycorName,
C_wsxcorName, byref(C_wsycorArray),
byref(C_wsxcorArray), byref(self.C_wsArray),
byref(self.C_dimlen1),
byref(self.C_dimlen2),
byref(self.C_wsfillVal))
wsArray1D = numpy.empty((self.C_dimlen1.value * self.C_dimlen2.value),
dtype=numpy.float)
for i in xrange(self.C_dimlen1.value):
for j in xrange(self.C_dimlen2.value):
wsArray1D[i * self.C_dimlen2.value + j] = self.C_wsArray[i][j]
# zvalues is the unique set of wsArray1D
zValues = list(set(wsArray1D))
# fillset = [wsfillVal]
# zVal is the set of zValues that do not equal wsFillVal
zVal = [
zValues[i] for i in xrange(len(zValues))
if zValues[i] != self.C_wsfillVal
]
C_nZones = len(zVal)
z_ycor = [0.0 for i in xrange(C_nZones)]
z_xcor = [0.0 for i in xrange(C_nZones)]
# read params (#194)
self.__uebLib.readParams(C_paramFile, byref(self.C_parvalArray), npar)
# read site variables (#200)
self.__uebLib.readSiteVars(C_sitevarFile, byref(self.C_strsvArray))
# read 2d NetCDF Data
for i in range(0, 32):
a = self.C_strsvArray.contents[i]
if a.svType == 1:
# print "%d %s %s\n" % (i, a.svFile,a.svVarName)
retvalue = self.__uebLib.read2DNC(
os.path.join(self.base_dir, a.svFile), a.svVarName,
byref(a.svArrayValues))
#//read input /forcing control file--all possible entries of input control have to be provided
#readInputForcVars(inputconFile, strinpforcArray);
# read input force variables (main.cpp, line 219)
self.__uebLib.readInputForcVars(cast(C_inputconFile, c_char_p),
self.C_strinpforcArray)
# elog.info('UEB Start Date: %s' % sd.strftime("%m-%d-%Y %H:%M:%S"))
# elog.info('UEB End Date: %s' % ed.strftime("%m-%d-%Y %H:%M:%S"))
# calculate model time span as a julian date (main.cpp, line 220)
modelSpan = jdutil.datetime_to_jd(datetime.datetime(*ModelEndDate)) - \
jdutil.datetime_to_jd(datetime.datetime(*ModelStartDate))
# setup the model start dates as UEB expects them
ModelStartHour = ModelStartDate.pop(
-1) # an integer representing the start hour (24 hour time)
ModelEndHour = ModelEndDate.pop(
-1) # an integer representing the end hour (24 hour time)
# ModelStartDate is a 3 element array: [year, month, day]
# ModelEndDate is a 3 element array: [year, month, day]
# convert Simulation Time parameters into ctypes
self.C_ModelStartDate = (c_int * len(ModelStartDate))(*ModelStartDate)
self.C_ModelEndDate = (c_int * len(ModelEndDate))(*ModelEndDate)
self.C_ModelDt = c_double(ModelDt)
self.C_ModelUTCOffset = c_double(ModelUTCOffset)
self.C_ModelStartHour = c_double(ModelStartHour)
self.C_ModelEndHour = c_double(ModelEndHour)
# calculate model time steps (main.cpp, line 222)
self.numTimeStep = int(math.ceil(modelSpan * (24. / ModelDt))) + 1
# initialize C_tsvarArray values (this replaces __uebLib.readTextData)
self.initialize_timeseries_variable_array(self.C_strinpforcArray,
self.numTimeStep)
# NOTE: C_strinpforcArray stores info about the forcing data files
# # read forcing data (main.cpp, line 226)
# if self.C_strsvArray.contents[16].svType != 3: # no accumulation zone (fixme: ???)
# for it in xrange(13):
# inftype = self.C_strinpforcArray.contents[it].infType
# print 'infFile: ',self.C_strinpforcArray.contents[it].infFile
# if inftype == 0:
#
# # read the files stored in C_strinpforcArray and populated C_tsvarArray
# self.__uebLib.readTextData(os.path.join(self.base_dir, self.C_strinpforcArray.contents[it].infFile), byref(self.C_tsvarArray.contents[it]), byref(C_ntimesteps[0]))
#
# elif inftype == 2 or inftype == -1:
# self.C_tsvarArray.contents[it] = (c_float * 2)()
# C_ntimesteps.contents[0] = 2
# # copy the default value if a single value is the option
# self.C_tsvarArray.contents[it][0] = self.C_strinpforcArray.contents[it].infType
# self.C_tsvarArray.contents[it][1] = self.C_strinpforcArray.contents[it].infdefValue
# :: this array is initialized to (numOut+1, numTimeStep+1) rather than (numOut, numTimeStep)
# :: b/c otherwise the calculations from RunUEB are incorrect for the first row.
# :: e.g.
# :: 2009 2010 5 30.000 23.999979
# :: rather than:
# :: 2009 10 1 0.000 0.569902
# ::
# :: I thought this was b/c numpy.float32 (and 64) are smaller than c_float, however this change
# :: below didn't fix the problem.
# ::
# create a numpy array for outputs
self.outvarArray = numpy.zeros(shape=(numOut + 1, self.numTimeStep),
dtype=numpy.float,
order="C")
# arrays_old = self.outvarArray.astype(numpy.float32)
arrays = self.outvarArray.astype(c_float)
rows, cols = self.outvarArray.shape
arrays_as_list = list(arrays)
#get ctypes handles
ctypes_arrays = [
numpy.ctypeslib.as_ctypes(array) for array in arrays_as_list
]
#Pack into pointer array
self.C_outvarArray = (POINTER(c_float) * rows)(*ctypes_arrays)
# x = numpy.zeros(shape=(numOut, self.numTimeStep), dtype=numpy.float, order="C")
# _floatpp = numpy.ctypeslib.ndpointer(dtype=numpy.uintp, ndim=1, flags='C')
# xpp = (x.__array_interface__['data'][0] + numpy.arange(x.shape[0])*x.strides[0]).astype(numpy.uintp)
# self.C_outvarArray = pointer(((POINTER(c_float) * self.numTimeStep) * numOut)())
# a = (c_float * self.numTimeStep)()
# outvarArray = pointer((a * numOut)())
# for i in xrange(numOut):
# outvarArray[i] = pointer((c_float * self.numTimeStep)())
# total grid size to compute progess
totalgrid = self.C_dimlen1.value * self.C_dimlen2.value
# read output control file (main.cpp, line 251)
# readOutputControl(outputconFile, aggoutputconFile, pOut, ncOut, aggOut, npout, nncout, naggout);
self.__uebLib.readOutputControl(cast(C_outputconFile, c_char_p),
cast(C_aggoutputconFile, c_char_p),
byref(self.C_pOut),
byref(self.C_ncOut), byref(C_aggOut),
byref(self.C_npout),
byref(self.C_nncout), byref(C_naggout))
# create output netcdf
self.C_outtSteps = self.numTimeStep / self.outtStride
self.t_out = numpy.empty(shape=(self.C_outtSteps),
dtype=numpy.float,
order="C")
for i in xrange(self.C_outtSteps):
self.t_out[i] = i * self.outtStride * ModelDt
# initialize the output arrays
aggoutvarArray = numpy.zeros(
(C_nZones, C_naggout.value, self.C_outtSteps), dtype=numpy.float)
totalAgg = numpy.empty((self.C_outtSteps, ), dtype=numpy.float)
ZonesArr = numpy.zeros((C_nZones, ), dtype=numpy.int32)
# main.cpp, line 290
# CREATE 3D NC OUTPUT FILES
# convert self.t_out into a float pointer
C_t_out = self.t_out.ctypes.data_as(POINTER(c_float))
for i in xrange(self.C_nncout.value):
'''
for (int icout = 0; icout < nncout; icout++)
retvalue = create3DNC_uebOutputs(ncOut[icout].outfName, (const char*)ncOut[icout].symbol, (const char*)ncOut[icout].units, tNameout, tUnitsout,
tlong_name, tcalendar, outtSteps, outDimord, self.t_out, &out_fillVal, watershedFile, wsvarName, wsycorName, wsxcorName);
'''
retvalue = self.__uebLib.create3DNC_uebOutputs(
self.C_ncOut[i].outfName, cast(self.C_ncOut[i].symbol,
c_char_p),
cast(self.C_ncOut[i].units,
c_char_p), C_tNameout, C_tUnitsout, C_tlong_name,
C_tcalendar, self.C_outtSteps, self.C_outDimord, C_t_out,
byref(C_out_fillVal), C_watershedFile, C_wsvarName,
C_wsycorName, C_wsxcorName)
# CREATE 3D NC AGGREGATE OUTPUT FILE
# convert z_ycor and x_xcor from list into ctype
C_z_xcor = numpy.asarray(z_xcor).ctypes.data_as(POINTER(c_float))
C_z_ycor = numpy.asarray(z_ycor).ctypes.data_as(POINTER(c_float))
retvalue = self.__uebLib.create3DNC_uebAggregatedOutputs(
C_aggoutputFile, C_aggOut, C_naggout, C_tNameout, C_tUnitsout,
C_tlong_name,
C_tcalendar, self.C_outtSteps, C_aggoutDimord, C_t_out,
byref(C_out_fillVal), C_watershedFile, C_wsvarName, C_wsycorName,
C_wsxcorName, C_nZones, C_zName, C_z_ycor, C_z_xcor)
# todo: create output element set
# print 'Output Calculations available at: '
# for pid in xrange(self.C_npout.value):
# print " Point(",self.C_pOut[pid].xcoord,", ",self.C_pOut[pid].ycoord,') '
# todo: This is where UEB grid points are defined!, expose these as input/output spatial objects
# main.cpp, line 303
self.activeCells = []
# print 'Calculations will be performed at: '
for iy in xrange(self.C_dimlen1.value):
for jx in xrange(self.C_dimlen2.value):
if self.C_wsArray[iy][
jx] != self.C_wsfillVal.value and self.C_strsvArray.contents[
16].svType != 3:
# print " Point(",jx,", ",iy,') '
self.activeCells.append((iy, jx))
# build output exchange items
xcoords = []
ycoords = []
for pid in xrange(self.C_npout.value):
xcoords.append(self.C_pOut[pid].xcoord)
ycoords.append(self.C_pOut[pid].ycoord)
self.pts = geometry.build_point_geometries(xcoords, ycoords)
self.__swe = self.outputs()['Snow Water Equivalent']
self.__swit = self.outputs()['Surface Water Input Total']
self.__swe.addGeometries2(self.pts)
self.__swit.addGeometries2(self.pts)
# build input exchange items
ds = nc.Dataset(C_watershedFile)
Xlist = ds.variables['x']
Ylist = ds.variables['y']
self.geoms = self.build_geometries(Xlist, Ylist)
self.inputs()['Precipitation'].addGeometries2(self.geoms)
self.inputs()['Temperature'].addGeometries2(self.geoms)
# set start, end, and timestep parameters
ts = datetime.timedelta(hours=ModelDt)
self.time_step(ts.total_seconds())
sd = datetime.datetime(*ModelStartDate)
ed = datetime.datetime(*ModelEndDate)
self.simulation_start(sd)
self.simulation_end(ed)
def insert_simulation(db, user, geomcount, valuecount):
print '%d geometries, %d values per geometry' % (geomcount, valuecount)
print 'Total number of data values: %d' % (valuecount * geomcount)
# create an exchange item
unit = mdl.create_unit('cubic meters per second')
variable = mdl.create_variable('streamflow')
# create exchange item
item = stdlib.ExchangeItem(name='Test',
desc='Test Exchange Item',
unit=unit,
variable=variable)
# set exchange item geometries
xcoords = numpy.random.rand(geomcount)
ycoords = numpy.random.rand(geomcount)
points = geometry.build_point_geometries(xcoords, ycoords)
item.addGeometries2(points)
# set exchange item values
start_time = dt.now() # set start time to 'now'
end_time = start_time + timedelta(
days=valuecount - 1) # create endtime base on the number of values
current_time = start_time # initial time
dates = [] # list to hold dates
values = [] # list to hold values for each date
# populate dates list
while current_time <= end_time:
# add date
dates.append(current_time)
# add some random values for each geometry
values.append([random.random() for pt in points])
# increment time by 1 day
current_time += timedelta(days=1)
# set dates and values in the exchange item
item.setValues2(values, dates)
# create the simulation
st = dt(2014, 3, 1, 12, 0, 0)
et = dt(2014, 3, 1, 23, 0, 0)
description = 'Some model descipription'
name = 'test simulation'
# turn off verbosity for simulation insert
os.environ['LOGGING_SHOWINFO'] = '0'
os.environ['LOGGING_SHOWDEBUG'] = '0'
db.create_simulation('My Simulation', user[0], None, item, st, et, 1,
'days', description, name)
os.environ['LOGGING_SHOWINFO'] = '1'
# query simulations
simulations = db.read.getAllSimulations()
simulation = db.read.getSimulationByName('My Simulation')
assert simulation is not None
def __init__(self, config_params):
super(ueb, self).__init__(config_params)
# build inputs and outputs
io = mdl.build_exchange_items_from_config(config_params)
# set input and output exchange items
self.inputs(value=io[stdlib.ExchangeItemType.INPUT])
self.outputs(value=io[stdlib.ExchangeItemType.OUTPUT])
# grab the C library path and the control file path
lib = config_params['lib']
conFile = config_params['control']
# load the UEB C library
self.__uebLib = cdll.LoadLibrary(join(os.path.dirname(__file__), lib))
# save the current directory for saving output data
self.curdir = os.path.dirname(os.path.abspath(conFile))
# the base directory for the control file is used to convert relative paths into absolute paths
self.base_dir = os.path.dirname(conFile)
# get param, sitevar, input, output, and watershed files
with open(conFile, 'r') as f:
# lines = f.readlines()
lines = f.read().splitlines() # this will auto strip the \n \r
C_paramFile = os.path.join(self.base_dir, lines[1])
C_sitevarFile = os.path.join(self.base_dir, lines[2])
C_inputconFile = os.path.join(self.base_dir, lines[3])
C_outputconFile = os.path.join(self.base_dir, lines[4])
C_watershedFile = os.path.join(self.base_dir, lines[5])
C_wsvarName = lines[6].split(' ')[0]
C_wsycorName = lines[6].split(' ')[1]
C_wsxcorName = lines[6].split(' ')[2]
C_aggoutputconFile = os.path.join(self.base_dir, lines[7])
C_aggoutputFile = os.path.join(self.base_dir, lines[8])
ModelStartDate = [int(float(l)) for l in lines[9].split(' ') if l != '']
ModelEndDate = [int(float(l)) for l in lines[10].split(' ') if l != '']
ModelDt = float(lines[11])
ModelUTCOffset = float(lines[12])
inpDailyorSubdaily = bool(lines[13]==True)
self.outtStride, outyStep, outxStep = [int(s) for s in lines[14].split(' ')]
C_wsxcorArray = c_float()
C_wsycorArray = c_float()
self.C_wsArray = pointer(pointer(c_int32()))
self.C_dimlen1 = c_int()
self.C_dimlen2 = c_int()
totalgrid = 0
self.C_wsfillVal = c_int(-9999)
npar = c_int(32)
tinitTime = c_int(0)
self.C_parvalArray = pointer(c_float(0));
numOut = 70 # hack: number of outputs?
self.C_pOut = pointer(pointOutput())
C_aggOut = pointer(aggOutput())
self.C_ncOut = pointer(ncOutput())
self.C_npout = c_int(0)
self.C_nncout = c_int(0)
C_naggout = c_int(0)
C_nZones = c_int(0)
C_tNameout = c_char_p("time")
tunits = (c_char*256)()
C_tUnitsout = pointer(tunits)
C_tlong_name = c_char_p("time")
C_tcalendar = c_char_p("standard")
self.t_out = pointer(c_float(0))
C_out_fillVal = c_float(-9999.0)
self.C_outDimord = c_int(0)
C_aggoutDimord = c_int(1)
self.outvarindx = c_int(17)
# aggoutvarindx = c_int(17)
# size = c_int()
# rank = c_int()
# irank = c_int()
# jrank = c_int()
# startTimeT = c_double(0.0)
# TotalTime = c_double(0.0)
# totalmodelrunTime = c_double(0.0)
# TsReadTime = c_double(0.0)
# TSStartTime = c_double()
# ComputeStartTime = c_double()
# ComputeTime = c_double(0.0)
# OutWriteTime = c_double()
self.uebVars = (c_char_p * 70)("Year", "Month", "Day", "dHour", "atff", "HRI", "Eacl", "Ema", "conZen", "Ta", "P", "V", "RH", "Qsi", "Qli", "Qnet","Us", "SWE", "tausn", "Pr", "Ps", "Alb", "QHs", "QEs", "Es", "SWIT", "QMs", "Q", "FM", "Tave", "TSURFs", "cump", "cumes", "cumMr", "Qnet", "smelt", "refDepth", "totalRefDepth", "cf", "Taufb", "Taufd", "Qsib", "Qsid", "Taub", "Taud", "Qsns", "Qsnc", "Qlns", "Qlnc", "Vz", "Rkinsc", "Rkinc", "Inmax", "intc", "ieff", "Ur", "Wc", "Tc", "Tac", "QHc", "QEc", "Ec", "Qpc", "Qmc", "Mc", "FMc", "SWIGM", "SWISM", "SWIR", "errMB")
C_zName = c_char_p("Outletlocations")
C_tcorvar = pointer((c_float * 13)())
self.C_tsvarArray = pointer((c_float * 13)())
C_tsvarArrayTemp = pointer((c_float * 5)())
#todo: [#] == pointer, * == pointer
self.C_tsvarArray = pointer((POINTER(c_float)*13)())
C_ntimesteps = pointer((c_int * 5)())
# create pointer to instance of sitevar struct array
self.C_strsvArray = pointer((sitevar * 32)())
# create pointer to instance of inpforcvar struct array
self.C_strinpforcArray = pointer((inpforcvar * 13)())
# mask = c_float()
# pcap_lookupnet(dev, ctypes.byref(net), ctypes.byref(mask), errbuf)
# read watershed netcdf file
self.__uebLib.readwsncFile(C_watershedFile, C_wsvarName, C_wsycorName, C_wsxcorName, byref(C_wsycorArray), byref(C_wsxcorArray), byref(self.C_wsArray), byref(self.C_dimlen1), byref(self.C_dimlen2), byref(self.C_wsfillVal))
wsArray1D = numpy.empty((self.C_dimlen1.value*self.C_dimlen2.value),dtype=numpy.float)
for i in xrange(self.C_dimlen1.value) :
for j in xrange(self.C_dimlen2.value):
wsArray1D[i*self.C_dimlen2.value + j] = self.C_wsArray[i][j];
# zvalues is the unique set of wsArray1D
zValues = list(set(wsArray1D))
# fillset = [wsfillVal]
# zVal is the set of zValues that do not equal wsFillVal
zVal = [zValues[i] for i in xrange(len(zValues)) if zValues[i] != self.C_wsfillVal]
C_nZones = len(zVal)
z_ycor = [0.0 for i in xrange(C_nZones)]
z_xcor = [0.0 for i in xrange(C_nZones)]
# read params (#194)
self.__uebLib.readParams(C_paramFile, byref(self.C_parvalArray), npar)
# read site variables (#200)
self.__uebLib.readSiteVars(C_sitevarFile, byref(self.C_strsvArray))
# read 2d NetCDF Data
for i in range(0,32):
a = self.C_strsvArray.contents[i]
if a.svType == 1:
# print "%d %s %s\n" % (i, a.svFile,a.svVarName)
retvalue = self.__uebLib.read2DNC(os.path.join(self.base_dir, a.svFile), a.svVarName, byref(a.svArrayValues))
#//read input /forcing control file--all possible entries of input control have to be provided
#readInputForcVars(inputconFile, strinpforcArray);
# read input force variables (main.cpp, line 219)
self.__uebLib.readInputForcVars(cast(C_inputconFile,c_char_p), self.C_strinpforcArray)
# elog.info('UEB Start Date: %s' % sd.strftime("%m-%d-%Y %H:%M:%S"))
# elog.info('UEB End Date: %s' % ed.strftime("%m-%d-%Y %H:%M:%S"))
# calculate model time span as a julian date (main.cpp, line 220)
modelSpan = jdutil.datetime_to_jd(datetime.datetime(*ModelEndDate)) - \
jdutil.datetime_to_jd(datetime.datetime(*ModelStartDate))
# setup the model start dates as UEB expects them
ModelStartHour = ModelStartDate.pop(-1) # an integer representing the start hour (24 hour time)
ModelEndHour = ModelEndDate.pop(-1) # an integer representing the end hour (24 hour time)
# ModelStartDate is a 3 element array: [year, month, day]
# ModelEndDate is a 3 element array: [year, month, day]
# convert Simulation Time parameters into ctypes
self.C_ModelStartDate = (c_int * len(ModelStartDate))(*ModelStartDate)
self.C_ModelEndDate =(c_int * len(ModelEndDate))(*ModelEndDate)
self.C_ModelDt = c_double(ModelDt)
self.C_ModelUTCOffset = c_double(ModelUTCOffset)
self.C_ModelStartHour = c_double(ModelStartHour)
self.C_ModelEndHour = c_double(ModelEndHour)
# calculate model time steps (main.cpp, line 222)
self.numTimeStep = int(math.ceil(modelSpan*(24./ModelDt)) ) + 1
# initialize C_tsvarArray values (this replaces __uebLib.readTextData)
self.initialize_timeseries_variable_array(self.C_strinpforcArray, self.numTimeStep)
# NOTE: C_strinpforcArray stores info about the forcing data files
# # read forcing data (main.cpp, line 226)
# if self.C_strsvArray.contents[16].svType != 3: # no accumulation zone (fixme: ???)
# for it in xrange(13):
# inftype = self.C_strinpforcArray.contents[it].infType
# print 'infFile: ',self.C_strinpforcArray.contents[it].infFile
# if inftype == 0:
#
# # read the files stored in C_strinpforcArray and populated C_tsvarArray
# self.__uebLib.readTextData(os.path.join(self.base_dir, self.C_strinpforcArray.contents[it].infFile), byref(self.C_tsvarArray.contents[it]), byref(C_ntimesteps[0]))
#
# elif inftype == 2 or inftype == -1:
# self.C_tsvarArray.contents[it] = (c_float * 2)()
# C_ntimesteps.contents[0] = 2
# # copy the default value if a single value is the option
# self.C_tsvarArray.contents[it][0] = self.C_strinpforcArray.contents[it].infType
# self.C_tsvarArray.contents[it][1] = self.C_strinpforcArray.contents[it].infdefValue
# :: this array is initialized to (numOut+1, numTimeStep+1) rather than (numOut, numTimeStep)
# :: b/c otherwise the calculations from RunUEB are incorrect for the first row.
# :: e.g.
# :: 2009 2010 5 30.000 23.999979
# :: rather than:
# :: 2009 10 1 0.000 0.569902
# ::
# :: I thought this was b/c numpy.float32 (and 64) are smaller than c_float, however this change
# :: below didn't fix the problem.
# ::
# create a numpy array for outputs
self.outvarArray = numpy.zeros(shape=(numOut+1, self.numTimeStep), dtype=numpy.float, order="C")
# arrays_old = self.outvarArray.astype(numpy.float32)
arrays = self.outvarArray.astype(c_float)
rows, cols = self.outvarArray.shape
arrays_as_list = list(arrays)
#get ctypes handles
ctypes_arrays = [numpy.ctypeslib.as_ctypes(array) for array in arrays_as_list]
#Pack into pointer array
self.C_outvarArray = (POINTER(c_float) * rows)(*ctypes_arrays)
# x = numpy.zeros(shape=(numOut, self.numTimeStep), dtype=numpy.float, order="C")
# _floatpp = numpy.ctypeslib.ndpointer(dtype=numpy.uintp, ndim=1, flags='C')
# xpp = (x.__array_interface__['data'][0] + numpy.arange(x.shape[0])*x.strides[0]).astype(numpy.uintp)
# self.C_outvarArray = pointer(((POINTER(c_float) * self.numTimeStep) * numOut)())
# a = (c_float * self.numTimeStep)()
# outvarArray = pointer((a * numOut)())
# for i in xrange(numOut):
# outvarArray[i] = pointer((c_float * self.numTimeStep)())
# total grid size to compute progess
totalgrid = self.C_dimlen1.value*self.C_dimlen2.value
# read output control file (main.cpp, line 251)
# readOutputControl(outputconFile, aggoutputconFile, pOut, ncOut, aggOut, npout, nncout, naggout);
self.__uebLib.readOutputControl(cast(C_outputconFile,c_char_p), cast(C_aggoutputconFile, c_char_p),
byref(self.C_pOut), byref(self.C_ncOut), byref(C_aggOut),
byref(self.C_npout), byref(self.C_nncout), byref(C_naggout))
# create output netcdf
self.C_outtSteps = self.numTimeStep / self.outtStride
self.t_out = numpy.empty(shape=(self.C_outtSteps), dtype=numpy.float, order="C")
for i in xrange(self.C_outtSteps):
self.t_out[i] = i*self.outtStride*ModelDt
# initialize the output arrays
aggoutvarArray = numpy.zeros((C_nZones,C_naggout.value, self.C_outtSteps), dtype=numpy.float)
totalAgg = numpy.empty((self.C_outtSteps,), dtype=numpy.float)
ZonesArr = numpy.zeros((C_nZones,), dtype=numpy.int32)
# main.cpp, line 290
# CREATE 3D NC OUTPUT FILES
# convert self.t_out into a float pointer
C_t_out = self.t_out.ctypes.data_as(POINTER(c_float))
for i in xrange(self.C_nncout.value):
'''
for (int icout = 0; icout < nncout; icout++)
retvalue = create3DNC_uebOutputs(ncOut[icout].outfName, (const char*)ncOut[icout].symbol, (const char*)ncOut[icout].units, tNameout, tUnitsout,
tlong_name, tcalendar, outtSteps, outDimord, self.t_out, &out_fillVal, watershedFile, wsvarName, wsycorName, wsxcorName);
'''
retvalue = self.__uebLib.create3DNC_uebOutputs(self.C_ncOut[i].outfName, cast(self.C_ncOut[i].symbol, c_char_p), cast(self.C_ncOut[i].units, c_char_p), C_tNameout, C_tUnitsout, C_tlong_name, C_tcalendar, self.C_outtSteps, self.C_outDimord, C_t_out, byref(C_out_fillVal), C_watershedFile, C_wsvarName, C_wsycorName, C_wsxcorName);
# CREATE 3D NC AGGREGATE OUTPUT FILE
# convert z_ycor and x_xcor from list into ctype
C_z_xcor = numpy.asarray(z_xcor).ctypes.data_as(POINTER(c_float))
C_z_ycor = numpy.asarray(z_ycor).ctypes.data_as(POINTER(c_float))
retvalue = self.__uebLib.create3DNC_uebAggregatedOutputs(C_aggoutputFile, C_aggOut, C_naggout, C_tNameout, C_tUnitsout, C_tlong_name, C_tcalendar, self.C_outtSteps, C_aggoutDimord, C_t_out, byref(C_out_fillVal), C_watershedFile, C_wsvarName, C_wsycorName, C_wsxcorName, C_nZones, C_zName, C_z_ycor, C_z_xcor);
# todo: create output element set
# print 'Output Calculations available at: '
# for pid in xrange(self.C_npout.value):
# print " Point(",self.C_pOut[pid].xcoord,", ",self.C_pOut[pid].ycoord,') '
# todo: This is where UEB grid points are defined!, expose these as input/output spatial objects
# main.cpp, line 303
self.activeCells = []
# print 'Calculations will be performed at: '
for iy in xrange(self.C_dimlen1.value):
for jx in xrange(self.C_dimlen2.value):
if self.C_wsArray[iy][jx] != self.C_wsfillVal.value and self.C_strsvArray.contents[16].svType != 3:
# print " Point(",jx,", ",iy,') '
self.activeCells.append((iy, jx))
# build output exchange items
xcoords = []
ycoords = []
for pid in xrange(self.C_npout.value):
xcoords.append(self.C_pOut[pid].xcoord)
ycoords.append(self.C_pOut[pid].ycoord)
self.pts = geometry.build_point_geometries(xcoords, ycoords)
self.__swe = self.outputs()['Snow Water Equivalent']
self.__swit = self.outputs()['Surface Water Input Total']
self.__swe.addGeometries2(self.pts)
self.__swit.addGeometries2(self.pts)
# build input exchange items
ds = nc.Dataset(C_watershedFile)
Xlist = ds.variables['x']
Ylist = ds.variables['y']
self.geoms = self.build_geometries(Xlist, Ylist)
self.inputs()['Precipitation'].addGeometries2(self.geoms)
self.inputs()['Temperature'].addGeometries2(self.geoms)
# set start, end, and timestep parameters
ts = datetime.timedelta(hours=ModelDt)
self.time_step(ts.total_seconds())
sd = datetime.datetime(*ModelStartDate)
ed = datetime.datetime(*ModelEndDate)
self.simulation_start(sd)
self.simulation_end(ed)
def __init__(self, args):
super(Wrapper, self).__init__()
handle = netCDF4.Dataset(args["ncpath"], "r")
variables = handle.variables.keys()
tdim = args["tdim"]
xdim = args["xdim"]
ydim = args["ydim"]
tunit = {args["tunit"]: 1}
if isinstance(args["starttime"], datetime.datetime):
st = args["starttime"]
else:
st = parser.parse(args["starttime"])
# make sure the variables provided exist in the nc file
assert tdim in variables, "time variable name not specified. Cannot continue"
assert xdim in variables, "x dimension variable name not specified. Cannot continue"
assert ydim in variables, "y dimension variable name not specified. Cannot continue"
# get data for these variables
timesteps = handle.variables[tdim][:]
times = []
for ts in timesteps:
# update the time unit value
tunit[args["tunit"]] = ts
# unpack the tunit dictionary to create a timedelta object
dt = datetime.timedelta(**tunit)
times.append(st + dt)
variables.remove(tdim)
x = handle.variables[xdim][:]
variables.remove(xdim)
y = handle.variables[ydim][:]
variables.remove(ydim)
# create flattened lists of x,y coords from meshgrid
xcoords, ycoords = numpy.meshgrid(x, y)
xcoords = xcoords.flatten()
ycoords = ycoords.flatten()
# loop through the remaining variables and expose them as outputs
# for var in variables:
for v in range(len(variables)):
var = variables[v]
# create a unit
unit = stdlib.Unit()
unit.UnitName(handle.variables[var].units if "units" in dir(handle.variables[var]) else "N/A")
unit.UnitTypeCV("N/A")
unit.UnitAbbreviation("N/A")
# create a variable
variable = stdlib.Variable()
variable.VariableNameCV(handle.variables[var].name)
variable.VariableDefinition("N/A")
# create geometries
geoms = geometry.build_point_geometries(xcoords, ycoords) # This takes way to long
# create exchange item
oei = stdlib.ExchangeItem(
name=variable.VariableNameCV(),
desc="Autogenerated variable parsed from %s" % args["ncpath"],
geometry=geoms,
unit=unit,
variable=variable,
type=stdlib.ExchangeItemType.OUTPUT,
)
# flatten each timestep of the data
values = [v.flatten() for v in handle.variables[var][:]]
# set these data. Skip occurrences with mismatched values and times arrays
if len(values) == len(times):
success = oei.setValues2(values, times)
# only expose the exchange item if data was set properly
if success:
# save the oei
self.outputs(oei)
# set metadata
name = args["ncpath"].split("/")[-1]
self.name(name)
self.description("NetCDF data component, " + name)
self.simulation_start(times[0])
self.simulation_end(times[-1])
self.status(stdlib.Status.READY)
def calc_ti_geometries(self):
sPrint('Reading Topographic Indices')
with open(self.topo_input, 'r') as sr:
lines = sr.readlines()
ncols = int(lines[0].split(' ')[-1].strip())
nrows = int(lines[1].split(' ')[-1].strip())
lowerx = float(lines[2].split(' ')[-1].strip())
lowery = float(lines[3].split(' ')[-1].strip())
cellsize = float(lines[4].split(' ')[-1].strip())
nodata = float(lines[5].split(' ')[-1].strip())
# read ti data
data = np.genfromtxt(self.topo_input, delimiter=' ', skip_header=6)
# build X and Y coordinate arrays
xi = np.linspace(lowerx, lowerx+ncols*cellsize, ncols)
yi = np.linspace(lowery+nrows*cellsize, lowery, nrows)
x,y = np.meshgrid(xi,yi) # generate 2d arrays from xi, yi
x = x.ravel() # convert to 1-d
y = y.ravel() # convert to 1-d
data = data.ravel() # convert to 1-d
# remove all nodata points from x, y arrays
nonzero = np.where(data != nodata)
x = x[nonzero]
y = y[nonzero]
with open(self.fac_input, 'r') as fac:
lines = fac.readlines()
fac_ncols = int(lines[0].split(' ')[-1].strip())
fac_nrows = int(lines[1].split(' ')[-1].strip())
fac_lowerx = float(lines[2].split(' ')[-1].strip())
fac_lowery = float(lines[3].split(' ')[-1].strip())
fac_cellsize = float(lines[4].split(' ')[-1].strip())
fac_nodata = float(lines[5].split(' ')[-1].strip())
# read ti data
fac_data = np.genfromtxt(self.fac_input, delimiter=' ', skip_header=6)
# build X and Y coordinate arrays
xi = np.linspace(fac_lowerx, fac_lowerx + fac_ncols * fac_cellsize, fac_ncols)
yi = np.linspace(fac_lowery + fac_nrows * fac_cellsize, fac_lowery, fac_nrows)
x,y = np.meshgrid(xi,yi) # generate 2d arrays from xi, yi
x = x.ravel() # convert to 1-d
y = y.ravel() # convert to 1-d
fac_data = fac_data.ravel() # convert to 1-d
# isolate the rivers based on threshold
threshold = np.max(fac_data) # todo, user input
rivers = np.where((fac_data != fac_nodata) & (fac_data >= threshold) )
# select only the x and y values that are considered rivers
x = x[rivers]
y = y[rivers]
self.rivers_idxs = rivers
sPrint('Building Geometry Objects')
tigeoms = geometry.build_point_geometries(x,y)
self.ti_geoms = tigeoms
