def test_base_outputs(self):
outputs = self.base.outputs()
self.assertTrue(outputs == {})
geom = geometry.fromWKT('POINT(1 2)')
unit = stdlib.Unit()
unit.UnitAbbreviation('cm')
unit.UnitName('centimeters')
var = stdlib.Variable()
var.VariableNameCV('my variable')
var.VariableDefinition('this is my variable definition')
oei = stdlib.ExchangeItem(name='output1',
desc='my description',
unit=unit,
variable=var,
geometry=geom,
type=stdlib.ExchangeItemType.OUTPUT)
outputs = self.base.outputs(oei)
self.assertTrue(len(outputs.items()) == 1)
# add duplicate
outputs = self.base.outputs(oei)
self.assertTrue(len(outputs.items()) == 1)
# try to add an input type
iei = stdlib.ExchangeItem(name='input1',
desc='my description',
unit=unit,
variable=var,
geometry=geom,
type=stdlib.ExchangeItemType.INPUT)
outputs = self.base.outputs(iei)
self.assertTrue(len(outputs.items()) == 1)
def setup_model(self, geomcount, valuecount):
# create an exchange item
unit = mdl.create_unit('cubic meters per second')
variable = mdl.create_variable('streamflow')
# create exchange item
item = stdlib.ExchangeItem(name='Test', desc='Test Exchange Item',
unit=unit, variable=variable)
# set exchange item geometries
xcoords = [i for i in range(geomcount)]
ycoords = [i*1.5 for i in range(geomcount)]
geoms = geometry.build_point_geometries(xcoords, ycoords)
item.addGeometries2(geoms)
# set exchange item values
start_time = dt.now()
end_time = start_time+timedelta(minutes=valuecount - 1)
time_step = 60 # in seconds
item.initializeDatesValues(start_datetime=start_time, end_datetime=end_time, timestep_in_seconds=time_step)
values = numpy.random.rand(valuecount, geomcount)
# for i in range(len(dates)):
item.setValuesBySlice(values) #, time_index_slice=(0,num_timesteps,1))
return item, geoms
def __init__(self, resultid, session):
# get result object and result timeseries
core = readCore(session)
obj = core.getResultByID(resultID=int(resultid))
readres = readResults(session)
results = readres.getTimeSeriesValuesByResultId(resultId=int(resultid))
# separate the date and value pairs in the timeseries
dates = [date.ValueDateTime for date in results]
values = [val.DataValue for val in results]
# basic exchange item info
id = uuid.uuid4().hex[:8]
name = obj.VariableObj.VariableCode
desc = obj.VariableObj.VariableDefinition
#unit = obj.UnitObj.UnitsName
#vari = obj.VariableObj.VariableNameCV
type = stdlib.ExchangeItemType.OUTPUT
start = min(dates)
end = max(dates)
# build variable
variable = stdlib.Variable()
variable.VariableDefinition(obj.VariableObj.VariableDefinition)
variable.VariableNameCV(obj.VariableObj.VariableNameCV)
# build unit
unit = stdlib.Unit()
unit.UnitAbbreviation(obj.UnitObj.UnitsAbbreviation)
unit.UnitName(obj.UnitObj.UnitsName)
unit.UnitTypeCV(obj.UnitObj.UnitsTypeCV)
# build geometries
# todo: need to specify srs and elevation
wkb = str(obj.FeatureActionObj.SamplingFeatureObj.FeatureGeometry.data)
geom = geometry.fromWKB(wkb)
# build exchange item object
oei = stdlib.ExchangeItem(id=id,
name=name,
desc=desc,
geometry=geom,
unit=unit,
variable=variable,
type=type)
# set global parameters
self.__id = id
self.__name = name
self.__start = start
self.__end = end
self.__output = {self.__name: oei}
self.__desc = obj.VariableObj.VariableDefinition
self.__current_time = self.simulation_start()
self.__obj = obj
self.__resultid = obj.ResultID
self.__session = session
self.__status = Status.Loaded
def __init__(self, args):
super(Wrapper, self).__init__()
self.args = args
wsdl = self.args['wsdl']
network = self.args['network']
variable = '%s:%s' % (network, self.args['variable'])
site = '%s:%s'%(network, self.args['site'])
st = self.args['start']
et = self.args['end']
# make sure that start and end time are valid date time objects
if not isinstance(st, datetime.datetime):
st = parser.parse(st)
if not isinstance(et, datetime.datetime):
et = parser.parse(et)
# format start and end datetime into wof acceptable format
start = st.strftime('%Y-%m-%dT%H:%M:%S')
end = et.strftime('%Y-%m-%dT%H:%M:%S')
# connect to the server
client = Client(wsdl)
# get data
queryInfo, timeSeries = client.service.GetValuesObject(site, variable, start, end)
label, series = timeSeries
# grab the first timeseries only
ts = series[0]
sourceInfo = ts['sourceInfo']
variable = ts['variable']
values = ts['values'][0]
variableName = variable['variableName']
unitName = variable['unit']['unitName']
unitType = variable['unit']['unitType']
unitAbbreviation = variable['unit']['unitAbbreviation']
siteDescription = sourceInfo['siteName']
geolocation_x = sourceInfo['geoLocation']['geogLocation']['longitude']
geolocation_y = sourceInfo['geoLocation']['geogLocation']['latitude']
geolocation_z = sourceInfo['elevation_m']
# build unit
unit = stdlib.Unit()
unit.UnitName(unitName)
unit.UnitAbbreviation(unitAbbreviation)
unit.UnitTypeCV(unitType)
# build variable
var = stdlib.Variable()
var.VariableNameCV(variableName)
var.VariableDefinition(' ')
# build geometry
wkt = 'POINT(%s %s %s)' % (geolocation_x, geolocation_y, geolocation_z)
geom = geometry.fromWKT(wkt)
# build output exchange items
oei = stdlib.ExchangeItem(name=variableName,
desc=siteDescription,
geometry=geom,
unit=unit,
variable=var,
type=stdlib.ExchangeItemType.OUTPUT,
# srs_epsg= ''
)
qualityControlLevelId = values['qualityControlLevel'][0]['qualityControlLevelCode']
qualityControlLevelDef = values['qualityControlLevel'][0]['definition']
qualityControlLevelExp = values['qualityControlLevel'][0]['explanation']
dates = []
vals = []
# set data values
for value in values.value:
vals.append(float(value.value))
dates.append(value._dateTime)
oei.setValues2(values=vals, timevalue=dates)
# set output
self.outputs(oei)
# set component metadata
self.description(siteDescription)
self.name(variableName)
self.simulation_start(start)
self.simulation_end(end)
def __init__(self, args):
super(Wrapper, self).__init__()
handle = netCDF4.Dataset(args['ncpath'], 'r')
variables = handle.variables.keys()
tdim = args['tdim']
xdim = args['xdim']
ydim = args['ydim']
tunit = {args['tunit']: 1}
if isinstance(args['starttime'], datetime.datetime):
st = args['starttime']
else:
st = parser.parse(args['starttime'])
# make sure the variables provided exist in the nc file
assert tdim in variables, 'time variable name not specified. Cannot continue'
assert xdim in variables, 'x dimension variable name not specified. Cannot continue'
assert ydim in variables, 'y dimension variable name not specified. Cannot continue'
# get data for these variables
timesteps = handle.variables[tdim][:]
times = []
for ts in timesteps:
# update the time unit value
tunit[args['tunit']] = ts
# unpack the tunit dictionary to create a timedelta object
dt = datetime.timedelta(**tunit)
times.append(st + dt)
variables.remove(tdim)
x = handle.variables[xdim][:]
variables.remove(xdim)
y = handle.variables[ydim][:]
variables.remove(ydim)
# create flattened lists of x,y coords from meshgrid
xcoords, ycoords = numpy.meshgrid(x, y)
xcoords = xcoords.flatten()
ycoords = ycoords.flatten()
# loop through the remaining variables and expose them as outputs
# for var in variables:
for v in range(len(variables)):
var = variables[v]
# create a unit
unit = stdlib.Unit()
unit.UnitName(handle.variables[var].units if 'units' in
dir(handle.variables[var]) else 'N/A')
unit.UnitTypeCV("N/A")
unit.UnitAbbreviation("N/A")
# create a variable
variable = stdlib.Variable()
variable.VariableNameCV(handle.variables[var].name)
variable.VariableDefinition("N/A")
# create geometries
geoms = geometry.build_point_geometries(
xcoords, ycoords) # This takes way to long
# create exchange item
oei = stdlib.ExchangeItem(
name=variable.VariableNameCV(),
desc='Autogenerated variable parsed from %s' % args['ncpath'],
geometry=geoms,
unit=unit,
variable=variable,
type=stdlib.ExchangeItemType.OUTPUT)
# flatten each timestep of the data
values = [v.flatten() for v in handle.variables[var][:]]
# set these data. Skip occurrences with mismatched values and times arrays
if len(values) == len(times):
success = oei.setValues2(values, times)
# only expose the exchange item if data was set properly
if success:
# save the oei
self.outputs(oei)
# set metadata
name = args['ncpath'].split('/')[-1]
self.name(name)
self.description('NetCDF data component, ' + name)
self.simulation_start(times[0])
self.simulation_end(times[-1])
self.status(stdlib.Status.READY)
def insert_simulation(db, user, geomcount, valuecount):
print '%d geometries, %d values per geometry' % (geomcount, valuecount)
print 'Total number of data values: %d' % (valuecount * geomcount)
# create an exchange item
unit = mdl.create_unit('cubic meters per second')
variable = mdl.create_variable('streamflow')
# create exchange item
item = stdlib.ExchangeItem(name='Test',
desc='Test Exchange Item',
unit=unit,
variable=variable)
# set exchange item geometries
xcoords = numpy.random.rand(geomcount)
ycoords = numpy.random.rand(geomcount)
points = geometry.build_point_geometries(xcoords, ycoords)
item.addGeometries2(points)
# set exchange item values
start_time = dt.now() # set start time to 'now'
end_time = start_time + timedelta(
days=valuecount - 1) # create endtime base on the number of values
current_time = start_time # initial time
dates = [] # list to hold dates
values = [] # list to hold values for each date
# populate dates list
while current_time <= end_time:
# add date
dates.append(current_time)
# add some random values for each geometry
values.append([random.random() for pt in points])
# increment time by 1 day
current_time += timedelta(days=1)
# set dates and values in the exchange item
item.setValues2(values, dates)
# create the simulation
st = dt(2014, 3, 1, 12, 0, 0)
et = dt(2014, 3, 1, 23, 0, 0)
description = 'Some model descipription'
name = 'test simulation'
# turn off verbosity for simulation insert
os.environ['LOGGING_SHOWINFO'] = '0'
os.environ['LOGGING_SHOWDEBUG'] = '0'
db.create_simulation('My Simulation', user[0], None, item, st, et, 1,
'days', description, name)
os.environ['LOGGING_SHOWINFO'] = '1'
# query simulations
simulations = db.read.getAllSimulations()
simulation = db.read.getSimulationByName('My Simulation')
assert simulation is not None
def build_exchange_items_from_config(params):
# get all inputs and outputs
iitems = params['input'] if 'input' in params else []
oitems = params['output'] if 'output' in params else []
eitems = iitems + oitems
items = {stdlib.ExchangeItemType.INPUT:[],stdlib.ExchangeItemType.OUTPUT:[]}
# loop through each input/output and create an exchange item
for io in eitems:
variable = None
unit = None
srs = None
geoms = []
# get all input and output exchange items as a list
iotype = stdlib.ExchangeItemType.OUTPUT if io['type'].upper() == stdlib.ExchangeItemType.OUTPUT else stdlib.ExchangeItemType.INPUT
for key, value in io.iteritems():
sPrint(key, MessageType.DEBUG)
if key == 'variable_name_cv':
sPrint('Creating Variable', MessageType.DEBUG)
variable = create_variable(value)
sPrint('Done Creating Variable', MessageType.DEBUG)
if 'variable_definition' in io.keys():
variable.VariableDefinition(io['variable_definition'])
elif key == 'unit_type_cv': unit = create_unit(value)
elif key == 'elementset' :
# check if the value is a path
if os.path.dirname(value ) != '':
gen_path = os.path.abspath(os.path.join(params['basedir'],value))
if not os.path.isfile(gen_path):
# get filepath relative to *.mdl
elog.critical('Could not find file at path %s, generated from relative path %s'%(gen_path, value))
raise Exception('Could not find file at path %s, generated from relative path %s'%(gen_path, value))
# parse the geometry from the shapefile
geoms, srs = utilities.spatial.read_shapefile(gen_path)
srs = srs.AutoIdentifyEPSG()
# otherwise it must be a wkt
else:
try:
# get the wkt text string
value = value.strip('\'').strip('"')
# parse the wkt string into a stdlib.Geometry object
geom = utilities.geometry.fromWKT(value)
for g in geom:
geoms.append(g)
except:
elog.warning('Could not load component geometry from *.mdl file')
# this is OK. Just set the geoms to [] and assume that they will be populated during initialize.
geom = []
if 'espg_code' in io:
srs = utilities.spatial.get_srs_from_epsg(io['epsg_code'])
# generate a unique uuid for this exchange item
id = uuid.uuid4().hex
# create exchange item
ei = stdlib.ExchangeItem(id,
name=variable.VariableNameCV(),
desc=variable.VariableDefinition(),
unit= unit,
variable=variable,
# srs_epsg=srs, #todo: this is causing problems
type=iotype)
# add geometry to exchange item (NEW)
ei.addGeometries2(geoms)
# save exchange items based on type
items[ei.type()].append(ei)
return items
def construct_oeis(self, weather_data):
# read the weather data csv
f = open(weather_data, 'r')
lines = f.readlines()
f.close()
# read ei metadata
ei_data = []
for i in range(16,len(lines)):
line = lines[i]
if line[0] == '#' and len(line) > 3:
if line[2] == 'V' and line[3] != '[':
data = line.split('=')
data = data[1].split(',')
trimmed = [d.strip() for d in data]
ei_data.append(trimmed)
elif line[0] != '#':
break
def make_date(datestr):
return dt.strptime(datestr, '%m-%d-%Y %H:%M:%S')
# parse the weather data dates and values into numpy arrays
date_arr = numpy.genfromtxt(weather_data, delimiter=',',
converters={'Date':make_date},
names= ['Date'], dtype=None,
usecols=[0])
date_arr = [d[0] for d in date_arr]
val_arr = numpy.genfromtxt(weather_data, delimiter=',', dtype=float)
val_arr = numpy.delete(val_arr,0,1)
# build exchange items
col_idx = 0
for item in ei_data:
# map to stdlib units and variables
unit = mdl.create_unit(item[2])
variable = mdl.create_variable(item[1])
uid = uuid.uuid4().hex
ei = stdlib.ExchangeItem(id=uid,
name=item[0],
unit=unit,
variable=variable,
desc=item[3],
type=stdlib.ExchangeItemType.OUTPUT)
# build geometry
pt = geometry.build_point_geometry(float(item[-2]), float(item[-1]))
ei.addGeometry(pt)
# add the oei to the outputs list
self.outputs(value=ei)
# save the data associated with this exchange item
self.weather_data[uid] = [date_arr, val_arr[:, col_idx]]
# increment the column index for the numpy array
col_idx += 1
def build_exchange_items(params):
exchange_items = []
oei = []
iei = []
# get all inputs and outputs
eitems = params['input'] if 'input' in params else [] + params[
'output'] if 'output' in params else []
itemid = 0
# loop through each input/output and create an exchange item
for io in eitems:
variable = None
unit = None
elementset = []
iotype = stlib.ExchangeItemType.Output if io['type'].lower(
) == 'output' else stlib.ExchangeItemType.Input
#if 'output' in io.keys(): type = stlib.ExchangeItemType.Output
#else: type = stlib.ExchangeItemType.Input
for key, value in io.iteritems():
if key == 'variable_name_cv': variable = create_variable(value)
elif key == 'unit_type_cv': unit = create_unit(value)
elif key == 'elementset':
# check if the value is a path
if os.path.dirname(value) != '':
if not os.path.isfile(value):
raise Exception('Could not find file: %s' % value)
geom, srs = read_shapefile(value)
# otherwise it must be a wkt
else:
try:
value = value.strip('\'').strip('"')
geoms = wkt.loads(value)
geom = []
if 'Multi' in geoms.geometryType():
geom = [g for g in geoms]
else:
geom = [geoms]
except:
raise Exception('Could not load WKT string: %s.' %
value)
srs = get_srs_from_epsg(io['epsg_code'])
for element in geom:
# define initial dataset for element
dv = stlib.DataValues()
# create element
elem = stlib.Geometry()
elem.geom(element)
elem.type(element.geom_type)
elem.srs(srs)
elem.datavalues(dv)
elementset.append(elem)
# increment item id
itemid += 1
id = iotype.upper() + str(itemid)
# create exchange item
ei = stlib.ExchangeItem(id,
name=variable.VariableNameCV(),
desc=variable.VariableDefinition(),
geometry=elementset,
unit=unit,
variable=variable,
type=iotype)
# add to exchange item
#for ds in datasets:
# ei.add_geometry(ds)
exchange_items.append(ei)
return exchange_items
def __init__(self, args):
super(wrapper, self).__init__()
self.args = args
session = self.args['session']
resultid = self.args['resultid']
# get result object and result timeseries
core = readCore(session)
obj = core.getResultByID(resultID=int(resultid))
readres = readResults(session)
results = readres.getTimeSeriesValuesByResultId(resultId=int(resultid))
# separate the date and value pairs in the timeseries
dates = [date.ValueDateTime for date in results]
values = [val.DataValue for val in results]
# basic exchange item info
name = obj.VariableObj.VariableCode
desc = obj.VariableObj.VariableDefinition
type = stdlib.ExchangeItemType.OUTPUT
start = min(dates)
end = max(dates)
# build variable
variable = stdlib.Variable()
variable.VariableDefinition(obj.VariableObj.VariableDefinition)
variable.VariableNameCV(obj.VariableObj.VariableNameCV)
# build unit
unit = stdlib.Unit()
unit.UnitAbbreviation(obj.UnitObj.UnitsAbbreviation)
unit.UnitName(obj.UnitObj.UnitsName)
unit.UnitTypeCV(obj.UnitObj.UnitsTypeCV)
# build geometries
# todo: need to specify srs and elevation
wkb = str(obj.FeatureActionObj.SamplingFeatureObj.FeatureGeometry.data)
geom = geometry.fromWKB(wkb)
# build exchange item object
oei = stdlib.ExchangeItem(name=name,
desc=desc,
geometry=geom,
unit=unit,
variable=variable,
type=type)
# set global parameters
self.name(name)
self.simulation_start(start)
self.simulation_end(end)
self.outputs(name=name, value=oei)
self.description(obj.VariableObj.VariableDefinition)
self.current_time(start)
# self.__obj = obj
# self.__resultid = obj.ResultID
# self.__session = session
# set model status
self.status(Status.Loaded)