def __init__(self, url=None, session_id=None):
self.json_net = None
self.warnings = []
self.files = []
self.template = None
self.attributes = []
self.attr_name_map = {}
self.nodes = {}
self.links = {}
self.groups = {}
self.node_id = PluginLib.temp_ids()
self.link_id = PluginLib.temp_ids()
self.group_id = PluginLib.temp_ids()
self.connection = JsonConnection(url)
if session_id is not None:
write_output("Using existing session %s" % session_id)
self.connection.session_id = session_id
else:
self.connection.login()
#3 steps: start, read, save
self.num_steps = 3
def __init__(self, url=None, session_id=None):
#Record the names of the files created by the plugin so we can
#display them to the user.
self.files = []
#A mapping from attr ID to attr object. Makes searching for attributes
#easier
self.attr_id_map = {}
self.connection = JsonConnection(url)
write_output("Connecting...")
if session_id is not None:
write_output("Using existing session %s"% session_id)
self.connection.session_id=session_id
else:
self.connection.login()
#Dictionaries, keyed on their name, where all the nodes, links & groups will go
self.hydra_nodes = {}
self.hydra_links = {}
self.hydra_groups = {}
self.hydra_group_members = {}
#Generators of NEGATIVE IDS for the nodes & links.
#Negative IDS are used as temporary client-side IDS until the network
#is saved, at which point the resources will be given permanent, positive IDS.
self.node_ids = temp_ids()
self.link_ids = temp_ids()
self.group_ids = temp_ids() # A group is an institution.
#A mapping from the name of a type to the type itself.
self.type_name_map = {}
self.num_steps = 3
def __init__(self, url=None, session_id=None):
self.json_net = None
self.warnings = []
self.files = []
self.template = None
self.attributes = []
self.attr_name_map = {}
self.nodes = {}
self.links = {}
self.groups = {}
self.node_id = PluginLib.temp_ids()
self.link_id = PluginLib.temp_ids()
self.group_id = PluginLib.temp_ids()
self.connection = JsonConnection(url)
if session_id is not None:
write_output("Using existing session %s"% session_id)
self.connection.session_id=session_id
else:
self.connection.login()
#3 steps: start, read, save
self.num_steps = 3
def __init__(self, url=None, username=None, password=None):
self.conn = JsonConnection(url)
if username is not None and password is not None:
self.session_id = self.conn.login(username=username,
password=password)
else:
self.session_id = self.conn.login()
self.projects = dict()
class HydraNetworkTree(object):
def __init__(self, url=None, username=None, password=None):
self.conn = JsonConnection(url)
if username is not None and password is not None:
self.session_id = self.conn.login(username=username,
password=password)
else:
self.session_id = self.conn.login()
self.projects = dict()
def get_tree(self):
project_list = self.conn.call('get_projects', {})
for project in project_list:
networks = self.conn.call('get_networks',
{'project_id': project['id']})
project['networks'] = networks
self.projects.update({project['id']: project})
def print_tree(self, color=True):
if color:
pr_col = '\033[1m'
ne_col = '\033[92m'
sc_col = '\033[94m'
endtag = '\033[0m'
else:
pr_col = ''
ne_col = ''
sc_col = ''
endtag = ''
for pid, project in self.projects.iteritems():
print('%sP %3d %s%s' %
(pr_col, pid, project['name'], endtag))
for network in project['networks']:
print(u'%sN%s \u2514\u2500%s%2d %s%s' %
(ne_col, endtag, ne_col, network['id'], network['name'],
endtag))
for scenario in network['scenarios']:
print(u'%sS%s \u2514\u2500%s%2d %s%s' %
(sc_col, endtag, sc_col, scenario['id'],
scenario['name'], endtag))
def __init__(self,
vars,
objs,
actual_time_steps,
url=None,
session_id=None):
self.vars = vars
self.objs = objs
self.actual_time_steps = actual_time_steps
self.network_id = None
self.scenario_id = None
self.network = None
self.res_scenario = None
self.connection = JsonConnection(url)
self.attrs = dict()
self.time_axis = dict()
if session_id is not None:
log.info("Using existing session %s", session_id)
self.connection.session_id = session_id
else:
self.connection.login()
def __init__(self, url=None, session_id=None):
#Record the names of the files created by the plugin so we can
#display them to the user.
self.files = []
#A mapping from attr ID to attr object. Makes searching for attributes
#easier
self.attr_id_map = {}
self.connection = JsonConnection(url)
write_output("Connecting...")
if session_id is not None:
write_output("Using existing session %s" % session_id)
self.connection.session_id = session_id
else:
self.connection.login()
self.network = None
self.num_steps = 3
def __init__(self, vars, objs, actual_time_steps, url=None, session_id=None):
self.vars=vars
self.objs=objs
self.actual_time_steps=actual_time_steps
self.network_id = None
self.scenario_id = None
self.network = None
self.res_scenario=None
self.connection = JsonConnection(url)
self.attrs = dict()
self.time_axis = dict()
if session_id is not None:
log.info("Using existing session %s", session_id)
self.connection.session_id = session_id
else:
self.connection.login()
def __init__(self, url=None, username=None, password=None):
super(HydraNetwork, self).__init__()
self.conn = JsonConnection(url=url, app_name='ShapefileApp')
self.url = url
self.username = username
self.password = password
self.session_id = None
self.project = None
self.hydra_network = None
self.hydra_scenario = None
self.hydra_attributes = None
self.attrs = dict()
self.attr_ids = dict()
self.epsg = None
self.nodes = dict()
self.links = []
self.link_names = dict()
self.node_names = dict()
self._node_coord_index = dict()
class Importer(object):
def __init__(self,
vars,
objs,
actual_time_steps,
url=None,
session_id=None):
self.vars = vars
self.objs = objs
self.actual_time_steps = actual_time_steps
self.network_id = None
self.scenario_id = None
self.network = None
self.res_scenario = None
self.connection = JsonConnection(url)
self.attrs = dict()
self.time_axis = dict()
if session_id is not None:
log.info("Using existing session %s", session_id)
self.connection.session_id = session_id
else:
self.connection.login()
def load_network(self, network_id, scenario_id):
"""
Load network and scenario from the server.
"""
try:
network_id = int(network_id)
except (TypeError, ValueError):
network_id = self.network_id
if network_id is None:
raise HydraPluginError("No network specified.")
try:
scenario_id = int(scenario_id)
except (TypeError, ValueError):
scenario_id = self.scenario_id
if scenario_id is None:
raise HydraPluginError("No scenario specified.")
self.network = self.connection.call(
'get_network', {
'network_id': int(network_id),
'include_data': 'Y',
'scenario_ids': [int(scenario_id)],
'template_id': None
})
self.res_scenario = self.network.scenarios[0].resourcescenarios
attrslist = self.connection.call('get_all_attributes', {})
for attr in attrslist:
self.attrs.update({attr.id: attr.name})
#####################################################
def set_network(self, network):
"""
Load network and scenario from the server.
"""
self.network = network
#self.res_scenario = self.network.scenarios[0].resourcescenarios
attrslist = self.connection.call('get_attributes', {})
for attr in attrslist:
self.attrs.update({attr.id: attr.name})
#####################################################
def create_timeseries(self, data):
timeseries = {'0': {}}
counter = 0
for time_s in self.actual_time_steps:
timeseries['0'][time_s] = json.dumps(data[counter])
counter += 1
return json.dumps(timeseries)
'''
def create_timeseries(self, data):
timeseries = {'0': {}}
index=0
for i, idx in enumerate(index):
timeseries['0'][self.time_axis[int(idx)]] = float(data[i])
index+=1
return json.dumps(timeseries)
'''
def create_scalar(self, value):
return json.dumps(value)
def create_array(self, index, data):
pass
def create_descriptor(self, value):
descriptor = json.dumps(value)
return descriptor
def save(self):
self.network.scenarios[0].resourcescenarios = self.res_scenario
self.connection.call('update_scenario',
{'scen': self.network.scenarios[0]})
def import_res(self):
self.import_vars()
self.import_objs()
def import_vars(self):
'''
import varaible to Hydra
'''
nodes = dict()
for node in self.network.nodes:
nodes.update({node.id: node.name})
for var in self.vars.keys():
for varModel in self.vars[var]:
for link in self.network.links:
#print "Owner: ", type(varModel.owner)
fromnode = nodes[link.node_1_id]
tonode = nodes[link.node_2_id]
if len(varModel.owner) == 2 and fromnode == varModel.owner[
0] and tonode == varModel.owner[1]:
# "It is here, link Var: ",var,": ",varModel.owner, ": ", varModel.data_set
for attr in link.attributes:
if self.attrs[attr.attr_id] == varModel.name:
if attr.attr_is_var == 'Y':
dataset = dict(name = 'Pyomo import - ' + link.name + ' ' \
+ varModel.name)
dataset['unit'] = varModel.unit
if len(varModel.data_set) > 1:
dataset['type'] = 'timeseries'
dataset[
'value'] = self.create_timeseries(
varModel.data_set)
elif len(varModel.data_set) == 1:
try:
data = float(varModel.data_set[0])
dataset['type'] = 'scalar'
dataset['value'] = \
self.create_scalar(data)
except ValueError:
dataset['type'] = 'descriptor'
dataset[
'value'] = self.create_descriptor(
varModel.data_set[0])
#print "link attr is added"
metadata = {}
dataset['metadata'] = json.dumps(metadata)
dataset[
'dimension'] = attr.resourcescenario.value.dimension
res_scen = dict(resource_attr_id=attr.id,
attr_id=attr.attr_id,
value=dataset)
self.res_scenario.append(
res_scen) #dataset
for node in self.network.nodes:
if node.name in varModel.owner and len(
varModel.owner) == 1:
# "node Var: ", var, ": ", varModel.owner,": ", varModel.data_set
for attr in node.attributes:
if self.attrs[attr.attr_id] == varModel.name:
if attr.attr_is_var == 'Y':
dataset = dict(name = 'Pyomo import - ' + node.name + ' ' \
+ varModel.name)
dataset['unit'] = varModel.unit
if len(varModel.data_set) > 1:
dataset['type'] = 'timeseries'
dataset[
'value'] = self.create_timeseries(
varModel.data_set)
elif len(varModel.data_set) == 1:
try:
data = float(varModel.data_set[0])
dataset['type'] = 'scalar'
dataset['value'] = \
self.create_scalar(data)
except ValueError:
dataset['type'] = 'descriptor'
dataset[
'value'] = self.create_descriptor(
varModel.data_set[0])
#print "node att is added"
'''
'''
metadata = {}
dataset['metadata'] = json.dumps(metadata)
dataset[
'dimension'] = attr.resourcescenario.value.dimension
res_scen = dict(resource_attr_id=attr.id,
attr_id=attr.attr_id,
value=dataset)
self.res_scenario.append(res_scen)
# "Attr is added ...."
def import_objs(self):
'''
import objects values to Hydra
'''
for var in self.objs.keys():
for varModel in self.objs[var]:
for attr in self.network.attributes:
if attr.attr_is_var == 'Y':
if self.attrs[attr.attr_id] == varModel.name:
dataset = dict(name = 'Pyomo import - ' + self.network.name + ' ' \
+ varModel.name)
dataset['unit'] = varModel.unit
if len(varModel.data_set) > 1:
dataset['type'] = 'timeseries'
dataset['value'] = self.create_timeseries(
varModel.data_set)
elif len(varModel.data_set) == 1:
try:
data = float(varModel.data_set[0])
dataset['type'] = 'scalar'
dataset['value'] = \
self.create_scalar(data)
except ValueError:
dataset['type'] = 'descriptor'
dataset['value'] = self.create_descriptor(
varModel.data_set[0])
'''
'''
res_scen = dict(resource_attr_id=attr.id,
attr_id=attr.attr_id,
value=dataset)
metadata = {}
dataset['metadata'] = json.dumps(metadata)
dataset[
'dimension'] = attr.resourcescenario.value.dimension
self.res_scenario.append(res_scen)
class ModelRunner(object):
"""
Exporter of Hydra networks to JSON or XML files.
"""
def __init__(self, url=None, session_id=None):
#Record the names of the files created by the plugin so we can
#display them to the user.
self.files = []
#A mapping from attr ID to attr object. Makes searching for attributes
#easier
self.attr_id_map = {}
self.connection = JsonConnection(url)
write_output("Connecting...")
if session_id is not None:
write_output("Using existing session %s" % session_id)
self.connection.session_id = session_id
else:
self.connection.login()
self.network = None
self.num_steps = 3
def get_attributes(self, template_id):
if template_id is not None:
#TODO: Find out why get_all_template_attributes isnt here.
attributes = self.connection.call('get_all_attributes',
{'template_id': template_id})
else:
attributes = self.connection.call('get_all_attributes', {})
attr_id_map = {}
for a in attributes:
attr_id_map[a.id] = a
self.attr_id_map = attr_id_map
def get_network_data(self, network_id, scenario_id):
"""
Retrieve the network, identify the parameters to set,
set them and run the model. Then identify the results
and set them back on the network.
"""
write_output("Retrieving Network")
write_progress(2, self.num_steps)
if network_id is not None:
if scenario_id is None:
raise HydraPluginError("A scenario ID must be specified.")
#The network ID can be specified to get the network...
try:
network_id = int(network_id)
network = self.connection.call(
'get_network', {
'network_id': network_id,
'scenario_ids': [int(scenario_id)]
})
write_output("Network retrieved")
except Exception, e:
log.exception(e)
raise HydraPluginError("Network %s not found." % network_id)
else:
class HobbesImporter(object):
"""
Importer of JSON files into Hydra. Also accepts XML files.
"""
def __init__(self, url=None, session_id=None):
self.json_net = None
self.warnings = []
self.files = []
self.template = None
self.attributes = []
self.attr_name_map = {}
self.nodes = {}
self.links = {}
self.groups = {}
self.node_id = PluginLib.temp_ids()
self.link_id = PluginLib.temp_ids()
self.group_id = PluginLib.temp_ids()
self.connection = JsonConnection(url)
if session_id is not None:
write_output("Using existing session %s" % session_id)
self.connection.session_id = session_id
else:
self.connection.login()
#3 steps: start, read, save
self.num_steps = 3
def fetch_project(self, project_id):
"""
If a project ID is not specified, a new one
must be created to hold the incoming network.
If an ID is specified, we must retrieve the project to make sure
it exists. If it does not exist, then throw an error.
Returns the project object so that the network can access it's ID.
"""
if project_id is not None:
try:
project = self.connection.call('get_project',
{'project_id': int(project_id)})
log.info('Loading existing project (ID=%s)' % project_id)
return project
except RequestError:
raise HydraPluginError("Project with ID %s not found" %
project_id)
#Using 'datetime.now()' in the name guarantees a unique project name.
new_project = dict(
name = "Hobbes Project created at %s" % (datetime.now()),
description = \
"Default project created by the %s plug-in." % \
(self.__class__.__name__),
)
saved_project = self.connection.call('add_project',
{'project': new_project})
return saved_project
def fetch_template(self, template_id):
self.template = self.connection.call('get_template',
{'template_id': int(template_id)})
self.attributes = self.connection.call(
'get_template_attributes', {'template_id': int(template_id)})
#Build a lookup dict of attributes by name
for a in self.attributes:
self.attr_name_map[a.name] = a
def fetch_remote_network(self):
"""
Request the hobbes network from the hobbes server
"""
write_output("Fetching Network")
write_progress(2, self.num_steps)
net_response = requests.get(
"http://cwn.casil.ucdavis.edu/network/get") #JSON Network
#http://cwn.casil.ucdavis.edu/excel/create?prmname=SR_CLE #XLS
if net_response.status_code != 200:
raise HydraPluginError(
"A connection error has occurred with status code: %s" %
net_response.status_code)
self.json_net = json.loads(net_response.content)
def upload_template(self):
"""
Upload the template file found in ./template.xml
"""
file_ = os.path.join(__location__, '../', 'template', 'template.xml')
with open(file_) as f:
xml_template = f.read()
self.template = self.connection.call('upload_template_xml',
{'template_xml': xml_template})
self.attributes = self.connection.call(
'get_template_attributes', {'template_id': self.template.id})
#Build a lookup dict of attributes by name
for a in self.attributes:
self.attr_name_map[a.name] = a
def import_network_topology(self, project_id=None):
"""
Read the file containing the network data and send it to
the server.
"""
if self.json_net is None:
self.fetch_remote_network()
for node in self.json_net:
props = node['properties']
node_type = props['type']
node_coords = node['geometry']['coordinates']
tmp_node_id = self.node_id.next()
#TODO: HACK. WHy are there 2 coordinates for the node?
if isinstance(node_coords[0], list):
#log.debug("Using 1st coords of %s (%s)", node_coords, props['type'])
node_coords = node_coords[0]
#log.info("X=%s, y=%s",node_coords[0], node_coords[1])
node = dict(
id=tmp_node_id,
name=props['prmname'],
x=str(
node_coords[0]), #swap these if they are lat-long, not x-y
y=str(node_coords[1]),
description=props['description'],
attributes=[],
types=[{
'template_id': self.template.id,
'id': node_type
}])
self.nodes[props['prmname']] = node
#ATTRIBUTES
#Find the matching type
for t in self.template.types:
if t.name == node_type:
node['types'][0]['id'] = t.id
#Assign the attributes to the node
for tattr in t.typeattrs:
node['attributes'].append({'attr_id': tattr.attr_id})
inlinks = [o['link_prmname'] for o in props.get('origins', [])]
for linkname in inlinks:
if linkname not in self.links:
link = dict(
id=self.link_id.next(),
name=linkname,
node_2_id=tmp_node_id,
attributes=[],
description="",
)
self.links[linkname] = link
else:
link = self.links[linkname]
link['node_2_id'] = tmp_node_id
outlinks = [o['link_prmname'] for o in props.get('terminals', [])]
for linkname in outlinks:
if linkname not in self.links:
link = dict(
id=self.link_id.next(),
name=linkname,
node_1_id=tmp_node_id,
attributes=[],
description="",
)
self.links[linkname] = link
else:
link = self.links[linkname]
link['node_1_id'] = tmp_node_id
node_groups = props['regions']
project = self.fetch_project(project_id)
project_id = project.id
write_output("Saving Network")
write_progress(3, self.num_steps)
for t in self.template.types:
if t.name == 'HobbesNetwork':
network_type = t.id
hydra_network = {
'name': "HOBBES Network (%s)" % datetime.now(),
'description':
"Hobbes Network, imported directly from the web API",
'nodes': self.nodes.values(),
'links': self.links.values(),
'project_id': project_id,
'projection': 'EPSG:2229',
'groups': [],
'scenarios': [],
'types': [{
'template_id': self.template.id,
'id': network_type
}]
}
#The network ID can be specified to get the network...
self.network = self.connection.call('add_network',
{'net': hydra_network})
return self.network
def make_repo_dataset(self, json_repo):
meta = {}
for k, v in json_repo.items():
meta[k] = str(v)
dataset = {
'dimension': 'dimensionless',
'unit': None,
'type': 'descriptor',
'value': json_repo['tag'],
'metadata': json.dumps(meta),
'name': 'repo',
}
return dataset
def import_data(self, include_timeseries=True):
scenario = {
"name": "Hobbes Import",
"description": "Import from hobbes",
}
#List of parameters to ignore
non_attributes = set([
'origins', 'prmname', 'regions', 'terminals', 'description',
'extras', 'type', 'repo', 'origin'
])
#Make a map from node name to a list of attributes. Do this by first
#making a node id map
node_name_id_map = {}
for n in self.network.nodes:
node_name_id_map[n.name] = n.id
node_attributes = self.connection.call('get_all_node_attributes',
{'network_id': self.network.id})
node_id_attr_map = {}
for a in node_attributes:
n_attrs = node_id_attr_map.get(a.ref_id, [])
n_attrs.append(a)
node_id_attr_map[a.ref_id] = n_attrs
resource_scenarios = []
#request data for first 2 nodes.
for node in self.json_net[:10]:
props = node['properties']
name = props['prmname']
node_id = node_name_id_map[name]
#repo is a special case
repo = self.make_repo_dataset(props['repo'])
repo_attr_id = self.attr_name_map['repo'].id
ra_id = None
for a in node_id_attr_map[node_id]:
if a.attr_id == repo_attr_id:
ra_id = a.id
break
repo_rs = dict(
resource_attr_id=ra_id,
attr_id=repo_attr_id,
is_var='N',
value=repo,
)
resource_scenarios.append(repo_rs)
for k, v in props.items():
if k not in non_attributes:
if isinstance(v, float):
attr_id = self.attr_name_map[k].id
dataset = dict(
name=k,
value=str(v),
type='scalar',
dimension='dimensionless',
unit=None,
)
ra_id = None
for a in node_id_attr_map[node_id]:
if a.attr_id == attr_id:
ra_id = a.id
break
resource_scenario = dict(
resource_attr_id=ra_id,
attr_id=attr_id,
is_var='N',
value=dataset,
)
resource_scenarios.append(resource_scenario)
#timeseries, requested from the hobbes server
if include_timeseries is True:
extras = props.get('extras', [])
if extras is not None and len(extras) > 0:
attr_response = requests.get(
"http://cwn.casil.ucdavis.edu/network/extras?prmname=%s"
% props['prmname']) #JSON attributes
else:
continue
if attr_response.status_code != 200:
raise HydraPluginError(
"A connection error has occurred with status code: %s"
% attr_response.status_code)
extra_data = json.loads(attr_response.content)
non_attrs = ['prmname', 'readme']
for k, v in extra_data.items():
if k in non_attrs:
continue
else:
if len(v) < 2:
continue
ts = self.parse_timeseries(v)
attr_id = self.attr_name_map[k].id
dataset = dict(
name=k,
value=json.dumps(ts),
type='timeseries',
dimension='dimensionless',
unit=None,
)
ra_id = None
for a in node_id_attr_map[node_id]:
if a.attr_id == attr_id:
ra_id = a.id
break
resource_scenario = dict(
resource_attr_id=ra_id,
attr_id=attr_id,
is_var='N',
value=dataset,
)
resource_scenarios.append(resource_scenario)
scenario['resourcescenarios'] = resource_scenarios
new_scenario = self.connection.call('add_scenario', {
'network_id': self.network.id,
'scen': scenario
})
self.scenario = new_scenario
return new_scenario
def parse_timeseries(self, timeseries_value):
"""
Convert a hobbes timeseries to a hydra timeseries
"""
timeformat = config.get('DEFAULT', 'datetime_format')
val = {}
for timeval in timeseries_value[1:]:
time = timeval[0]
split = time.split('-')
d = datetime(year=int(split[0]),
month=int(split[1]),
day=int(split[2]))
tstime = datetime.strftime(d, timeformat)
val[tstime] = float(timeval[1])
return {"idx1": val}
class HobbesImporter(object):
"""
Importer of JSON files into Hydra. Also accepts XML files.
"""
def __init__(self, url=None, session_id=None):
self.json_net = None
self.warnings = []
self.files = []
self.template = None
self.attributes = []
self.attr_name_map = {}
self.nodes = {}
self.links = {}
self.groups = {}
self.node_id = PluginLib.temp_ids()
self.link_id = PluginLib.temp_ids()
self.group_id = PluginLib.temp_ids()
self.connection = JsonConnection(url)
if session_id is not None:
write_output("Using existing session %s"% session_id)
self.connection.session_id=session_id
else:
self.connection.login()
#3 steps: start, read, save
self.num_steps = 3
def fetch_project(self, project_id):
"""
If a project ID is not specified, a new one
must be created to hold the incoming network.
If an ID is specified, we must retrieve the project to make sure
it exists. If it does not exist, then throw an error.
Returns the project object so that the network can access it's ID.
"""
if project_id is not None:
try:
project = self.connection.call('get_project', {'project_id':int(project_id)})
log.info('Loading existing project (ID=%s)' % project_id)
return project
except RequestError:
raise HydraPluginError("Project with ID %s not found"%project_id)
#Using 'datetime.now()' in the name guarantees a unique project name.
new_project = dict(
name = "Hobbes Project created at %s" % (datetime.now()),
description = \
"Default project created by the %s plug-in." % \
(self.__class__.__name__),
)
saved_project = self.connection.call('add_project', {'project':new_project})
return saved_project
def fetch_template(self, template_id):
self.template = self.connection.call('get_template',
{'template_id':int(template_id)})
self.attributes = self.connection.call('get_template_attributes',
{'template_id':int(template_id)})
#Build a lookup dict of attributes by name
for a in self.attributes:
self.attr_name_map[a.name] = a
def fetch_remote_network(self):
"""
Request the hobbes network from the hobbes server
"""
write_output("Fetching Network")
write_progress(2, self.num_steps)
net_response = requests.get("http://cwn.casil.ucdavis.edu/network/get") #JSON Network
#http://cwn.casil.ucdavis.edu/excel/create?prmname=SR_CLE #XLS
if net_response.status_code != 200:
raise HydraPluginError("A connection error has occurred with status code: %s"%net_response.status_code)
self.json_net = json.loads(net_response.content)
def upload_template(self):
"""
Upload the template file found in ./template.xml
"""
file_ = os.path.join(__location__, '../', 'template', 'template.xml')
with open(file_) as f:
xml_template = f.read()
self.template = self.connection.call('upload_template_xml',
{'template_xml':xml_template})
self.attributes = self.connection.call('get_template_attributes',
{'template_id':self.template.id})
#Build a lookup dict of attributes by name
for a in self.attributes:
self.attr_name_map[a.name] = a
def import_network_topology(self, project_id=None):
"""
Read the file containing the network data and send it to
the server.
"""
if self.json_net is None:
self.fetch_remote_network()
for node in self.json_net:
props = node['properties']
node_type = props['type']
node_coords = node['geometry']['coordinates']
tmp_node_id = self.node_id.next()
#TODO: HACK. WHy are there 2 coordinates for the node?
if isinstance(node_coords[0], list):
#log.debug("Using 1st coords of %s (%s)", node_coords, props['type'])
node_coords = node_coords[0]
#log.info("X=%s, y=%s",node_coords[0], node_coords[1])
node = dict(
id = tmp_node_id,
name = props['prmname'],
x = str(node_coords[0]), #swap these if they are lat-long, not x-y
y = str(node_coords[1]),
description = props['description'],
attributes = [],
types = [{'template_id':self.template.id,
'id':node_type}]
)
self.nodes[props['prmname']] = node
#ATTRIBUTES
#Find the matching type
for t in self.template.types:
if t.name == node_type:
node['types'][0]['id'] = t.id
#Assign the attributes to the node
for tattr in t.typeattrs:
node['attributes'].append(
{'attr_id':tattr.attr_id}
)
inlinks = [o['link_prmname'] for o in props.get('origins', [])]
for linkname in inlinks:
if linkname not in self.links:
link = dict(
id=self.link_id.next(),
name = linkname,
node_2_id = tmp_node_id,
attributes = [],
description = "",
)
self.links[linkname] = link
else:
link = self.links[linkname]
link['node_2_id'] = tmp_node_id
outlinks = [o['link_prmname'] for o in props.get('terminals', [])]
for linkname in outlinks:
if linkname not in self.links:
link = dict(
id=self.link_id.next(),
name = linkname,
node_1_id = tmp_node_id,
attributes = [],
description = "",
)
self.links[linkname] = link
else:
link = self.links[linkname]
link['node_1_id'] = tmp_node_id
node_groups = props['regions']
project = self.fetch_project(project_id)
project_id = project.id
write_output("Saving Network")
write_progress(3, self.num_steps)
for t in self.template.types:
if t.name == 'HobbesNetwork':
network_type = t.id
hydra_network = {
'name' : "HOBBES Network (%s)"%datetime.now(),
'description' : "Hobbes Network, imported directly from the web API",
'nodes': self.nodes.values(),
'links': self.links.values(),
'project_id' : project_id,
'projection':'EPSG:2229',
'groups' : [],
'scenarios': [],
'types' : [{'template_id':self.template.id,
'id':network_type}]
}
#The network ID can be specified to get the network...
self.network = self.connection.call('add_network', {'net':hydra_network})
return self.network
def make_repo_dataset(self, json_repo):
meta = {}
for k, v in json_repo.items():
meta[k] = str(v)
dataset = {
'dimension': 'dimensionless',
'unit' : None,
'type' : 'descriptor',
'value' : json_repo['tag'],
'metadata' : json.dumps(meta),
'name' : 'repo',
}
return dataset
def import_data(self, include_timeseries=True):
scenario = {
"name" : "Hobbes Import",
"description" : "Import from hobbes",
}
#List of parameters to ignore
non_attributes = set(['origins', 'prmname', 'regions', 'terminals', 'description', 'extras', 'type', 'repo', 'origin'])
#Make a map from node name to a list of attributes. Do this by first
#making a node id map
node_name_id_map = {}
for n in self.network.nodes:
node_name_id_map[n.name] = n.id
node_attributes = self.connection.call('get_all_node_attributes', {'network_id':self.network.id})
node_id_attr_map = {}
for a in node_attributes:
n_attrs = node_id_attr_map.get(a.ref_id, [])
n_attrs.append(a)
node_id_attr_map[a.ref_id] = n_attrs
resource_scenarios = []
#request data for first 2 nodes.
for node in self.json_net[:10]:
props = node['properties']
name = props['prmname']
node_id = node_name_id_map[name]
#repo is a special case
repo = self.make_repo_dataset(props['repo'])
repo_attr_id = self.attr_name_map['repo'].id
ra_id = None
for a in node_id_attr_map[node_id]:
if a.attr_id == repo_attr_id:
ra_id = a.id
break
repo_rs = dict(
resource_attr_id = ra_id,
attr_id = repo_attr_id,
is_var = 'N',
value = repo,
)
resource_scenarios.append(repo_rs)
for k, v in props.items():
if k not in non_attributes:
if isinstance(v, float):
attr_id = self.attr_name_map[k].id
dataset = dict(
name = k,
value = str(v),
type = 'scalar',
dimension = 'dimensionless',
unit = None,
)
ra_id = None
for a in node_id_attr_map[node_id]:
if a.attr_id == attr_id:
ra_id = a.id
break
resource_scenario = dict(
resource_attr_id = ra_id,
attr_id = attr_id,
is_var = 'N',
value = dataset,
)
resource_scenarios.append(resource_scenario)
#timeseries, requested from the hobbes server
if include_timeseries is True:
extras = props.get('extras', [])
if extras is not None and len(extras) > 0:
attr_response = requests.get("http://cwn.casil.ucdavis.edu/network/extras?prmname=%s"%props['prmname']) #JSON attributes
else:
continue
if attr_response.status_code != 200:
raise HydraPluginError("A connection error has occurred with status code: %s"%attr_response.status_code)
extra_data = json.loads(attr_response.content)
non_attrs = ['prmname', 'readme']
for k, v in extra_data.items():
if k in non_attrs:
continue
else:
if len(v) < 2:
continue
ts = self.parse_timeseries(v)
attr_id = self.attr_name_map[k].id
dataset = dict(
name = k,
value = json.dumps(ts),
type = 'timeseries',
dimension = 'dimensionless',
unit = None,
)
ra_id = None
for a in node_id_attr_map[node_id]:
if a.attr_id == attr_id:
ra_id = a.id
break
resource_scenario = dict(
resource_attr_id = ra_id,
attr_id = attr_id,
is_var = 'N',
value = dataset,
)
resource_scenarios.append(resource_scenario)
scenario['resourcescenarios'] = resource_scenarios
new_scenario = self.connection.call('add_scenario',
{'network_id':self.network.id,
'scen':scenario
})
self.scenario = new_scenario
return new_scenario
def parse_timeseries(self, timeseries_value):
"""
Convert a hobbes timeseries to a hydra timeseries
"""
timeformat = config.get('DEFAULT', 'datetime_format')
val = {}
for timeval in timeseries_value[1:]:
time = timeval[0]
split = time.split('-')
d = datetime(year=int(split[0]), month=int(split[1]), day=int(split[2]))
tstime = datetime.strftime(d, timeformat)
val[tstime] = float(timeval[1])
return {"idx1": val}
class HydraNetwork(HydraResource):
def __init__(self, url=None, username=None, password=None):
super(HydraNetwork, self).__init__()
self.conn = JsonConnection(url=url, app_name='ShapefileApp')
self.url = url
self.username = username
self.password = password
self.session_id = None
self.project = None
self.hydra_network = None
self.hydra_scenario = None
self.hydra_attributes = None
self.attrs = dict()
self.attr_ids = dict()
self.epsg = None
self.nodes = dict()
self.links = []
self.link_names = dict()
self.node_names = dict()
self._node_coord_index = dict()
def login(self):
if self.username is not None and self.password is not None:
self.session_id = self.conn.login(username=self.username,
password=self.password)
else:
self.session_id = self.conn.login()
def load_attributes(self):
self.hydra_attributes = self.conn.call('get_all_attributes', {})
for attr in self.hydra_attributes:
self.attrs[attr.id] = attr
self.attr_ids[attr.name] = attr.id
def load_network(self, network_id, scenario_id):
"""Load a network from HydraPlatform.
"""
if self.project is None:
self.load_project(network_id=network_id)
self.hydra_network = self.conn.call('get_network',
{'network_id': network_id,
'include_data': 'Y'})
self.load_attributes()
for scenario in self.hydra_network['scenarios']:
if scenario['id'] == scenario_id:
self.hydra_scenario = scenario
break
res_scen_dict = dict()
for res_scen in self.hydra_scenario['resourcescenarios']:
res_scen_dict.update({res_scen['resource_attr_id']: res_scen})
# Create node index
node_dict = dict()
for node in self.hydra_network['nodes']:
node_dict.update({node['id']: node})
self.name = self.hydra_network['name']
self.description = self.hydra_network['description']
if 'projection' in self.hydra_network.keys():
try:
self.epsg = \
int(self.hydra_network['projection'].split(':')[1])
except ValueError:
warnings.warn('Could not load EPSG code.')
# Add network attributes
for res_attr in self.hydra_network['attributes']:
if res_scen_dict.get(res_attr['id']) is not None:
res_scen = res_scen_dict[res_attr['id']]
self.add_attribute(self.attrs[res_attr['attr_id']],
res_attr, res_scen)
else:
self.add_attribute(self.attrs[res_attr['attr_id']],
res_attr, None)
# Add nodes and attributes
for node in self.hydra_network['nodes']:
n_node = HydraNode(x=float(node['x']), y=float(node['y']))
n_node.name = node['name']
n_node.layout = node['layout']
n_node.id = node['id']
n_node.types = node['types']
for res_attr in node['attributes']:
if res_scen_dict.get(res_attr['id']) is not None:
res_scen = res_scen_dict[res_attr['id']]
n_node.add_attribute(self.attrs[res_attr['attr_id']],
res_attr, res_scen)
else:
n_node.add_attribute(self.attrs[res_attr['attr_id']],
res_attr, None)
self.add_node(n_node)
# Add segments and attributes
for link in self.hydra_network['links']:
n_link = HydraLink(start_node=self.nodes[link['node_1_id']],
end_node=self.nodes[link['node_2_id']])
n_link.name = link['name']
n_link.layout = link['layout']
n_link.id = link['id']
n_link.types = link['types']
for res_attr in link['attributes']:
if res_scen_dict.get(res_attr['id']) is not None:
res_scen = res_scen_dict[res_attr['id']]
n_link.add_attribute(self.attrs[res_attr['attr_id']],
res_attr, res_scen)
else:
n_link.add_attribute(self.attrs[res_attr['attr_id']],
res_attr, None)
self.add_link(n_link)
def load_project(self, project_id=None, network_id=None):
"""Load a project by its ID or by a network ID.
"""
if project_id is not None:
self.project = self.conn.call('get_project',
{'project_id': project_id})
elif network_id is not None:
self.project = self.conn.call('get_network_project',
{'network_id': network_id})
def create_project(self, name=None):
self.project = dict()
if name is not None:
self.project['name'] = name
else:
self.project['name'] = "Shapefile import @ %s" % datetime.now()
self.project['description'] = ''
self.project['status'] = 'A'
self.project = self.conn.call('add_project',
{'project': self.project})
def add_node(self, node):
if self.node_names.get(node.name.lower()) is not None:
add_string = ' (%s)' % self.node_names[node.name.lower()]
self.node_names[node.name.lower()] += 1
node.name += add_string
else:
self.node_names[node.name.lower()] = 1
self.nodes[node.id] = node
self._node_coord_index[(node.x, node.y)] = node.id
def add_link(self, link):
if self.link_names.get(link.name.lower()) is not None:
add_string = ' (%s)' % self.link_names[link.name.lower()]
self.link_names[link.name.lower()] += 1
link.name += add_string
else:
self.link_names[link.name.lower()] = 1
self.links.append(link)
def save_network(self, network_name=None, project_name=None):
"""Save the network to HydraPlatform server.
"""
if self.project is None:
self.create_project(name=project_name)
self.hydra_network = dict()
self.hydra_scenario = dict()
self.hydra_scenario['name'] = 'Scenario created by ShapefileApp'
self.hydra_scenario['description'] = \
'Standard scenario created by ShapefileApp.'
self.hydra_scenario['id'] = -1
self.hydra_scenario['resourcescenarios'] = []
if network_name is not None:
self.hydra_network['name'] = network_name
else:
self.hydra_network['name'] = \
"Network imported from shapefile (%s)." % self.name
self.hydra_network['description'] = \
"Network imported from %s" % self.name
if self.epsg is not None:
self.hydra_network['projection'] = \
'EPSG:%s' % self.epsg
self.hydra_network['nodes'] = []
self.hydra_network['links'] = []
self.hydra_network['attributes'] = []
self.hydra_network['scenarios'] = []
self.hydra_network['project_id'] = self.project['id']
for node in self.nodes.values():
hydra_node = self.create_hydra_node(node)
self.hydra_network['nodes'].append(hydra_node)
for link in self.links:
hydra_link = self.create_hydra_link(link)
self.hydra_network['links'].append(hydra_link)
self.hydra_network['scenarios'].append(self.hydra_scenario)
net_summary = self.conn.call('add_network',
{'net': self.hydra_network})
return net_summary
def create_hydra_node(self, node):
"""Build a node dict from a HydraNode object.
"""
hydra_node = dict()
hydra_node['id'] = node.id
hydra_node['name'] = node.name
hydra_node['description'] = ''
hydra_node['attributes'] = []
hydra_node['x'] = repr(node.x)
hydra_node['y'] = repr(node.y)
#TODO: Add attributes
return hydra_node
def create_hydra_link(self, link):
"""Build a link dict from a HydraLink object.
"""
hydra_link = dict()
#TODO: Finalise this function
return hydra_link
# import
import pandas
from pandas.io.json import json_normalize
import sys
from os.path import expanduser
from os.path import join
import json
from HydraLib.PluginLib import JsonConnection
url = 'www.openaguadss.org'
conn = JsonConnection(url=url)
conn.login(username = '******', password = '')
def get_project_by_name(conn, project_name):
return conn.call('get_project_by_name',
{'project_name':project_name})
def get_network_by_name(conn, project_id, network_name):
return conn.call('get_network_by_name',
{'project_id':project_id, 'network_name':network_name})
# get project ID
project_name = 'Water Allocation Demo'
project = get_project_by_name(conn, project_name)
project_id = project.id
# get network ID
network_name = 'Water Allocation Network'
network = get_network_by_name(conn, project_id, network_name)
network_id = network.id
class NetworkImporter(object):
"""
Exporter of Hydra networks to JSON or XML files.
"""
def __init__(self, url=None, session_id=None):
#Record the names of the files created by the plugin so we can
#display them to the user.
self.files = []
#A mapping from attr ID to attr object. Makes searching for attributes
#easier
self.attr_id_map = {}
self.connection = JsonConnection(url)
write_output("Connecting...")
if session_id is not None:
write_output("Using existing session %s"% session_id)
self.connection.session_id=session_id
else:
self.connection.login()
#Dictionaries, keyed on their name, where all the nodes, links & groups will go
self.hydra_nodes = {}
self.hydra_links = {}
self.hydra_groups = {}
self.hydra_group_members = {}
#Generators of NEGATIVE IDS for the nodes & links.
#Negative IDS are used as temporary client-side IDS until the network
#is saved, at which point the resources will be given permanent, positive IDS.
self.node_ids = temp_ids()
self.link_ids = temp_ids()
self.group_ids = temp_ids() # A group is an institution.
#A mapping from the name of a type to the type itself.
self.type_name_map = {}
self.num_steps = 3
def fetch_project(self, project_id):
"""
If a project ID is not specified, a new one
must be created to hold the incoming network.
If an ID is specified, we must retrieve the project to make sure
it exists. If it does not exist, then throw an error.
Returns the project object so that the network can access it's ID.
"""
if project_id is not None:
try:
project = self.connection.call('get_project', {'project_id':project_id})
log.info('Loading existing project (ID=%s)' % project_id)
return project
except RequestError, e:
log.exception(e)
raise HydraPluginError("An error occurred retrieving project ID %s"%project_id)
#Using 'datetime.now()' in the name guarantees a unique project name.
new_project = dict(
name = "Hobbes Project created at %s" % (datetime.now()),
description = \
"Default project created by the %s plug-in." % \
(self.__class__.__name__),
)
saved_project = self.connection.call('add_project', {'project':new_project})
return saved_project
class Importer (object):
def __init__(self, vars, objs, actual_time_steps, url=None, session_id=None):
self.vars=vars
self.objs=objs
self.actual_time_steps=actual_time_steps
self.network_id = None
self.scenario_id = None
self.network = None
self.res_scenario=None
self.connection = JsonConnection(url)
self.attrs = dict()
self.time_axis = dict()
if session_id is not None:
log.info("Using existing session %s", session_id)
self.connection.session_id = session_id
else:
self.connection.login()
def load_network(self, network_id, scenario_id):
"""
Load network and scenario from the server.
"""
try:
network_id = int(network_id)
except (TypeError, ValueError):
network_id = self.network_id
if network_id is None:
raise HydraPluginError("No network specified.")
try:
scenario_id = int(scenario_id)
except (TypeError, ValueError):
scenario_id = self.scenario_id
if scenario_id is None:
raise HydraPluginError("No scenario specified.")
self.network = self.connection.call('get_network',
{'network_id': int(network_id),
'include_data': 'Y',
'scenario_ids': [int(scenario_id)],
'template_id': None})
self.res_scenario = self.network.scenarios[0].resourcescenarios
attrslist = self.connection.call('get_all_attributes', {})
for attr in attrslist:
self.attrs.update({attr.id: attr.name})
#####################################################
def set_network(self, network):
"""
Load network and scenario from the server.
"""
self.network =network
#self.res_scenario = self.network.scenarios[0].resourcescenarios
attrslist = self.connection.call('get_attributes', {})
for attr in attrslist:
self.attrs.update({attr.id: attr.name})
#####################################################
def create_timeseries(self, data):
timeseries = {'0': {}}
counter=0
for time_s in self.actual_time_steps:
timeseries['0'][time_s]=json.dumps(data[counter])
counter+=1
return json.dumps(timeseries)
'''
def create_timeseries(self, data):
timeseries = {'0': {}}
index=0
for i, idx in enumerate(index):
timeseries['0'][self.time_axis[int(idx)]] = float(data[i])
index+=1
return json.dumps(timeseries)
'''
def create_scalar(self, value):
return json.dumps(value)
def create_array(self, index, data):
pass
def create_descriptor(self, value):
descriptor = json.dumps(value)
return descriptor
def save(self):
self.network.scenarios[0].resourcescenarios = self.res_scenario
self.connection.call('update_scenario', {'scen':self.network.scenarios[0]})
def import_res(self):
self.import_vars()
self.import_objs()
def import_vars(self):
'''
import varaible to Hydra
'''
nodes = dict()
for node in self.network.nodes:
nodes.update({node.id: node.name})
for var in self.vars.keys():
for varModel in self.vars[var]:
for link in self.network.links:
#print "Owner: ", type(varModel.owner)
fromnode = nodes[link.node_1_id]
tonode = nodes[link.node_2_id]
if len(varModel.owner)==2 and fromnode == varModel.owner[0] and tonode == varModel.owner[1]:
# "It is here, link Var: ",var,": ",varModel.owner, ": ", varModel.data_set
for attr in link.attributes:
if self.attrs[attr.attr_id] == varModel.name:
if attr.attr_is_var == 'Y':
dataset = dict(name = 'Pyomo import - ' + link.name + ' ' \
+ varModel.name)
dataset['unit'] = varModel.unit
if len(varModel.data_set)>1:
dataset['type'] = 'timeseries'
dataset['value'] = self.create_timeseries(varModel.data_set)
elif len(varModel.data_set) == 1:
try:
data = float(varModel.data_set [0])
dataset['type'] = 'scalar'
dataset['value'] = \
self.create_scalar(data)
except ValueError:
dataset['type'] = 'descriptor'
dataset['value'] = self.create_descriptor(varModel.data_set [0])
#print "link attr is added"
metadata={}
dataset['metadata']=json.dumps(metadata)
dataset['dimension']=attr.resourcescenario.value.dimension
res_scen = dict(resource_attr_id = attr.id,
attr_id = attr.attr_id,
value = dataset)
self.res_scenario.append(res_scen) #dataset
for node in self.network.nodes:
if node.name in varModel.owner and len(varModel.owner)==1:
# "node Var: ", var, ": ", varModel.owner,": ", varModel.data_set
for attr in node.attributes:
if self.attrs[attr.attr_id] == varModel.name:
if attr.attr_is_var == 'Y':
dataset = dict(name = 'Pyomo import - ' + node.name + ' ' \
+ varModel.name)
dataset['unit'] = varModel.unit
if len(varModel.data_set)>1:
dataset['type'] = 'timeseries'
dataset['value'] = self.create_timeseries(varModel.data_set)
elif len(varModel.data_set) == 1:
try:
data = float(varModel.data_set [0])
dataset['type'] = 'scalar'
dataset['value'] = \
self.create_scalar(data)
except ValueError:
dataset['type'] = 'descriptor'
dataset['value'] = self.create_descriptor(varModel.data_set [0])
#print "node att is added"
'''
'''
metadata={}
dataset['metadata']=json.dumps(metadata)
dataset['dimension']=attr.resourcescenario.value.dimension
res_scen = dict(resource_attr_id = attr.id,
attr_id = attr.attr_id,
value = dataset)
self.res_scenario.append(res_scen)
# "Attr is added ...."
def import_objs(self):
'''
import objects values to Hydra
'''
for var in self.objs.keys():
for varModel in self.objs[var]:
for attr in self.network.attributes:
if attr.attr_is_var == 'Y':
if self.attrs[attr.attr_id] == varModel.name:
dataset = dict(name = 'Pyomo import - ' + self.network.name + ' ' \
+ varModel.name)
dataset['unit'] = varModel.unit
if len(varModel.data_set)>1:
dataset['type'] = 'timeseries'
dataset['value'] = self.create_timeseries(varModel.data_set)
elif len(varModel.data_set) == 1:
try:
data = float(varModel.data_set [0])
dataset['type'] = 'scalar'
dataset['value'] = \
self.create_scalar(data)
except ValueError:
dataset['type'] = 'descriptor'
dataset['value'] = self.create_descriptor(varModel.data_set [0])
'''
'''
res_scen = dict(resource_attr_id = attr.id,
attr_id = attr.attr_id,
value = dataset)
metadata={}
dataset['metadata']=json.dumps(metadata)
dataset['dimension']=attr.resourcescenario.value.dimension
self.res_scenario.append(res_scen)
