def __init__(self, url=None, session_id=None): self.json_net = None self.warnings = [] self.files = [] self.template = None self.attributes = [] self.attr_name_map = {} self.nodes = {} self.links = {} self.groups = {} self.node_id = PluginLib.temp_ids() self.link_id = PluginLib.temp_ids() self.group_id = PluginLib.temp_ids() self.connection = JsonConnection(url) if session_id is not None: write_output("Using existing session %s" % session_id) self.connection.session_id = session_id else: self.connection.login() #3 steps: start, read, save self.num_steps = 3
def __init__(self, url=None, session_id=None): #Record the names of the files created by the plugin so we can #display them to the user. self.files = [] #A mapping from attr ID to attr object. Makes searching for attributes #easier self.attr_id_map = {} self.connection = JsonConnection(url) write_output("Connecting...") if session_id is not None: write_output("Using existing session %s"% session_id) self.connection.session_id=session_id else: self.connection.login() #Dictionaries, keyed on their name, where all the nodes, links & groups will go self.hydra_nodes = {} self.hydra_links = {} self.hydra_groups = {} self.hydra_group_members = {} #Generators of NEGATIVE IDS for the nodes & links. #Negative IDS are used as temporary client-side IDS until the network #is saved, at which point the resources will be given permanent, positive IDS. self.node_ids = temp_ids() self.link_ids = temp_ids() self.group_ids = temp_ids() # A group is an institution. #A mapping from the name of a type to the type itself. self.type_name_map = {} self.num_steps = 3
def __init__(self, url=None, session_id=None): self.json_net = None self.warnings = [] self.files = [] self.template = None self.attributes = [] self.attr_name_map = {} self.nodes = {} self.links = {} self.groups = {} self.node_id = PluginLib.temp_ids() self.link_id = PluginLib.temp_ids() self.group_id = PluginLib.temp_ids() self.connection = JsonConnection(url) if session_id is not None: write_output("Using existing session %s"% session_id) self.connection.session_id=session_id else: self.connection.login() #3 steps: start, read, save self.num_steps = 3
def __init__(self, url=None, username=None, password=None): self.conn = JsonConnection(url) if username is not None and password is not None: self.session_id = self.conn.login(username=username, password=password) else: self.session_id = self.conn.login() self.projects = dict()
class HydraNetworkTree(object): def __init__(self, url=None, username=None, password=None): self.conn = JsonConnection(url) if username is not None and password is not None: self.session_id = self.conn.login(username=username, password=password) else: self.session_id = self.conn.login() self.projects = dict() def get_tree(self): project_list = self.conn.call('get_projects', {}) for project in project_list: networks = self.conn.call('get_networks', {'project_id': project['id']}) project['networks'] = networks self.projects.update({project['id']: project}) def print_tree(self, color=True): if color: pr_col = '\033[1m' ne_col = '\033[92m' sc_col = '\033[94m' endtag = '\033[0m' else: pr_col = '' ne_col = '' sc_col = '' endtag = '' for pid, project in self.projects.iteritems(): print('%sP %3d %s%s' % (pr_col, pid, project['name'], endtag)) for network in project['networks']: print(u'%sN%s \u2514\u2500%s%2d %s%s' % (ne_col, endtag, ne_col, network['id'], network['name'], endtag)) for scenario in network['scenarios']: print(u'%sS%s \u2514\u2500%s%2d %s%s' % (sc_col, endtag, sc_col, scenario['id'], scenario['name'], endtag))
def __init__(self, vars, objs, actual_time_steps, url=None, session_id=None): self.vars = vars self.objs = objs self.actual_time_steps = actual_time_steps self.network_id = None self.scenario_id = None self.network = None self.res_scenario = None self.connection = JsonConnection(url) self.attrs = dict() self.time_axis = dict() if session_id is not None: log.info("Using existing session %s", session_id) self.connection.session_id = session_id else: self.connection.login()
def __init__(self, url=None, session_id=None): #Record the names of the files created by the plugin so we can #display them to the user. self.files = [] #A mapping from attr ID to attr object. Makes searching for attributes #easier self.attr_id_map = {} self.connection = JsonConnection(url) write_output("Connecting...") if session_id is not None: write_output("Using existing session %s" % session_id) self.connection.session_id = session_id else: self.connection.login() self.network = None self.num_steps = 3
def __init__(self, vars, objs, actual_time_steps, url=None, session_id=None): self.vars=vars self.objs=objs self.actual_time_steps=actual_time_steps self.network_id = None self.scenario_id = None self.network = None self.res_scenario=None self.connection = JsonConnection(url) self.attrs = dict() self.time_axis = dict() if session_id is not None: log.info("Using existing session %s", session_id) self.connection.session_id = session_id else: self.connection.login()
def __init__(self, url=None, username=None, password=None): super(HydraNetwork, self).__init__() self.conn = JsonConnection(url=url, app_name='ShapefileApp') self.url = url self.username = username self.password = password self.session_id = None self.project = None self.hydra_network = None self.hydra_scenario = None self.hydra_attributes = None self.attrs = dict() self.attr_ids = dict() self.epsg = None self.nodes = dict() self.links = [] self.link_names = dict() self.node_names = dict() self._node_coord_index = dict()
class Importer(object): def __init__(self, vars, objs, actual_time_steps, url=None, session_id=None): self.vars = vars self.objs = objs self.actual_time_steps = actual_time_steps self.network_id = None self.scenario_id = None self.network = None self.res_scenario = None self.connection = JsonConnection(url) self.attrs = dict() self.time_axis = dict() if session_id is not None: log.info("Using existing session %s", session_id) self.connection.session_id = session_id else: self.connection.login() def load_network(self, network_id, scenario_id): """ Load network and scenario from the server. """ try: network_id = int(network_id) except (TypeError, ValueError): network_id = self.network_id if network_id is None: raise HydraPluginError("No network specified.") try: scenario_id = int(scenario_id) except (TypeError, ValueError): scenario_id = self.scenario_id if scenario_id is None: raise HydraPluginError("No scenario specified.") self.network = self.connection.call( 'get_network', { 'network_id': int(network_id), 'include_data': 'Y', 'scenario_ids': [int(scenario_id)], 'template_id': None }) self.res_scenario = self.network.scenarios[0].resourcescenarios attrslist = self.connection.call('get_all_attributes', {}) for attr in attrslist: self.attrs.update({attr.id: attr.name}) ##################################################### def set_network(self, network): """ Load network and scenario from the server. """ self.network = network #self.res_scenario = self.network.scenarios[0].resourcescenarios attrslist = self.connection.call('get_attributes', {}) for attr in attrslist: self.attrs.update({attr.id: attr.name}) ##################################################### def create_timeseries(self, data): timeseries = {'0': {}} counter = 0 for time_s in self.actual_time_steps: timeseries['0'][time_s] = json.dumps(data[counter]) counter += 1 return json.dumps(timeseries) ''' def create_timeseries(self, data): timeseries = {'0': {}} index=0 for i, idx in enumerate(index): timeseries['0'][self.time_axis[int(idx)]] = float(data[i]) index+=1 return json.dumps(timeseries) ''' def create_scalar(self, value): return json.dumps(value) def create_array(self, index, data): pass def create_descriptor(self, value): descriptor = json.dumps(value) return descriptor def save(self): self.network.scenarios[0].resourcescenarios = self.res_scenario self.connection.call('update_scenario', {'scen': self.network.scenarios[0]}) def import_res(self): self.import_vars() self.import_objs() def import_vars(self): ''' import varaible to Hydra ''' nodes = dict() for node in self.network.nodes: nodes.update({node.id: node.name}) for var in self.vars.keys(): for varModel in self.vars[var]: for link in self.network.links: #print "Owner: ", type(varModel.owner) fromnode = nodes[link.node_1_id] tonode = nodes[link.node_2_id] if len(varModel.owner) == 2 and fromnode == varModel.owner[ 0] and tonode == varModel.owner[1]: # "It is here, link Var: ",var,": ",varModel.owner, ": ", varModel.data_set for attr in link.attributes: if self.attrs[attr.attr_id] == varModel.name: if attr.attr_is_var == 'Y': dataset = dict(name = 'Pyomo import - ' + link.name + ' ' \ + varModel.name) dataset['unit'] = varModel.unit if len(varModel.data_set) > 1: dataset['type'] = 'timeseries' dataset[ 'value'] = self.create_timeseries( varModel.data_set) elif len(varModel.data_set) == 1: try: data = float(varModel.data_set[0]) dataset['type'] = 'scalar' dataset['value'] = \ self.create_scalar(data) except ValueError: dataset['type'] = 'descriptor' dataset[ 'value'] = self.create_descriptor( varModel.data_set[0]) #print "link attr is added" metadata = {} dataset['metadata'] = json.dumps(metadata) dataset[ 'dimension'] = attr.resourcescenario.value.dimension res_scen = dict(resource_attr_id=attr.id, attr_id=attr.attr_id, value=dataset) self.res_scenario.append( res_scen) #dataset for node in self.network.nodes: if node.name in varModel.owner and len( varModel.owner) == 1: # "node Var: ", var, ": ", varModel.owner,": ", varModel.data_set for attr in node.attributes: if self.attrs[attr.attr_id] == varModel.name: if attr.attr_is_var == 'Y': dataset = dict(name = 'Pyomo import - ' + node.name + ' ' \ + varModel.name) dataset['unit'] = varModel.unit if len(varModel.data_set) > 1: dataset['type'] = 'timeseries' dataset[ 'value'] = self.create_timeseries( varModel.data_set) elif len(varModel.data_set) == 1: try: data = float(varModel.data_set[0]) dataset['type'] = 'scalar' dataset['value'] = \ self.create_scalar(data) except ValueError: dataset['type'] = 'descriptor' dataset[ 'value'] = self.create_descriptor( varModel.data_set[0]) #print "node att is added" ''' ''' metadata = {} dataset['metadata'] = json.dumps(metadata) dataset[ 'dimension'] = attr.resourcescenario.value.dimension res_scen = dict(resource_attr_id=attr.id, attr_id=attr.attr_id, value=dataset) self.res_scenario.append(res_scen) # "Attr is added ...." def import_objs(self): ''' import objects values to Hydra ''' for var in self.objs.keys(): for varModel in self.objs[var]: for attr in self.network.attributes: if attr.attr_is_var == 'Y': if self.attrs[attr.attr_id] == varModel.name: dataset = dict(name = 'Pyomo import - ' + self.network.name + ' ' \ + varModel.name) dataset['unit'] = varModel.unit if len(varModel.data_set) > 1: dataset['type'] = 'timeseries' dataset['value'] = self.create_timeseries( varModel.data_set) elif len(varModel.data_set) == 1: try: data = float(varModel.data_set[0]) dataset['type'] = 'scalar' dataset['value'] = \ self.create_scalar(data) except ValueError: dataset['type'] = 'descriptor' dataset['value'] = self.create_descriptor( varModel.data_set[0]) ''' ''' res_scen = dict(resource_attr_id=attr.id, attr_id=attr.attr_id, value=dataset) metadata = {} dataset['metadata'] = json.dumps(metadata) dataset[ 'dimension'] = attr.resourcescenario.value.dimension self.res_scenario.append(res_scen)
class ModelRunner(object): """ Exporter of Hydra networks to JSON or XML files. """ def __init__(self, url=None, session_id=None): #Record the names of the files created by the plugin so we can #display them to the user. self.files = [] #A mapping from attr ID to attr object. Makes searching for attributes #easier self.attr_id_map = {} self.connection = JsonConnection(url) write_output("Connecting...") if session_id is not None: write_output("Using existing session %s" % session_id) self.connection.session_id = session_id else: self.connection.login() self.network = None self.num_steps = 3 def get_attributes(self, template_id): if template_id is not None: #TODO: Find out why get_all_template_attributes isnt here. attributes = self.connection.call('get_all_attributes', {'template_id': template_id}) else: attributes = self.connection.call('get_all_attributes', {}) attr_id_map = {} for a in attributes: attr_id_map[a.id] = a self.attr_id_map = attr_id_map def get_network_data(self, network_id, scenario_id): """ Retrieve the network, identify the parameters to set, set them and run the model. Then identify the results and set them back on the network. """ write_output("Retrieving Network") write_progress(2, self.num_steps) if network_id is not None: if scenario_id is None: raise HydraPluginError("A scenario ID must be specified.") #The network ID can be specified to get the network... try: network_id = int(network_id) network = self.connection.call( 'get_network', { 'network_id': network_id, 'scenario_ids': [int(scenario_id)] }) write_output("Network retrieved") except Exception, e: log.exception(e) raise HydraPluginError("Network %s not found." % network_id) else:
class HobbesImporter(object): """ Importer of JSON files into Hydra. Also accepts XML files. """ def __init__(self, url=None, session_id=None): self.json_net = None self.warnings = [] self.files = [] self.template = None self.attributes = [] self.attr_name_map = {} self.nodes = {} self.links = {} self.groups = {} self.node_id = PluginLib.temp_ids() self.link_id = PluginLib.temp_ids() self.group_id = PluginLib.temp_ids() self.connection = JsonConnection(url) if session_id is not None: write_output("Using existing session %s" % session_id) self.connection.session_id = session_id else: self.connection.login() #3 steps: start, read, save self.num_steps = 3 def fetch_project(self, project_id): """ If a project ID is not specified, a new one must be created to hold the incoming network. If an ID is specified, we must retrieve the project to make sure it exists. If it does not exist, then throw an error. Returns the project object so that the network can access it's ID. """ if project_id is not None: try: project = self.connection.call('get_project', {'project_id': int(project_id)}) log.info('Loading existing project (ID=%s)' % project_id) return project except RequestError: raise HydraPluginError("Project with ID %s not found" % project_id) #Using 'datetime.now()' in the name guarantees a unique project name. new_project = dict( name = "Hobbes Project created at %s" % (datetime.now()), description = \ "Default project created by the %s plug-in." % \ (self.__class__.__name__), ) saved_project = self.connection.call('add_project', {'project': new_project}) return saved_project def fetch_template(self, template_id): self.template = self.connection.call('get_template', {'template_id': int(template_id)}) self.attributes = self.connection.call( 'get_template_attributes', {'template_id': int(template_id)}) #Build a lookup dict of attributes by name for a in self.attributes: self.attr_name_map[a.name] = a def fetch_remote_network(self): """ Request the hobbes network from the hobbes server """ write_output("Fetching Network") write_progress(2, self.num_steps) net_response = requests.get( "http://cwn.casil.ucdavis.edu/network/get") #JSON Network #http://cwn.casil.ucdavis.edu/excel/create?prmname=SR_CLE #XLS if net_response.status_code != 200: raise HydraPluginError( "A connection error has occurred with status code: %s" % net_response.status_code) self.json_net = json.loads(net_response.content) def upload_template(self): """ Upload the template file found in ./template.xml """ file_ = os.path.join(__location__, '../', 'template', 'template.xml') with open(file_) as f: xml_template = f.read() self.template = self.connection.call('upload_template_xml', {'template_xml': xml_template}) self.attributes = self.connection.call( 'get_template_attributes', {'template_id': self.template.id}) #Build a lookup dict of attributes by name for a in self.attributes: self.attr_name_map[a.name] = a def import_network_topology(self, project_id=None): """ Read the file containing the network data and send it to the server. """ if self.json_net is None: self.fetch_remote_network() for node in self.json_net: props = node['properties'] node_type = props['type'] node_coords = node['geometry']['coordinates'] tmp_node_id = self.node_id.next() #TODO: HACK. WHy are there 2 coordinates for the node? if isinstance(node_coords[0], list): #log.debug("Using 1st coords of %s (%s)", node_coords, props['type']) node_coords = node_coords[0] #log.info("X=%s, y=%s",node_coords[0], node_coords[1]) node = dict( id=tmp_node_id, name=props['prmname'], x=str( node_coords[0]), #swap these if they are lat-long, not x-y y=str(node_coords[1]), description=props['description'], attributes=[], types=[{ 'template_id': self.template.id, 'id': node_type }]) self.nodes[props['prmname']] = node #ATTRIBUTES #Find the matching type for t in self.template.types: if t.name == node_type: node['types'][0]['id'] = t.id #Assign the attributes to the node for tattr in t.typeattrs: node['attributes'].append({'attr_id': tattr.attr_id}) inlinks = [o['link_prmname'] for o in props.get('origins', [])] for linkname in inlinks: if linkname not in self.links: link = dict( id=self.link_id.next(), name=linkname, node_2_id=tmp_node_id, attributes=[], description="", ) self.links[linkname] = link else: link = self.links[linkname] link['node_2_id'] = tmp_node_id outlinks = [o['link_prmname'] for o in props.get('terminals', [])] for linkname in outlinks: if linkname not in self.links: link = dict( id=self.link_id.next(), name=linkname, node_1_id=tmp_node_id, attributes=[], description="", ) self.links[linkname] = link else: link = self.links[linkname] link['node_1_id'] = tmp_node_id node_groups = props['regions'] project = self.fetch_project(project_id) project_id = project.id write_output("Saving Network") write_progress(3, self.num_steps) for t in self.template.types: if t.name == 'HobbesNetwork': network_type = t.id hydra_network = { 'name': "HOBBES Network (%s)" % datetime.now(), 'description': "Hobbes Network, imported directly from the web API", 'nodes': self.nodes.values(), 'links': self.links.values(), 'project_id': project_id, 'projection': 'EPSG:2229', 'groups': [], 'scenarios': [], 'types': [{ 'template_id': self.template.id, 'id': network_type }] } #The network ID can be specified to get the network... self.network = self.connection.call('add_network', {'net': hydra_network}) return self.network def make_repo_dataset(self, json_repo): meta = {} for k, v in json_repo.items(): meta[k] = str(v) dataset = { 'dimension': 'dimensionless', 'unit': None, 'type': 'descriptor', 'value': json_repo['tag'], 'metadata': json.dumps(meta), 'name': 'repo', } return dataset def import_data(self, include_timeseries=True): scenario = { "name": "Hobbes Import", "description": "Import from hobbes", } #List of parameters to ignore non_attributes = set([ 'origins', 'prmname', 'regions', 'terminals', 'description', 'extras', 'type', 'repo', 'origin' ]) #Make a map from node name to a list of attributes. Do this by first #making a node id map node_name_id_map = {} for n in self.network.nodes: node_name_id_map[n.name] = n.id node_attributes = self.connection.call('get_all_node_attributes', {'network_id': self.network.id}) node_id_attr_map = {} for a in node_attributes: n_attrs = node_id_attr_map.get(a.ref_id, []) n_attrs.append(a) node_id_attr_map[a.ref_id] = n_attrs resource_scenarios = [] #request data for first 2 nodes. for node in self.json_net[:10]: props = node['properties'] name = props['prmname'] node_id = node_name_id_map[name] #repo is a special case repo = self.make_repo_dataset(props['repo']) repo_attr_id = self.attr_name_map['repo'].id ra_id = None for a in node_id_attr_map[node_id]: if a.attr_id == repo_attr_id: ra_id = a.id break repo_rs = dict( resource_attr_id=ra_id, attr_id=repo_attr_id, is_var='N', value=repo, ) resource_scenarios.append(repo_rs) for k, v in props.items(): if k not in non_attributes: if isinstance(v, float): attr_id = self.attr_name_map[k].id dataset = dict( name=k, value=str(v), type='scalar', dimension='dimensionless', unit=None, ) ra_id = None for a in node_id_attr_map[node_id]: if a.attr_id == attr_id: ra_id = a.id break resource_scenario = dict( resource_attr_id=ra_id, attr_id=attr_id, is_var='N', value=dataset, ) resource_scenarios.append(resource_scenario) #timeseries, requested from the hobbes server if include_timeseries is True: extras = props.get('extras', []) if extras is not None and len(extras) > 0: attr_response = requests.get( "http://cwn.casil.ucdavis.edu/network/extras?prmname=%s" % props['prmname']) #JSON attributes else: continue if attr_response.status_code != 200: raise HydraPluginError( "A connection error has occurred with status code: %s" % attr_response.status_code) extra_data = json.loads(attr_response.content) non_attrs = ['prmname', 'readme'] for k, v in extra_data.items(): if k in non_attrs: continue else: if len(v) < 2: continue ts = self.parse_timeseries(v) attr_id = self.attr_name_map[k].id dataset = dict( name=k, value=json.dumps(ts), type='timeseries', dimension='dimensionless', unit=None, ) ra_id = None for a in node_id_attr_map[node_id]: if a.attr_id == attr_id: ra_id = a.id break resource_scenario = dict( resource_attr_id=ra_id, attr_id=attr_id, is_var='N', value=dataset, ) resource_scenarios.append(resource_scenario) scenario['resourcescenarios'] = resource_scenarios new_scenario = self.connection.call('add_scenario', { 'network_id': self.network.id, 'scen': scenario }) self.scenario = new_scenario return new_scenario def parse_timeseries(self, timeseries_value): """ Convert a hobbes timeseries to a hydra timeseries """ timeformat = config.get('DEFAULT', 'datetime_format') val = {} for timeval in timeseries_value[1:]: time = timeval[0] split = time.split('-') d = datetime(year=int(split[0]), month=int(split[1]), day=int(split[2])) tstime = datetime.strftime(d, timeformat) val[tstime] = float(timeval[1]) return {"idx1": val}
class HobbesImporter(object): """ Importer of JSON files into Hydra. Also accepts XML files. """ def __init__(self, url=None, session_id=None): self.json_net = None self.warnings = [] self.files = [] self.template = None self.attributes = [] self.attr_name_map = {} self.nodes = {} self.links = {} self.groups = {} self.node_id = PluginLib.temp_ids() self.link_id = PluginLib.temp_ids() self.group_id = PluginLib.temp_ids() self.connection = JsonConnection(url) if session_id is not None: write_output("Using existing session %s"% session_id) self.connection.session_id=session_id else: self.connection.login() #3 steps: start, read, save self.num_steps = 3 def fetch_project(self, project_id): """ If a project ID is not specified, a new one must be created to hold the incoming network. If an ID is specified, we must retrieve the project to make sure it exists. If it does not exist, then throw an error. Returns the project object so that the network can access it's ID. """ if project_id is not None: try: project = self.connection.call('get_project', {'project_id':int(project_id)}) log.info('Loading existing project (ID=%s)' % project_id) return project except RequestError: raise HydraPluginError("Project with ID %s not found"%project_id) #Using 'datetime.now()' in the name guarantees a unique project name. new_project = dict( name = "Hobbes Project created at %s" % (datetime.now()), description = \ "Default project created by the %s plug-in." % \ (self.__class__.__name__), ) saved_project = self.connection.call('add_project', {'project':new_project}) return saved_project def fetch_template(self, template_id): self.template = self.connection.call('get_template', {'template_id':int(template_id)}) self.attributes = self.connection.call('get_template_attributes', {'template_id':int(template_id)}) #Build a lookup dict of attributes by name for a in self.attributes: self.attr_name_map[a.name] = a def fetch_remote_network(self): """ Request the hobbes network from the hobbes server """ write_output("Fetching Network") write_progress(2, self.num_steps) net_response = requests.get("http://cwn.casil.ucdavis.edu/network/get") #JSON Network #http://cwn.casil.ucdavis.edu/excel/create?prmname=SR_CLE #XLS if net_response.status_code != 200: raise HydraPluginError("A connection error has occurred with status code: %s"%net_response.status_code) self.json_net = json.loads(net_response.content) def upload_template(self): """ Upload the template file found in ./template.xml """ file_ = os.path.join(__location__, '../', 'template', 'template.xml') with open(file_) as f: xml_template = f.read() self.template = self.connection.call('upload_template_xml', {'template_xml':xml_template}) self.attributes = self.connection.call('get_template_attributes', {'template_id':self.template.id}) #Build a lookup dict of attributes by name for a in self.attributes: self.attr_name_map[a.name] = a def import_network_topology(self, project_id=None): """ Read the file containing the network data and send it to the server. """ if self.json_net is None: self.fetch_remote_network() for node in self.json_net: props = node['properties'] node_type = props['type'] node_coords = node['geometry']['coordinates'] tmp_node_id = self.node_id.next() #TODO: HACK. WHy are there 2 coordinates for the node? if isinstance(node_coords[0], list): #log.debug("Using 1st coords of %s (%s)", node_coords, props['type']) node_coords = node_coords[0] #log.info("X=%s, y=%s",node_coords[0], node_coords[1]) node = dict( id = tmp_node_id, name = props['prmname'], x = str(node_coords[0]), #swap these if they are lat-long, not x-y y = str(node_coords[1]), description = props['description'], attributes = [], types = [{'template_id':self.template.id, 'id':node_type}] ) self.nodes[props['prmname']] = node #ATTRIBUTES #Find the matching type for t in self.template.types: if t.name == node_type: node['types'][0]['id'] = t.id #Assign the attributes to the node for tattr in t.typeattrs: node['attributes'].append( {'attr_id':tattr.attr_id} ) inlinks = [o['link_prmname'] for o in props.get('origins', [])] for linkname in inlinks: if linkname not in self.links: link = dict( id=self.link_id.next(), name = linkname, node_2_id = tmp_node_id, attributes = [], description = "", ) self.links[linkname] = link else: link = self.links[linkname] link['node_2_id'] = tmp_node_id outlinks = [o['link_prmname'] for o in props.get('terminals', [])] for linkname in outlinks: if linkname not in self.links: link = dict( id=self.link_id.next(), name = linkname, node_1_id = tmp_node_id, attributes = [], description = "", ) self.links[linkname] = link else: link = self.links[linkname] link['node_1_id'] = tmp_node_id node_groups = props['regions'] project = self.fetch_project(project_id) project_id = project.id write_output("Saving Network") write_progress(3, self.num_steps) for t in self.template.types: if t.name == 'HobbesNetwork': network_type = t.id hydra_network = { 'name' : "HOBBES Network (%s)"%datetime.now(), 'description' : "Hobbes Network, imported directly from the web API", 'nodes': self.nodes.values(), 'links': self.links.values(), 'project_id' : project_id, 'projection':'EPSG:2229', 'groups' : [], 'scenarios': [], 'types' : [{'template_id':self.template.id, 'id':network_type}] } #The network ID can be specified to get the network... self.network = self.connection.call('add_network', {'net':hydra_network}) return self.network def make_repo_dataset(self, json_repo): meta = {} for k, v in json_repo.items(): meta[k] = str(v) dataset = { 'dimension': 'dimensionless', 'unit' : None, 'type' : 'descriptor', 'value' : json_repo['tag'], 'metadata' : json.dumps(meta), 'name' : 'repo', } return dataset def import_data(self, include_timeseries=True): scenario = { "name" : "Hobbes Import", "description" : "Import from hobbes", } #List of parameters to ignore non_attributes = set(['origins', 'prmname', 'regions', 'terminals', 'description', 'extras', 'type', 'repo', 'origin']) #Make a map from node name to a list of attributes. Do this by first #making a node id map node_name_id_map = {} for n in self.network.nodes: node_name_id_map[n.name] = n.id node_attributes = self.connection.call('get_all_node_attributes', {'network_id':self.network.id}) node_id_attr_map = {} for a in node_attributes: n_attrs = node_id_attr_map.get(a.ref_id, []) n_attrs.append(a) node_id_attr_map[a.ref_id] = n_attrs resource_scenarios = [] #request data for first 2 nodes. for node in self.json_net[:10]: props = node['properties'] name = props['prmname'] node_id = node_name_id_map[name] #repo is a special case repo = self.make_repo_dataset(props['repo']) repo_attr_id = self.attr_name_map['repo'].id ra_id = None for a in node_id_attr_map[node_id]: if a.attr_id == repo_attr_id: ra_id = a.id break repo_rs = dict( resource_attr_id = ra_id, attr_id = repo_attr_id, is_var = 'N', value = repo, ) resource_scenarios.append(repo_rs) for k, v in props.items(): if k not in non_attributes: if isinstance(v, float): attr_id = self.attr_name_map[k].id dataset = dict( name = k, value = str(v), type = 'scalar', dimension = 'dimensionless', unit = None, ) ra_id = None for a in node_id_attr_map[node_id]: if a.attr_id == attr_id: ra_id = a.id break resource_scenario = dict( resource_attr_id = ra_id, attr_id = attr_id, is_var = 'N', value = dataset, ) resource_scenarios.append(resource_scenario) #timeseries, requested from the hobbes server if include_timeseries is True: extras = props.get('extras', []) if extras is not None and len(extras) > 0: attr_response = requests.get("http://cwn.casil.ucdavis.edu/network/extras?prmname=%s"%props['prmname']) #JSON attributes else: continue if attr_response.status_code != 200: raise HydraPluginError("A connection error has occurred with status code: %s"%attr_response.status_code) extra_data = json.loads(attr_response.content) non_attrs = ['prmname', 'readme'] for k, v in extra_data.items(): if k in non_attrs: continue else: if len(v) < 2: continue ts = self.parse_timeseries(v) attr_id = self.attr_name_map[k].id dataset = dict( name = k, value = json.dumps(ts), type = 'timeseries', dimension = 'dimensionless', unit = None, ) ra_id = None for a in node_id_attr_map[node_id]: if a.attr_id == attr_id: ra_id = a.id break resource_scenario = dict( resource_attr_id = ra_id, attr_id = attr_id, is_var = 'N', value = dataset, ) resource_scenarios.append(resource_scenario) scenario['resourcescenarios'] = resource_scenarios new_scenario = self.connection.call('add_scenario', {'network_id':self.network.id, 'scen':scenario }) self.scenario = new_scenario return new_scenario def parse_timeseries(self, timeseries_value): """ Convert a hobbes timeseries to a hydra timeseries """ timeformat = config.get('DEFAULT', 'datetime_format') val = {} for timeval in timeseries_value[1:]: time = timeval[0] split = time.split('-') d = datetime(year=int(split[0]), month=int(split[1]), day=int(split[2])) tstime = datetime.strftime(d, timeformat) val[tstime] = float(timeval[1]) return {"idx1": val}
class HydraNetwork(HydraResource): def __init__(self, url=None, username=None, password=None): super(HydraNetwork, self).__init__() self.conn = JsonConnection(url=url, app_name='ShapefileApp') self.url = url self.username = username self.password = password self.session_id = None self.project = None self.hydra_network = None self.hydra_scenario = None self.hydra_attributes = None self.attrs = dict() self.attr_ids = dict() self.epsg = None self.nodes = dict() self.links = [] self.link_names = dict() self.node_names = dict() self._node_coord_index = dict() def login(self): if self.username is not None and self.password is not None: self.session_id = self.conn.login(username=self.username, password=self.password) else: self.session_id = self.conn.login() def load_attributes(self): self.hydra_attributes = self.conn.call('get_all_attributes', {}) for attr in self.hydra_attributes: self.attrs[attr.id] = attr self.attr_ids[attr.name] = attr.id def load_network(self, network_id, scenario_id): """Load a network from HydraPlatform. """ if self.project is None: self.load_project(network_id=network_id) self.hydra_network = self.conn.call('get_network', {'network_id': network_id, 'include_data': 'Y'}) self.load_attributes() for scenario in self.hydra_network['scenarios']: if scenario['id'] == scenario_id: self.hydra_scenario = scenario break res_scen_dict = dict() for res_scen in self.hydra_scenario['resourcescenarios']: res_scen_dict.update({res_scen['resource_attr_id']: res_scen}) # Create node index node_dict = dict() for node in self.hydra_network['nodes']: node_dict.update({node['id']: node}) self.name = self.hydra_network['name'] self.description = self.hydra_network['description'] if 'projection' in self.hydra_network.keys(): try: self.epsg = \ int(self.hydra_network['projection'].split(':')[1]) except ValueError: warnings.warn('Could not load EPSG code.') # Add network attributes for res_attr in self.hydra_network['attributes']: if res_scen_dict.get(res_attr['id']) is not None: res_scen = res_scen_dict[res_attr['id']] self.add_attribute(self.attrs[res_attr['attr_id']], res_attr, res_scen) else: self.add_attribute(self.attrs[res_attr['attr_id']], res_attr, None) # Add nodes and attributes for node in self.hydra_network['nodes']: n_node = HydraNode(x=float(node['x']), y=float(node['y'])) n_node.name = node['name'] n_node.layout = node['layout'] n_node.id = node['id'] n_node.types = node['types'] for res_attr in node['attributes']: if res_scen_dict.get(res_attr['id']) is not None: res_scen = res_scen_dict[res_attr['id']] n_node.add_attribute(self.attrs[res_attr['attr_id']], res_attr, res_scen) else: n_node.add_attribute(self.attrs[res_attr['attr_id']], res_attr, None) self.add_node(n_node) # Add segments and attributes for link in self.hydra_network['links']: n_link = HydraLink(start_node=self.nodes[link['node_1_id']], end_node=self.nodes[link['node_2_id']]) n_link.name = link['name'] n_link.layout = link['layout'] n_link.id = link['id'] n_link.types = link['types'] for res_attr in link['attributes']: if res_scen_dict.get(res_attr['id']) is not None: res_scen = res_scen_dict[res_attr['id']] n_link.add_attribute(self.attrs[res_attr['attr_id']], res_attr, res_scen) else: n_link.add_attribute(self.attrs[res_attr['attr_id']], res_attr, None) self.add_link(n_link) def load_project(self, project_id=None, network_id=None): """Load a project by its ID or by a network ID. """ if project_id is not None: self.project = self.conn.call('get_project', {'project_id': project_id}) elif network_id is not None: self.project = self.conn.call('get_network_project', {'network_id': network_id}) def create_project(self, name=None): self.project = dict() if name is not None: self.project['name'] = name else: self.project['name'] = "Shapefile import @ %s" % datetime.now() self.project['description'] = '' self.project['status'] = 'A' self.project = self.conn.call('add_project', {'project': self.project}) def add_node(self, node): if self.node_names.get(node.name.lower()) is not None: add_string = ' (%s)' % self.node_names[node.name.lower()] self.node_names[node.name.lower()] += 1 node.name += add_string else: self.node_names[node.name.lower()] = 1 self.nodes[node.id] = node self._node_coord_index[(node.x, node.y)] = node.id def add_link(self, link): if self.link_names.get(link.name.lower()) is not None: add_string = ' (%s)' % self.link_names[link.name.lower()] self.link_names[link.name.lower()] += 1 link.name += add_string else: self.link_names[link.name.lower()] = 1 self.links.append(link) def save_network(self, network_name=None, project_name=None): """Save the network to HydraPlatform server. """ if self.project is None: self.create_project(name=project_name) self.hydra_network = dict() self.hydra_scenario = dict() self.hydra_scenario['name'] = 'Scenario created by ShapefileApp' self.hydra_scenario['description'] = \ 'Standard scenario created by ShapefileApp.' self.hydra_scenario['id'] = -1 self.hydra_scenario['resourcescenarios'] = [] if network_name is not None: self.hydra_network['name'] = network_name else: self.hydra_network['name'] = \ "Network imported from shapefile (%s)." % self.name self.hydra_network['description'] = \ "Network imported from %s" % self.name if self.epsg is not None: self.hydra_network['projection'] = \ 'EPSG:%s' % self.epsg self.hydra_network['nodes'] = [] self.hydra_network['links'] = [] self.hydra_network['attributes'] = [] self.hydra_network['scenarios'] = [] self.hydra_network['project_id'] = self.project['id'] for node in self.nodes.values(): hydra_node = self.create_hydra_node(node) self.hydra_network['nodes'].append(hydra_node) for link in self.links: hydra_link = self.create_hydra_link(link) self.hydra_network['links'].append(hydra_link) self.hydra_network['scenarios'].append(self.hydra_scenario) net_summary = self.conn.call('add_network', {'net': self.hydra_network}) return net_summary def create_hydra_node(self, node): """Build a node dict from a HydraNode object. """ hydra_node = dict() hydra_node['id'] = node.id hydra_node['name'] = node.name hydra_node['description'] = '' hydra_node['attributes'] = [] hydra_node['x'] = repr(node.x) hydra_node['y'] = repr(node.y) #TODO: Add attributes return hydra_node def create_hydra_link(self, link): """Build a link dict from a HydraLink object. """ hydra_link = dict() #TODO: Finalise this function return hydra_link
# import import pandas from pandas.io.json import json_normalize import sys from os.path import expanduser from os.path import join import json from HydraLib.PluginLib import JsonConnection url = 'www.openaguadss.org' conn = JsonConnection(url=url) conn.login(username = '******', password = '') def get_project_by_name(conn, project_name): return conn.call('get_project_by_name', {'project_name':project_name}) def get_network_by_name(conn, project_id, network_name): return conn.call('get_network_by_name', {'project_id':project_id, 'network_name':network_name}) # get project ID project_name = 'Water Allocation Demo' project = get_project_by_name(conn, project_name) project_id = project.id # get network ID network_name = 'Water Allocation Network' network = get_network_by_name(conn, project_id, network_name) network_id = network.id
class NetworkImporter(object): """ Exporter of Hydra networks to JSON or XML files. """ def __init__(self, url=None, session_id=None): #Record the names of the files created by the plugin so we can #display them to the user. self.files = [] #A mapping from attr ID to attr object. Makes searching for attributes #easier self.attr_id_map = {} self.connection = JsonConnection(url) write_output("Connecting...") if session_id is not None: write_output("Using existing session %s"% session_id) self.connection.session_id=session_id else: self.connection.login() #Dictionaries, keyed on their name, where all the nodes, links & groups will go self.hydra_nodes = {} self.hydra_links = {} self.hydra_groups = {} self.hydra_group_members = {} #Generators of NEGATIVE IDS for the nodes & links. #Negative IDS are used as temporary client-side IDS until the network #is saved, at which point the resources will be given permanent, positive IDS. self.node_ids = temp_ids() self.link_ids = temp_ids() self.group_ids = temp_ids() # A group is an institution. #A mapping from the name of a type to the type itself. self.type_name_map = {} self.num_steps = 3 def fetch_project(self, project_id): """ If a project ID is not specified, a new one must be created to hold the incoming network. If an ID is specified, we must retrieve the project to make sure it exists. If it does not exist, then throw an error. Returns the project object so that the network can access it's ID. """ if project_id is not None: try: project = self.connection.call('get_project', {'project_id':project_id}) log.info('Loading existing project (ID=%s)' % project_id) return project except RequestError, e: log.exception(e) raise HydraPluginError("An error occurred retrieving project ID %s"%project_id) #Using 'datetime.now()' in the name guarantees a unique project name. new_project = dict( name = "Hobbes Project created at %s" % (datetime.now()), description = \ "Default project created by the %s plug-in." % \ (self.__class__.__name__), ) saved_project = self.connection.call('add_project', {'project':new_project}) return saved_project
class Importer (object): def __init__(self, vars, objs, actual_time_steps, url=None, session_id=None): self.vars=vars self.objs=objs self.actual_time_steps=actual_time_steps self.network_id = None self.scenario_id = None self.network = None self.res_scenario=None self.connection = JsonConnection(url) self.attrs = dict() self.time_axis = dict() if session_id is not None: log.info("Using existing session %s", session_id) self.connection.session_id = session_id else: self.connection.login() def load_network(self, network_id, scenario_id): """ Load network and scenario from the server. """ try: network_id = int(network_id) except (TypeError, ValueError): network_id = self.network_id if network_id is None: raise HydraPluginError("No network specified.") try: scenario_id = int(scenario_id) except (TypeError, ValueError): scenario_id = self.scenario_id if scenario_id is None: raise HydraPluginError("No scenario specified.") self.network = self.connection.call('get_network', {'network_id': int(network_id), 'include_data': 'Y', 'scenario_ids': [int(scenario_id)], 'template_id': None}) self.res_scenario = self.network.scenarios[0].resourcescenarios attrslist = self.connection.call('get_all_attributes', {}) for attr in attrslist: self.attrs.update({attr.id: attr.name}) ##################################################### def set_network(self, network): """ Load network and scenario from the server. """ self.network =network #self.res_scenario = self.network.scenarios[0].resourcescenarios attrslist = self.connection.call('get_attributes', {}) for attr in attrslist: self.attrs.update({attr.id: attr.name}) ##################################################### def create_timeseries(self, data): timeseries = {'0': {}} counter=0 for time_s in self.actual_time_steps: timeseries['0'][time_s]=json.dumps(data[counter]) counter+=1 return json.dumps(timeseries) ''' def create_timeseries(self, data): timeseries = {'0': {}} index=0 for i, idx in enumerate(index): timeseries['0'][self.time_axis[int(idx)]] = float(data[i]) index+=1 return json.dumps(timeseries) ''' def create_scalar(self, value): return json.dumps(value) def create_array(self, index, data): pass def create_descriptor(self, value): descriptor = json.dumps(value) return descriptor def save(self): self.network.scenarios[0].resourcescenarios = self.res_scenario self.connection.call('update_scenario', {'scen':self.network.scenarios[0]}) def import_res(self): self.import_vars() self.import_objs() def import_vars(self): ''' import varaible to Hydra ''' nodes = dict() for node in self.network.nodes: nodes.update({node.id: node.name}) for var in self.vars.keys(): for varModel in self.vars[var]: for link in self.network.links: #print "Owner: ", type(varModel.owner) fromnode = nodes[link.node_1_id] tonode = nodes[link.node_2_id] if len(varModel.owner)==2 and fromnode == varModel.owner[0] and tonode == varModel.owner[1]: # "It is here, link Var: ",var,": ",varModel.owner, ": ", varModel.data_set for attr in link.attributes: if self.attrs[attr.attr_id] == varModel.name: if attr.attr_is_var == 'Y': dataset = dict(name = 'Pyomo import - ' + link.name + ' ' \ + varModel.name) dataset['unit'] = varModel.unit if len(varModel.data_set)>1: dataset['type'] = 'timeseries' dataset['value'] = self.create_timeseries(varModel.data_set) elif len(varModel.data_set) == 1: try: data = float(varModel.data_set [0]) dataset['type'] = 'scalar' dataset['value'] = \ self.create_scalar(data) except ValueError: dataset['type'] = 'descriptor' dataset['value'] = self.create_descriptor(varModel.data_set [0]) #print "link attr is added" metadata={} dataset['metadata']=json.dumps(metadata) dataset['dimension']=attr.resourcescenario.value.dimension res_scen = dict(resource_attr_id = attr.id, attr_id = attr.attr_id, value = dataset) self.res_scenario.append(res_scen) #dataset for node in self.network.nodes: if node.name in varModel.owner and len(varModel.owner)==1: # "node Var: ", var, ": ", varModel.owner,": ", varModel.data_set for attr in node.attributes: if self.attrs[attr.attr_id] == varModel.name: if attr.attr_is_var == 'Y': dataset = dict(name = 'Pyomo import - ' + node.name + ' ' \ + varModel.name) dataset['unit'] = varModel.unit if len(varModel.data_set)>1: dataset['type'] = 'timeseries' dataset['value'] = self.create_timeseries(varModel.data_set) elif len(varModel.data_set) == 1: try: data = float(varModel.data_set [0]) dataset['type'] = 'scalar' dataset['value'] = \ self.create_scalar(data) except ValueError: dataset['type'] = 'descriptor' dataset['value'] = self.create_descriptor(varModel.data_set [0]) #print "node att is added" ''' ''' metadata={} dataset['metadata']=json.dumps(metadata) dataset['dimension']=attr.resourcescenario.value.dimension res_scen = dict(resource_attr_id = attr.id, attr_id = attr.attr_id, value = dataset) self.res_scenario.append(res_scen) # "Attr is added ...." def import_objs(self): ''' import objects values to Hydra ''' for var in self.objs.keys(): for varModel in self.objs[var]: for attr in self.network.attributes: if attr.attr_is_var == 'Y': if self.attrs[attr.attr_id] == varModel.name: dataset = dict(name = 'Pyomo import - ' + self.network.name + ' ' \ + varModel.name) dataset['unit'] = varModel.unit if len(varModel.data_set)>1: dataset['type'] = 'timeseries' dataset['value'] = self.create_timeseries(varModel.data_set) elif len(varModel.data_set) == 1: try: data = float(varModel.data_set [0]) dataset['type'] = 'scalar' dataset['value'] = \ self.create_scalar(data) except ValueError: dataset['type'] = 'descriptor' dataset['value'] = self.create_descriptor(varModel.data_set [0]) ''' ''' res_scen = dict(resource_attr_id = attr.id, attr_id = attr.attr_id, value = dataset) metadata={} dataset['metadata']=json.dumps(metadata) dataset['dimension']=attr.resourcescenario.value.dimension self.res_scenario.append(res_scen)