def __init__(self, args, link_export_flag, steps): if link_export_flag == 'l': self.links_as_name = True else: self.links_as_name = False self.steps = steps self.use_gams_date_index = False self.network_id = int(args.network) self.scenario_id = int(args.scenario) self.template_id = int( args.template_id) if args.template_id is not None else None self.output_file = args.output self.export_by_type = args.export_by_type self.time_index = [] write_progress(1, self.steps) self.output_file_contents = [] self.output_file_contents.append("#%s\n" % ("*" * 78, )) self.output_file_contents.append( "# Data exported from Hydra using PyomoPlugin.\n") self.output_file_contents.append( "# (c) Copyright 2015, University of Manchester\n") self.time_index = {} self.connect(args)
def get_network_data(self, network_id, scenario_id): """ Retrieve the network, identify the parameters to set, set them and run the model. Then identify the results and set them back on the network. """ write_output("Retrieving Network") write_progress(2, self.num_steps) if network_id is not None: if scenario_id is None: raise HydraPluginError("A scenario ID must be specified.") #The network ID can be specified to get the network... try: network_id = int(network_id) network = self.connection.call( 'get_network', { 'network_id': network_id, 'scenario_ids': [int(scenario_id)] }) write_output("Network retrieved") except Exception, e: log.exception(e) raise HydraPluginError("Network %s not found." % network_id)
def save_file(self): ''' save output file ''' write_progress(8, self.steps) log.info("writing data to file") file = open(self.output_file, "w") file.write("".join(self.output_file_contents)) file.close()
def fetch_remote_network(self): """ Request the hobbes network from the hobbes server """ write_output("Fetching Network") write_progress(2, self.num_steps) net_response = requests.get("http://cwn.casil.ucdavis.edu/network/get") #JSON Network #http://cwn.casil.ucdavis.edu/excel/create?prmname=SR_CLE #XLS if net_response.status_code != 200: raise HydraPluginError("A connection error has occurred with status code: %s"%net_response.status_code) self.json_net = json.loads(net_response.content)
def import_result(args, vars, objs, actual_time_steps, url, session_id): write_progress(1, steps) imp=Importer(vars, objs, actual_time_steps, url, session_id) write_progress(2, steps) imp.load_network(args.network, args.scenario) write_progress(3, steps) imp.import_res() write_progress(4, steps) imp.save()
def fetch_remote_network(self): """ Request the hobbes network from the hobbes server """ write_output("Fetching Network") write_progress(2, self.num_steps) net_response = requests.get( "http://cwn.casil.ucdavis.edu/network/get") #JSON Network #http://cwn.casil.ucdavis.edu/excel/create?prmname=SR_CLE #XLS if net_response.status_code != 200: raise HydraPluginError( "A connection error has occurred with status code: %s" % net_response.status_code) self.json_net = json.loads(net_response.content)
def import_result(args, vars, objs, actual_time_steps): write_progress(9, steps) imp = Importer(vars, objs, actual_time_steps, args.server_url, args.session_id) write_progress(10, steps) imp.load_network(args.network, args.scenario) write_progress(11, steps) # imp.set_network(network) imp.import_res() write_progress(12, steps) imp.save()
def import_result(args, vars, objs, actual_time_steps): write_progress(9, steps) imp = Importer(vars, objs, actual_time_steps, args.server_url, args.session_id) write_progress(10, steps) imp.load_network(args.network, args.scenario) write_progress(11, steps) #imp.set_network(network) imp.import_res() write_progress(12, steps) imp.save()
def run(): parser = commandline_parser() args = parser.parse_args() hobbes_importer = HobbesImporter(url=args.server_url, session_id=args.session_id) scenarios = [] errors = [] network_id = None scenario_id = None try: write_progress(1, hobbes_importer.num_steps) validate_plugin_xml(os.path.join(__location__, 'plugin.xml')) #This step is to avoid doing the request to make the template and #then again for the data. hobbes_importer.fetch_remote_network() if args.template_id is None: tmpl = HobbesTemplateBuilder() tmpl.convert(hobbes_importer.json_net) hobbes_importer.upload_template() else: hobbes_importer.fetch_template(args.template_id) net = hobbes_importer.import_network_topology(args.project_id) scenario = hobbes_importer.import_data() #scenarios = [s.id for s in net.scenarios] network_id = net.id scenario_id = scenario.id message = "Import complete" except HydraPluginError as e: message = "An error has occurred" errors = [e.message] log.exception(e) except Exception, e: message = "An unknown error has occurred" log.exception(e) errors = [e]
def run(): parser = commandline_parser() args = parser.parse_args() hobbes_importer = HobbesImporter(url=args.server_url, session_id=args.session_id) scenarios = [] errors = [] network_id = None scenario_id = None try: write_progress(1, hobbes_importer.num_steps) validate_plugin_xml(os.path.join(__location__, 'plugin.xml')) #This step is to avoid doing the request to make the template and #then again for the data. hobbes_importer.fetch_remote_network() if args.template_id is None: tmpl = HobbesTemplateBuilder() tmpl.convert(hobbes_importer.json_net) hobbes_importer.upload_template() else: hobbes_importer.fetch_template(args.template_id) net = hobbes_importer.import_network_topology(args.project_id) scenario = hobbes_importer.import_data() #scenarios = [s.id for s in net.scenarios] network_id = net.id scenario_id = scenario.id message = "Import complete" except HydraPluginError as e: message="An error has occurred" errors = [e.message] log.exception(e) except Exception, e: message="An unknown error has occurred" log.exception(e) errors = [e]
def __init__(self, args, link_export_flag, steps): if link_export_flag == 'l': self.links_as_name = True else: self.links_as_name = False self.steps=steps self.use_gams_date_index=False self.network_id = int(args.network) self.scenario_id = int(args.scenario) self.template_id = int(args.template_id) if args.template_id is not None else None self.output_file = args.output self.export_by_type =args.export_by_type self.time_index = [] write_progress(1, self.steps) self.output_file_contents=[]; self.output_file_contents.append("#%s\n"%("*"*78,)) self.output_file_contents.append("# Data exported from Hydra using PyomoPlugin.\n") self.output_file_contents.append("# (c) Copyright 2015, University of Manchester\n") self.time_index = {} self.connect(args)
def write_network(self, network, target_dir): write_output("Writing network to file") write_progress(3, self.num_steps) if self.as_xml is False: file_name = "network_%s.json" % (network['name']) self.files.append(os.path.join(target_dir, file_name)) network_file = open(os.path.join(target_dir, file_name), 'w') network_file.write( json.dumps(network, sort_keys=True, indent=4, separators=(',', ': '))) else: file_name = "network_%s.xml" % (network['name']) self.files.append(os.path.join(target_dir, file_name)) network_file = open(os.path.join(target_dir, file_name), 'w') json_network = {'network': network} network_file.write(json2xml(json_network)) write_output("Network Written to %s " % (target_dir))
try: check_args(args) netword_id = convert_to_int(args.network, "Network Id") scenario_id = convert_to_int(args.scenario, "scenario Id") link_export_flag = "nn" if args.link_name is True: link_export_flag = "l" network = export_data(args) vars, objs = run_model(args.output, args.model_file) actual_time_steps = read_inputData(args.output) import_result(args, vars, objs, actual_time_steps) message = "Run successfully" print PluginLib.create_xml_response("PyomoAuto", args.network, [args.scenario], message=message) except HydraPluginError, e: write_progress(steps, steps) log.exception(e) err = PluginLib.create_xml_response("PyomoAuto", args.network, [args.scenario], errors=[e.message]) print err except Exception as e: write_progress(steps, steps) errors = [] if e.message == "": if hasattr(e, "strerror"): errors = [e.strerror] else: errors = [e.message] log.exception(e) err = PluginLib.create_xml_response("PyomoAuto", args.network, [args.scenario], errors=[e.message]) print err
parser.add_argument( '-c', '--session-id', help='''Session ID. If this does not exist, a login will be attempted based on details in config.''') return parser if __name__ == '__main__': parser = commandline_parser() args = parser.parse_args() jp_runner = ModelRunner(url=args.server_url, session_id=args.session_id) errors = [] try: write_output("Starting App") write_progress(1, jp_runner.num_steps) validate_plugin_xml(os.path.join(__location__, 'plugin.xml')) jp_runner.get_network_data(args.network_id, args.scenario_id) jp_runner.run_model() message = "Model Run Complete" except HydraPluginError as e: message = "An error has occurred" errors = [e.message] log.exception(e) except Exception, e: message = "An error has occurred" log.exception(e) errors = [e]
def export_network (self): ''' export the network from Hydra ''' write_progress(2, self.steps) net = self.connection.call('get_network', {'network_id':self.network_id, 'include_data': 'Y', 'template_id':self.template_id, 'scenario_ids':[self.scenario_id]}) log.info("Network retrieved") attrs = self.connection.call('get_all_attributes', {}) log.info("%s attributes retrieved", len(attrs)) self.net=net self.network= HydraNetwork() self.network.load(net , attrs) log.info("Loading net into network.") nodes_map=dict () for node in net.nodes: nodes_map[node.id]=node.name self.get_longest_node_link_name(); write_progress(3, self.steps) self.output_file_contents.append("# Network-ID: "+str(self.network_id)); self.output_file_contents.append("\n# Scenario-ID: "+str(self.scenario_id)); self.output_file_contents.append("\n#" + "*"*100) self.write_nodes() write_progress(4, self.steps) self.write_links(nodes_map) write_progress(5, self.steps) self.export_node_groups() nodes_types=self.network.get_node_types(template_id=self.template_id) links_types=self.network.get_link_types(template_id=self.template_id) self.export_node_types(nodes_types) self.export_links_types(links_types) write_progress(6, self.steps) if len(self.time_index)>0: self.output_file_contents.append('\nset time_step:=') for timestep in self.time_index.keys(): self.output_file_contents.append(" " +str(timestep)) self.output_file_contents.append(';\n') self.output_file_contents.append('\nset actual_time_step:=') for timestep in self.time_index.values(): self.output_file_contents.append(" " +str(timestep)) self.output_file_contents.append(';\n') write_progress(7, self.steps) if self.export_by_type is True: self.export_data_using_types(nodes_types, links_types) else: self.export_data_using_attributes()
netword_id = convert_to_int(args.network, "Network Id") scenario_id = convert_to_int(args.scenario, "scenario Id") link_export_flag = 'nn' if args.link_name is True: link_export_flag = 'l' network = export_data(args) vars, objs = run_model(args.output, args.model_file) actual_time_steps = read_inputData(args.output) import_result(args, vars, objs, actual_time_steps) message = "Run successfully" print PluginLib.create_xml_response('PyomoAuto', args.network, [args.scenario], message=message) except HydraPluginError, e: write_progress(steps, steps) log.exception(e) err = PluginLib.create_xml_response('PyomoAuto', args.network, [args.scenario], errors=[e.message]) print err except Exception as e: write_progress(steps, steps) errors = [] if e.message == '': if hasattr(e, 'strerror'): errors = [e.strerror] else: errors = [e.message] log.exception(e)
def import_network_topology(self, project_id=None): """ Read the file containing the network data and send it to the server. """ if self.json_net is None: self.fetch_remote_network() for node in self.json_net: props = node['properties'] node_type = props['type'] node_coords = node['geometry']['coordinates'] tmp_node_id = self.node_id.next() #TODO: HACK. WHy are there 2 coordinates for the node? if isinstance(node_coords[0], list): #log.debug("Using 1st coords of %s (%s)", node_coords, props['type']) node_coords = node_coords[0] #log.info("X=%s, y=%s",node_coords[0], node_coords[1]) node = dict( id = tmp_node_id, name = props['prmname'], x = str(node_coords[0]), #swap these if they are lat-long, not x-y y = str(node_coords[1]), description = props['description'], attributes = [], types = [{'template_id':self.template.id, 'id':node_type}] ) self.nodes[props['prmname']] = node #ATTRIBUTES #Find the matching type for t in self.template.types: if t.name == node_type: node['types'][0]['id'] = t.id #Assign the attributes to the node for tattr in t.typeattrs: node['attributes'].append( {'attr_id':tattr.attr_id} ) inlinks = [o['link_prmname'] for o in props.get('origins', [])] for linkname in inlinks: if linkname not in self.links: link = dict( id=self.link_id.next(), name = linkname, node_2_id = tmp_node_id, attributes = [], description = "", ) self.links[linkname] = link else: link = self.links[linkname] link['node_2_id'] = tmp_node_id outlinks = [o['link_prmname'] for o in props.get('terminals', [])] for linkname in outlinks: if linkname not in self.links: link = dict( id=self.link_id.next(), name = linkname, node_1_id = tmp_node_id, attributes = [], description = "", ) self.links[linkname] = link else: link = self.links[linkname] link['node_1_id'] = tmp_node_id node_groups = props['regions'] project = self.fetch_project(project_id) project_id = project.id write_output("Saving Network") write_progress(3, self.num_steps) for t in self.template.types: if t.name == 'HobbesNetwork': network_type = t.id hydra_network = { 'name' : "HOBBES Network (%s)"%datetime.now(), 'description' : "Hobbes Network, imported directly from the web API", 'nodes': self.nodes.values(), 'links': self.links.values(), 'project_id' : project_id, 'projection':'EPSG:2229', 'groups' : [], 'scenarios': [], 'types' : [{'template_id':self.template.id, 'id':network_type}] } #The network ID can be specified to get the network... self.network = self.connection.call('add_network', {'net':hydra_network}) return self.network
def import_network_topology(self, project_id=None): """ Read the file containing the network data and send it to the server. """ if self.json_net is None: self.fetch_remote_network() for node in self.json_net: props = node['properties'] node_type = props['type'] node_coords = node['geometry']['coordinates'] tmp_node_id = self.node_id.next() #TODO: HACK. WHy are there 2 coordinates for the node? if isinstance(node_coords[0], list): #log.debug("Using 1st coords of %s (%s)", node_coords, props['type']) node_coords = node_coords[0] #log.info("X=%s, y=%s",node_coords[0], node_coords[1]) node = dict( id=tmp_node_id, name=props['prmname'], x=str( node_coords[0]), #swap these if they are lat-long, not x-y y=str(node_coords[1]), description=props['description'], attributes=[], types=[{ 'template_id': self.template.id, 'id': node_type }]) self.nodes[props['prmname']] = node #ATTRIBUTES #Find the matching type for t in self.template.types: if t.name == node_type: node['types'][0]['id'] = t.id #Assign the attributes to the node for tattr in t.typeattrs: node['attributes'].append({'attr_id': tattr.attr_id}) inlinks = [o['link_prmname'] for o in props.get('origins', [])] for linkname in inlinks: if linkname not in self.links: link = dict( id=self.link_id.next(), name=linkname, node_2_id=tmp_node_id, attributes=[], description="", ) self.links[linkname] = link else: link = self.links[linkname] link['node_2_id'] = tmp_node_id outlinks = [o['link_prmname'] for o in props.get('terminals', [])] for linkname in outlinks: if linkname not in self.links: link = dict( id=self.link_id.next(), name=linkname, node_1_id=tmp_node_id, attributes=[], description="", ) self.links[linkname] = link else: link = self.links[linkname] link['node_1_id'] = tmp_node_id node_groups = props['regions'] project = self.fetch_project(project_id) project_id = project.id write_output("Saving Network") write_progress(3, self.num_steps) for t in self.template.types: if t.name == 'HobbesNetwork': network_type = t.id hydra_network = { 'name': "HOBBES Network (%s)" % datetime.now(), 'description': "Hobbes Network, imported directly from the web API", 'nodes': self.nodes.values(), 'links': self.links.values(), 'project_id': project_id, 'projection': 'EPSG:2229', 'groups': [], 'scenarios': [], 'types': [{ 'template_id': self.template.id, 'id': network_type }] } #The network ID can be specified to get the network... self.network = self.connection.call('add_network', {'net': hydra_network}) return self.network
def export_network(self): ''' export the network from Hydra ''' write_progress(2, self.steps) net = self.connection.call( 'get_network', { 'network_id': self.network_id, 'include_data': 'Y', 'template_id': self.template_id, 'scenario_ids': [self.scenario_id] }) log.info("Network retrieved") attrs = self.connection.call('get_all_attributes', {}) log.info("%s attributes retrieved", len(attrs)) self.net = net self.network = HydraNetwork() self.network.load(net, attrs) log.info("Loading net into network.") nodes_map = dict() for node in net.nodes: nodes_map[node.id] = node.name self.get_longest_node_link_name() write_progress(3, self.steps) self.output_file_contents.append("# Network-ID: " + str(self.network_id)) self.output_file_contents.append("\n# Scenario-ID: " + str(self.scenario_id)) self.output_file_contents.append("\n#" + "*" * 100) self.write_nodes() write_progress(4, self.steps) self.write_links(nodes_map) write_progress(5, self.steps) self.export_node_groups() nodes_types = self.network.get_node_types(template_id=self.template_id) links_types = self.network.get_link_types(template_id=self.template_id) self.export_node_types(nodes_types) self.export_links_types(links_types) write_progress(6, self.steps) if len(self.time_index) > 0: self.output_file_contents.append('\nset time_step:=') for timestep in self.time_index.keys(): self.output_file_contents.append(" " + str(timestep)) self.output_file_contents.append(';\n') self.output_file_contents.append('\nset actual_time_step:=') for timestep in self.time_index.values(): self.output_file_contents.append(" " + str(timestep)) self.output_file_contents.append(';\n') write_progress(7, self.steps) if self.export_by_type is True: self.export_data_using_types(nodes_types, links_types) else: self.export_data_using_attributes()
parser.add_argument('-c', '--session-id', help='''Session ID. If this does not exist, a login will be attempted based on details in config.''') return parser if __name__ == '__main__': parser = commandline_parser() args = parser.parse_args() network_importer = NetworkImporter(url=args.server_url, session_id=args.session_id) errors = [] network_id = None scenario_id = None try: write_output("Starting App") write_progress(1, network_importer.num_steps) validate_plugin_xml(os.path.join(__location__, 'plugin.xml')) network_importer.get__network() network_importer.get_template(args.template_id) network = network_importer.import_network(args.template_id, args.project_id) network_id = network.id scenario = network_importer.import_scenario() scenario_id = scenario.id message = "Import Complete" except HydraPluginError as e:
def import_network(self, template_id, project_id): write_output("Writing network to file") write_progress(3, self.num_steps) for j_node in self._network.nodes: log.info("Node: %s", j_node.component_type) node_type = self.type_name_map.get(j_node.component_type) y = j_node.y if y > 800000: y = y - 1000000 elif y > 400000: y = y - 400000 node = dict( id = self.node_ids.next(), name = j_node.name, description = " Node", x = str(j_node.x), y = str(y), attributes = [], types = [{'template_id':int(template_id), 'id':int(node_type.id)}] ) self.hydra_nodes[j_node.name] = node for j_link in self._network.links: link_type = self.type_name_map.get(j_link.component_type) log.info("Link: %s", j_link.component_type) link = dict( id = self.link_ids.next(), name = j_link.name, description = " Link", node_1_id = self.hydra_nodes[j_link.start_node.name]['id'], node_2_id = self.hydra_nodes[j_link.end_node.name]['id'], attributes = [], types = [{'template_id':int(template_id), 'id':link_type.id}] ) self.hydra_links[j_link.name] = link for j_inst in self._network.institutions: group_type = self.type_name_map.get(j_inst.component_type) log.info("Group: %s", j_inst.component_type) group = dict( id = self.group_ids.next(), name = j_inst.name, description = "A Model Institution", attributes = [], types = [{'template_id':int(template_id), 'id':group_type.id}] ) self.hydra_groups[j_inst.name] = group project = self.fetch_project(project_id) project_id = project.id network_type = self.type_name_map.get('Network') hydra_network = { 'name' : " Network (%s)"%datetime.now(), 'description' : " Network, imported directly from the prototype", 'nodes': self.hydra_nodes.values(), 'links': self.hydra_links.values(), 'project_id' : project_id, 'projection':'EPSG:2229', 'resourcegroups': self.hydra_groups.values(), 'scenarios': [], 'types' : [{'template_id':int(template_id), 'id':network_type.id}], } self.network = self.connection.call('add_network', {'net':hydra_network}) return self.network