def __init__(self, args, link_export_flag, steps):
if link_export_flag == 'l':
self.links_as_name = True
else:
self.links_as_name = False
self.steps = steps
self.use_gams_date_index = False
self.network_id = int(args.network)
self.scenario_id = int(args.scenario)
self.template_id = int(
args.template_id) if args.template_id is not None else None
self.output_file = args.output
self.export_by_type = args.export_by_type
self.time_index = []
write_progress(1, self.steps)
self.output_file_contents = []
self.output_file_contents.append("#%s\n" % ("*" * 78, ))
self.output_file_contents.append(
"# Data exported from Hydra using PyomoPlugin.\n")
self.output_file_contents.append(
"# (c) Copyright 2015, University of Manchester\n")
self.time_index = {}
self.connect(args)
def get_network_data(self, network_id, scenario_id):
"""
Retrieve the network, identify the parameters to set,
set them and run the model. Then identify the results
and set them back on the network.
"""
write_output("Retrieving Network")
write_progress(2, self.num_steps)
if network_id is not None:
if scenario_id is None:
raise HydraPluginError("A scenario ID must be specified.")
#The network ID can be specified to get the network...
try:
network_id = int(network_id)
network = self.connection.call(
'get_network', {
'network_id': network_id,
'scenario_ids': [int(scenario_id)]
})
write_output("Network retrieved")
except Exception, e:
log.exception(e)
raise HydraPluginError("Network %s not found." % network_id)
def save_file(self):
'''
save output file
'''
write_progress(8, self.steps)
log.info("writing data to file")
file = open(self.output_file, "w")
file.write("".join(self.output_file_contents))
file.close()
def save_file(self):
'''
save output file
'''
write_progress(8, self.steps)
log.info("writing data to file")
file = open(self.output_file, "w")
file.write("".join(self.output_file_contents))
file.close()
def fetch_remote_network(self):
"""
Request the hobbes network from the hobbes server
"""
write_output("Fetching Network")
write_progress(2, self.num_steps)
net_response = requests.get("http://cwn.casil.ucdavis.edu/network/get") #JSON Network
#http://cwn.casil.ucdavis.edu/excel/create?prmname=SR_CLE #XLS
if net_response.status_code != 200:
raise HydraPluginError("A connection error has occurred with status code: %s"%net_response.status_code)
self.json_net = json.loads(net_response.content)
def import_result(args, vars, objs, actual_time_steps, url, session_id):
write_progress(1, steps)
imp=Importer(vars, objs, actual_time_steps, url, session_id)
write_progress(2, steps)
imp.load_network(args.network, args.scenario)
write_progress(3, steps)
imp.import_res()
write_progress(4, steps)
imp.save()
def fetch_remote_network(self):
"""
Request the hobbes network from the hobbes server
"""
write_output("Fetching Network")
write_progress(2, self.num_steps)
net_response = requests.get(
"http://cwn.casil.ucdavis.edu/network/get") #JSON Network
#http://cwn.casil.ucdavis.edu/excel/create?prmname=SR_CLE #XLS
if net_response.status_code != 200:
raise HydraPluginError(
"A connection error has occurred with status code: %s" %
net_response.status_code)
self.json_net = json.loads(net_response.content)
def import_result(args, vars, objs, actual_time_steps):
write_progress(9, steps)
imp = Importer(vars, objs, actual_time_steps, args.server_url, args.session_id)
write_progress(10, steps)
imp.load_network(args.network, args.scenario)
write_progress(11, steps)
# imp.set_network(network)
imp.import_res()
write_progress(12, steps)
imp.save()
def import_result(args, vars, objs, actual_time_steps):
write_progress(9, steps)
imp = Importer(vars, objs, actual_time_steps, args.server_url,
args.session_id)
write_progress(10, steps)
imp.load_network(args.network, args.scenario)
write_progress(11, steps)
#imp.set_network(network)
imp.import_res()
write_progress(12, steps)
imp.save()
def run():
parser = commandline_parser()
args = parser.parse_args()
hobbes_importer = HobbesImporter(url=args.server_url,
session_id=args.session_id)
scenarios = []
errors = []
network_id = None
scenario_id = None
try:
write_progress(1, hobbes_importer.num_steps)
validate_plugin_xml(os.path.join(__location__, 'plugin.xml'))
#This step is to avoid doing the request to make the template and
#then again for the data.
hobbes_importer.fetch_remote_network()
if args.template_id is None:
tmpl = HobbesTemplateBuilder()
tmpl.convert(hobbes_importer.json_net)
hobbes_importer.upload_template()
else:
hobbes_importer.fetch_template(args.template_id)
net = hobbes_importer.import_network_topology(args.project_id)
scenario = hobbes_importer.import_data()
#scenarios = [s.id for s in net.scenarios]
network_id = net.id
scenario_id = scenario.id
message = "Import complete"
except HydraPluginError as e:
message = "An error has occurred"
errors = [e.message]
log.exception(e)
except Exception, e:
message = "An unknown error has occurred"
log.exception(e)
errors = [e]
def run():
parser = commandline_parser()
args = parser.parse_args()
hobbes_importer = HobbesImporter(url=args.server_url, session_id=args.session_id)
scenarios = []
errors = []
network_id = None
scenario_id = None
try:
write_progress(1, hobbes_importer.num_steps)
validate_plugin_xml(os.path.join(__location__, 'plugin.xml'))
#This step is to avoid doing the request to make the template and
#then again for the data.
hobbes_importer.fetch_remote_network()
if args.template_id is None:
tmpl = HobbesTemplateBuilder()
tmpl.convert(hobbes_importer.json_net)
hobbes_importer.upload_template()
else:
hobbes_importer.fetch_template(args.template_id)
net = hobbes_importer.import_network_topology(args.project_id)
scenario = hobbes_importer.import_data()
#scenarios = [s.id for s in net.scenarios]
network_id = net.id
scenario_id = scenario.id
message = "Import complete"
except HydraPluginError as e:
message="An error has occurred"
errors = [e.message]
log.exception(e)
except Exception, e:
message="An unknown error has occurred"
log.exception(e)
errors = [e]
def __init__(self, args, link_export_flag, steps):
if link_export_flag == 'l':
self.links_as_name = True
else:
self.links_as_name = False
self.steps=steps
self.use_gams_date_index=False
self.network_id = int(args.network)
self.scenario_id = int(args.scenario)
self.template_id = int(args.template_id) if args.template_id is not None else None
self.output_file = args.output
self.export_by_type =args.export_by_type
self.time_index = []
write_progress(1, self.steps)
self.output_file_contents=[];
self.output_file_contents.append("#%s\n"%("*"*78,))
self.output_file_contents.append("# Data exported from Hydra using PyomoPlugin.\n")
self.output_file_contents.append("# (c) Copyright 2015, University of Manchester\n")
self.time_index = {}
self.connect(args)
def write_network(self, network, target_dir):
write_output("Writing network to file")
write_progress(3, self.num_steps)
if self.as_xml is False:
file_name = "network_%s.json" % (network['name'])
self.files.append(os.path.join(target_dir, file_name))
network_file = open(os.path.join(target_dir, file_name), 'w')
network_file.write(
json.dumps(network,
sort_keys=True,
indent=4,
separators=(',', ': ')))
else:
file_name = "network_%s.xml" % (network['name'])
self.files.append(os.path.join(target_dir, file_name))
network_file = open(os.path.join(target_dir, file_name), 'w')
json_network = {'network': network}
network_file.write(json2xml(json_network))
write_output("Network Written to %s " % (target_dir))
try:
check_args(args)
netword_id = convert_to_int(args.network, "Network Id")
scenario_id = convert_to_int(args.scenario, "scenario Id")
link_export_flag = "nn"
if args.link_name is True:
link_export_flag = "l"
network = export_data(args)
vars, objs = run_model(args.output, args.model_file)
actual_time_steps = read_inputData(args.output)
import_result(args, vars, objs, actual_time_steps)
message = "Run successfully"
print PluginLib.create_xml_response("PyomoAuto", args.network, [args.scenario], message=message)
except HydraPluginError, e:
write_progress(steps, steps)
log.exception(e)
err = PluginLib.create_xml_response("PyomoAuto", args.network, [args.scenario], errors=[e.message])
print err
except Exception as e:
write_progress(steps, steps)
errors = []
if e.message == "":
if hasattr(e, "strerror"):
errors = [e.strerror]
else:
errors = [e.message]
log.exception(e)
err = PluginLib.create_xml_response("PyomoAuto", args.network, [args.scenario], errors=[e.message])
print err
parser.add_argument(
'-c',
'--session-id',
help='''Session ID. If this does not exist, a login will be
attempted based on details in config.''')
return parser
if __name__ == '__main__':
parser = commandline_parser()
args = parser.parse_args()
jp_runner = ModelRunner(url=args.server_url, session_id=args.session_id)
errors = []
try:
write_output("Starting App")
write_progress(1, jp_runner.num_steps)
validate_plugin_xml(os.path.join(__location__, 'plugin.xml'))
jp_runner.get_network_data(args.network_id, args.scenario_id)
jp_runner.run_model()
message = "Model Run Complete"
except HydraPluginError as e:
message = "An error has occurred"
errors = [e.message]
log.exception(e)
except Exception, e:
message = "An error has occurred"
log.exception(e)
errors = [e]
def export_network (self):
'''
export the network from Hydra
'''
write_progress(2, self.steps)
net = self.connection.call('get_network', {'network_id':self.network_id,
'include_data': 'Y',
'template_id':self.template_id,
'scenario_ids':[self.scenario_id]})
log.info("Network retrieved")
attrs = self.connection.call('get_all_attributes', {})
log.info("%s attributes retrieved", len(attrs))
self.net=net
self.network= HydraNetwork()
self.network.load(net , attrs)
log.info("Loading net into network.")
nodes_map=dict ()
for node in net.nodes:
nodes_map[node.id]=node.name
self.get_longest_node_link_name();
write_progress(3, self.steps)
self.output_file_contents.append("# Network-ID: "+str(self.network_id));
self.output_file_contents.append("\n# Scenario-ID: "+str(self.scenario_id));
self.output_file_contents.append("\n#" + "*"*100)
self.write_nodes()
write_progress(4, self.steps)
self.write_links(nodes_map)
write_progress(5, self.steps)
self.export_node_groups()
nodes_types=self.network.get_node_types(template_id=self.template_id)
links_types=self.network.get_link_types(template_id=self.template_id)
self.export_node_types(nodes_types)
self.export_links_types(links_types)
write_progress(6, self.steps)
if len(self.time_index)>0:
self.output_file_contents.append('\nset time_step:=')
for timestep in self.time_index.keys():
self.output_file_contents.append(" " +str(timestep))
self.output_file_contents.append(';\n')
self.output_file_contents.append('\nset actual_time_step:=')
for timestep in self.time_index.values():
self.output_file_contents.append(" " +str(timestep))
self.output_file_contents.append(';\n')
write_progress(7, self.steps)
if self.export_by_type is True:
self.export_data_using_types(nodes_types, links_types)
else:
self.export_data_using_attributes()
netword_id = convert_to_int(args.network, "Network Id")
scenario_id = convert_to_int(args.scenario, "scenario Id")
link_export_flag = 'nn'
if args.link_name is True:
link_export_flag = 'l'
network = export_data(args)
vars, objs = run_model(args.output, args.model_file)
actual_time_steps = read_inputData(args.output)
import_result(args, vars, objs, actual_time_steps)
message = "Run successfully"
print PluginLib.create_xml_response('PyomoAuto',
args.network, [args.scenario],
message=message)
except HydraPluginError, e:
write_progress(steps, steps)
log.exception(e)
err = PluginLib.create_xml_response('PyomoAuto',
args.network, [args.scenario],
errors=[e.message])
print err
except Exception as e:
write_progress(steps, steps)
errors = []
if e.message == '':
if hasattr(e, 'strerror'):
errors = [e.strerror]
else:
errors = [e.message]
log.exception(e)
def import_network_topology(self, project_id=None):
"""
Read the file containing the network data and send it to
the server.
"""
if self.json_net is None:
self.fetch_remote_network()
for node in self.json_net:
props = node['properties']
node_type = props['type']
node_coords = node['geometry']['coordinates']
tmp_node_id = self.node_id.next()
#TODO: HACK. WHy are there 2 coordinates for the node?
if isinstance(node_coords[0], list):
#log.debug("Using 1st coords of %s (%s)", node_coords, props['type'])
node_coords = node_coords[0]
#log.info("X=%s, y=%s",node_coords[0], node_coords[1])
node = dict(
id = tmp_node_id,
name = props['prmname'],
x = str(node_coords[0]), #swap these if they are lat-long, not x-y
y = str(node_coords[1]),
description = props['description'],
attributes = [],
types = [{'template_id':self.template.id,
'id':node_type}]
)
self.nodes[props['prmname']] = node
#ATTRIBUTES
#Find the matching type
for t in self.template.types:
if t.name == node_type:
node['types'][0]['id'] = t.id
#Assign the attributes to the node
for tattr in t.typeattrs:
node['attributes'].append(
{'attr_id':tattr.attr_id}
)
inlinks = [o['link_prmname'] for o in props.get('origins', [])]
for linkname in inlinks:
if linkname not in self.links:
link = dict(
id=self.link_id.next(),
name = linkname,
node_2_id = tmp_node_id,
attributes = [],
description = "",
)
self.links[linkname] = link
else:
link = self.links[linkname]
link['node_2_id'] = tmp_node_id
outlinks = [o['link_prmname'] for o in props.get('terminals', [])]
for linkname in outlinks:
if linkname not in self.links:
link = dict(
id=self.link_id.next(),
name = linkname,
node_1_id = tmp_node_id,
attributes = [],
description = "",
)
self.links[linkname] = link
else:
link = self.links[linkname]
link['node_1_id'] = tmp_node_id
node_groups = props['regions']
project = self.fetch_project(project_id)
project_id = project.id
write_output("Saving Network")
write_progress(3, self.num_steps)
for t in self.template.types:
if t.name == 'HobbesNetwork':
network_type = t.id
hydra_network = {
'name' : "HOBBES Network (%s)"%datetime.now(),
'description' : "Hobbes Network, imported directly from the web API",
'nodes': self.nodes.values(),
'links': self.links.values(),
'project_id' : project_id,
'projection':'EPSG:2229',
'groups' : [],
'scenarios': [],
'types' : [{'template_id':self.template.id,
'id':network_type}]
}
#The network ID can be specified to get the network...
self.network = self.connection.call('add_network', {'net':hydra_network})
return self.network
def import_network_topology(self, project_id=None):
"""
Read the file containing the network data and send it to
the server.
"""
if self.json_net is None:
self.fetch_remote_network()
for node in self.json_net:
props = node['properties']
node_type = props['type']
node_coords = node['geometry']['coordinates']
tmp_node_id = self.node_id.next()
#TODO: HACK. WHy are there 2 coordinates for the node?
if isinstance(node_coords[0], list):
#log.debug("Using 1st coords of %s (%s)", node_coords, props['type'])
node_coords = node_coords[0]
#log.info("X=%s, y=%s",node_coords[0], node_coords[1])
node = dict(
id=tmp_node_id,
name=props['prmname'],
x=str(
node_coords[0]), #swap these if they are lat-long, not x-y
y=str(node_coords[1]),
description=props['description'],
attributes=[],
types=[{
'template_id': self.template.id,
'id': node_type
}])
self.nodes[props['prmname']] = node
#ATTRIBUTES
#Find the matching type
for t in self.template.types:
if t.name == node_type:
node['types'][0]['id'] = t.id
#Assign the attributes to the node
for tattr in t.typeattrs:
node['attributes'].append({'attr_id': tattr.attr_id})
inlinks = [o['link_prmname'] for o in props.get('origins', [])]
for linkname in inlinks:
if linkname not in self.links:
link = dict(
id=self.link_id.next(),
name=linkname,
node_2_id=tmp_node_id,
attributes=[],
description="",
)
self.links[linkname] = link
else:
link = self.links[linkname]
link['node_2_id'] = tmp_node_id
outlinks = [o['link_prmname'] for o in props.get('terminals', [])]
for linkname in outlinks:
if linkname not in self.links:
link = dict(
id=self.link_id.next(),
name=linkname,
node_1_id=tmp_node_id,
attributes=[],
description="",
)
self.links[linkname] = link
else:
link = self.links[linkname]
link['node_1_id'] = tmp_node_id
node_groups = props['regions']
project = self.fetch_project(project_id)
project_id = project.id
write_output("Saving Network")
write_progress(3, self.num_steps)
for t in self.template.types:
if t.name == 'HobbesNetwork':
network_type = t.id
hydra_network = {
'name': "HOBBES Network (%s)" % datetime.now(),
'description':
"Hobbes Network, imported directly from the web API",
'nodes': self.nodes.values(),
'links': self.links.values(),
'project_id': project_id,
'projection': 'EPSG:2229',
'groups': [],
'scenarios': [],
'types': [{
'template_id': self.template.id,
'id': network_type
}]
}
#The network ID can be specified to get the network...
self.network = self.connection.call('add_network',
{'net': hydra_network})
return self.network
def export_network(self):
'''
export the network from Hydra
'''
write_progress(2, self.steps)
net = self.connection.call(
'get_network', {
'network_id': self.network_id,
'include_data': 'Y',
'template_id': self.template_id,
'scenario_ids': [self.scenario_id]
})
log.info("Network retrieved")
attrs = self.connection.call('get_all_attributes', {})
log.info("%s attributes retrieved", len(attrs))
self.net = net
self.network = HydraNetwork()
self.network.load(net, attrs)
log.info("Loading net into network.")
nodes_map = dict()
for node in net.nodes:
nodes_map[node.id] = node.name
self.get_longest_node_link_name()
write_progress(3, self.steps)
self.output_file_contents.append("# Network-ID: " +
str(self.network_id))
self.output_file_contents.append("\n# Scenario-ID: " +
str(self.scenario_id))
self.output_file_contents.append("\n#" + "*" * 100)
self.write_nodes()
write_progress(4, self.steps)
self.write_links(nodes_map)
write_progress(5, self.steps)
self.export_node_groups()
nodes_types = self.network.get_node_types(template_id=self.template_id)
links_types = self.network.get_link_types(template_id=self.template_id)
self.export_node_types(nodes_types)
self.export_links_types(links_types)
write_progress(6, self.steps)
if len(self.time_index) > 0:
self.output_file_contents.append('\nset time_step:=')
for timestep in self.time_index.keys():
self.output_file_contents.append(" " + str(timestep))
self.output_file_contents.append(';\n')
self.output_file_contents.append('\nset actual_time_step:=')
for timestep in self.time_index.values():
self.output_file_contents.append(" " + str(timestep))
self.output_file_contents.append(';\n')
write_progress(7, self.steps)
if self.export_by_type is True:
self.export_data_using_types(nodes_types, links_types)
else:
self.export_data_using_attributes()
parser.add_argument('-c', '--session-id',
help='''Session ID. If this does not exist, a login will be
attempted based on details in config.''')
return parser
if __name__ == '__main__':
parser = commandline_parser()
args = parser.parse_args()
network_importer = NetworkImporter(url=args.server_url, session_id=args.session_id)
errors = []
network_id = None
scenario_id = None
try:
write_output("Starting App")
write_progress(1, network_importer.num_steps)
validate_plugin_xml(os.path.join(__location__, 'plugin.xml'))
network_importer.get__network()
network_importer.get_template(args.template_id)
network = network_importer.import_network(args.template_id, args.project_id)
network_id = network.id
scenario = network_importer.import_scenario()
scenario_id = scenario.id
message = "Import Complete"
except HydraPluginError as e:
def import_network(self, template_id, project_id):
write_output("Writing network to file")
write_progress(3, self.num_steps)
for j_node in self._network.nodes:
log.info("Node: %s", j_node.component_type)
node_type = self.type_name_map.get(j_node.component_type)
y = j_node.y
if y > 800000:
y = y - 1000000
elif y > 400000:
y = y - 400000
node = dict(
id = self.node_ids.next(),
name = j_node.name,
description = " Node",
x = str(j_node.x),
y = str(y),
attributes = [],
types = [{'template_id':int(template_id), 'id':int(node_type.id)}]
)
self.hydra_nodes[j_node.name] = node
for j_link in self._network.links:
link_type = self.type_name_map.get(j_link.component_type)
log.info("Link: %s", j_link.component_type)
link = dict(
id = self.link_ids.next(),
name = j_link.name,
description = " Link",
node_1_id = self.hydra_nodes[j_link.start_node.name]['id'],
node_2_id = self.hydra_nodes[j_link.end_node.name]['id'],
attributes = [],
types = [{'template_id':int(template_id), 'id':link_type.id}]
)
self.hydra_links[j_link.name] = link
for j_inst in self._network.institutions:
group_type = self.type_name_map.get(j_inst.component_type)
log.info("Group: %s", j_inst.component_type)
group = dict(
id = self.group_ids.next(),
name = j_inst.name,
description = "A Model Institution",
attributes = [],
types = [{'template_id':int(template_id), 'id':group_type.id}]
)
self.hydra_groups[j_inst.name] = group
project = self.fetch_project(project_id)
project_id = project.id
network_type = self.type_name_map.get('Network')
hydra_network = {
'name' : " Network (%s)"%datetime.now(),
'description' : " Network, imported directly from the prototype",
'nodes': self.hydra_nodes.values(),
'links': self.hydra_links.values(),
'project_id' : project_id,
'projection':'EPSG:2229',
'resourcegroups': self.hydra_groups.values(),
'scenarios': [],
'types' : [{'template_id':int(template_id), 'id':network_type.id}],
}
self.network = self.connection.call('add_network', {'net':hydra_network})
return self.network