def export_network(is_licensed):
exporter = GAMSExporter(args)
write_progress(2, steps)
exporter.get_network(is_licensed)
write_progress(3, steps)
exporter.export_network()
if (args.gams_date_time_index is True):
exporter.use_gams_date_index = True
write_progress(4, steps)
exporter.write_time_index()
if args.export_by_type is True:
exporter.export_data_using_types()
else:
exporter.export_data_using_attributes()
write_progress(5, steps)
write_output("Writing output file")
exporter.write_file()
return exporter
def run_gams_model(args):
log.info("Running GAMS model .....")
cur_time = datetime.now().replace(microsecond=0)
write_progress(6, steps)
working_directory = os.path.dirname(args.gms_file)
if working_directory == '':
working_directory = '.'
model = GamsModel(args.gams_path, working_directory)
write_progress(7, steps)
model.add_job(args.gms_file)
write_progress(8, steps)
model.run()
write_progress(9, steps)
log.info("Running GAMS model finsihed")
# if result file is not provided, it looks for it automatically at GAMS WD
if args.gdx_file is None:
log.info("Extracting results from %s.", working_directory)
files_list = get_files_list(working_directory, '.gdx')
for file_ in files_list:
dt = parser.parse(files_list[file_])
delta = (dt - cur_time).total_seconds()
if delta >= 0:
args.gdx_file = os.path.join(working_directory, file_)
if args.gdx_file is None:
raise HydraPluginError('Result file is not provided/found.')
def read_results(is_licensed, args, network, connection):
"""
Instantiate a GAMSImport class, assign the network, read the
gdx and gms files, update the network's data and then save
the network.
"""
write_progress(10, steps)
gdximport = GAMSImporter(args, connection)
write_progress(11, steps)
gdximport.load_gams_file(args.gms_file)
write_progress(12, steps)
gdximport.set_network(is_licensed, network)
write_progress(13, steps)
gdximport.parse_time_index()
write_progress(14, steps)
gdximport.open_gdx_file(args.gdx_file)
write_progress(15, steps)
gdximport.read_gdx_data()
write_progress(16, steps)
gdximport.parse_variables('variables')
gdximport.parse_variables('positive variables')
gdximport.parse_variables('positive variable')
gdximport.parse_variables('binary variables')
gdximport.parse_variables('parameters')
write_progress(17, steps)
gdximport.assign_attr_data()
write_progress(18, steps)
gdximport.save()
elif args.output == None:
args.output = get_input_file_name(args.gms_file)
if args.output is None:
raise HydraPluginError('No output file specified')
elif os.path.exists(os.path.dirname(os.path.realpath(
args.output))) == False:
raise HydraPluginError('Output file directory ' +
os.path.dirname(args.output) +
' does not exist.')
if __name__ == '__main__':
try:
is_licensed = check_lic()
steps = 18
write_progress(1, steps)
cmd_parser = commandline_parser()
args = cmd_parser.parse_args()
check_args(args)
exporter = export_network(is_licensed)
run_gams_model(args)
#if the mode is Auto, it will get the network from the exporter
read_results(is_licensed, args, exporter.hydranetwork,
exporter.connection)
message = "Run successfully"
errors = []
except HydraPluginError as e:
log.exception(e)
write_progress(steps, steps)
errors = [e.message]
def import_results(is_licensed, args):
write_progress(1, steps)
gdximport = GAMSImporter(args)
write_progress(2, steps)
gdximport.load_network(is_licensed)
write_progress(3, steps)
gdximport.load_gams_file(args.gms_file)
write_progress(4, steps)
gdximport.parse_time_index()
write_progress(5, steps)
gdximport.open_gdx_file(args.gdx_file)
write_progress(6, steps)
gdximport.read_gdx_data()
write_progress(7, steps)
gdximport.parse_variables('variables')
gdximport.parse_variables('positive variables')
gdximport.parse_variables('positive variable')
gdximport.parse_variables('binary variables')
gdximport.parse_variables('parameters')
write_progress(8, steps)
gdximport.assign_attr_data()
write_progress(9, steps)
gdximport.save()
def export(self, network_id, scenario_id, output_folder):
"""
Export a network (and possibly a scenario) to a folder. If the
scenario and output folders are not specified, all the scenarios
will be exported and the output location will be the desktop.
"""
write_output("Retrieving Network")
write_progress(2, self.num_steps)
if network_id is not None:
#The network ID can be specified to get the network...
try:
network_id = int(network_id)
x = time.time()
network = self.call('get_network', {'network_id':network_id})
log.info("Network retrieved in %s", time.time()-x)
except:
raise HydraPluginError("Network %s not found."%network_id)
else:
raise HydraPluginError("A network ID must be specified!")
if output_folder is None:
log.info("No output folder specified. Defaulting to desktop.")
output_folder = os.path.expanduser("~/Desktop")
elif not os.path.exists(output_folder):
raise HydraPluginError("Output folder %s does not exist"%output_folder)
network_dir = os.path.join(output_folder, "network_%s"%(network.name).replace(" ", "_"))
if not os.path.exists(network_dir):
os.mkdir(network_dir)
else:
logging.info("%s already exists", network_dir)
for export_num in range(100):
new_network_dir = os.path.join(output_folder, "%s(%s)"%(network_dir,export_num))
if not os.path.exists(new_network_dir):
logging.info("exporting to %s", new_network_dir)
os.mkdir(new_network_dir)
network_dir = new_network_dir
break
network.network_dir = network_dir
if network.scenarios is None:
raise HydraPluginError("Network %s has no scenarios!"%(network))
if scenario_id is not None:
write_progress(3, self.num_steps)
for scenario in network.scenarios:
if int(scenario.id) == int(scenario_id):
log.info("Exporting Scenario %s"%(scenario.name))
self.export_network(network, scenario)
break
else:
raise HydraPluginError("No scenario with ID %s found"%(args.scenario))
else:
log.info("No Scenario specified, exporting them all!")
for scenario in network.scenarios:
log.info("Exporting Scenario %s"%(scenario.name))
self.export_network(network, scenario)
self.files.append(network_dir)
def export_network(self, network, scenario):
"""
Write the output files based on the given network and scenario.
"""
write_output("Exporting network")
log.info("\n************NETWORK****************")
scenario.target_dir = os.path.join(network.network_dir, scenario.name.replace(' ', '_'))
if not os.path.exists(scenario.target_dir):
os.mkdir(scenario.target_dir)
network_file = open(os.path.join(scenario.target_dir, "network.csv"), 'w')
network_attributes = self.get_resource_attributes([network])
network_attributes_string = ""
if len(network_attributes) > 0:
network_attributes_string = ',%s'%(','.join(network_attributes.values()))
network_heading = "ID, Name, Type, Nodes, Links, Groups, Rules%s, Description\n" % (network_attributes_string)
metadata_heading = "Name %s\n"%(network_attributes_string)
network_attr_units = []
for attr_id, attr_name in network_attributes.items():
network_attr_units.append(self.get_attr_unit(scenario, attr_id, attr_name))
network_units_heading = "Units,,,,,,,,,,%s\n"%(','.join(network_attr_units))
values = ["" for attr_id in network_attributes.keys()]
metadata_placeholder = ["" for attr_id in network_attributes.keys()]
if network.attributes is not None:
for r_attr in network.attributes:
attr_name = network_attributes[r_attr.attr_id]
value, metadata = self.get_attr_value(scenario, r_attr, attr_name, network.name)
idx = network_attributes.keys().index(r_attr.attr_id)
values[idx] = value
metadata_placeholder[idx] = metadata
if network.types is not None and len(network.types) > 0:
net_type = network.types[0]['name']
else:
net_type = ""
#Leave the links, groups and rules blank for now, as there may not be
#any links, groups or rules.
network_data = {
"id" : network.id,
"name" : network.name,
"type" : net_type,
"nodes" : "",
"links" : "",
"groups" : "",
"rules" : "",
"values" : ",%s"%(",".join(values)) if len(values) > 0 else "",
"description" : network.description,
}
write_progress(4, self.num_steps)
node_map = dict()
if network.nodes:
node_map = self.export_nodes(scenario, network.nodes)
network_data['nodes'] = "nodes.csv"
else:
log.warning("Network has no nodes!")
write_progress(5, self.num_steps)
link_map = dict()
if network.links:
link_map = self.export_links(scenario, network.links, node_map)
network_data['links'] = "links.csv"
else:
log.warning("Network has no links!")
write_progress(6, self.num_steps)
group_map = dict()
if network.resourcegroups:
group_map = self.export_resourcegroups(scenario, network.resourcegroups, node_map, link_map)
network_data['groups'] = "groups.csv"
else:
log.warning("Network has no resourcegroups.")
write_progress(7, self.num_steps)
rules = self.export_rules(scenario, node_map, link_map, group_map)
if len(rules) > 0:
network_data['rules'] = "rules.csv"
network_entry = "%(id)s,%(name)s,%(type)s,%(nodes)s,%(links)s,%(groups)s,%(rules)s%(values)s,%(description)s\n"%network_data
if scenario.get('start_time') is not None and \
scenario.get('end_time') is not None and\
scenario.get('time_step') is not None:
network_heading = "ID, Name, Type, Nodes, Links, Groups, Rules, starttime, endtime, timestep %s, Description\n" % (network_attributes_string)
network_data['starttime'] = scenario.start_time
network_data['endtime'] = scenario.end_time
network_data['timestep'] = scenario.time_step
network_entry = "%(id)s,%(name)s,%(type)s,%(nodes)s,%(links)s,%(groups)s,%(rules)s,%(starttime)s,%(endtime)s,%(timestep)s%(values)s,%(description)s\n"%network_data
log.info("Exporting network metadata")
if metadata_placeholder.count("") != len(metadata_placeholder):
warnings = self.write_metadata(os.path.join(scenario.target_dir, 'network_metadata.csv'),
metadata_heading,
[(network.name, metadata_placeholder)])
self.warnings.extend(warnings)
network_file.write(network_heading)
network_file.write(network_units_heading)
network_file.write(network_entry)
log.info("Network export complete")
log.info("networks written to file: %s", network_file.name)
to UTC.''')
parser.add_argument('-u', '--server-url',
help='''Specify the URL of the server to which this
plug-in connects.''')
parser.add_argument('-c', '--session_id',
help='''Session ID. If this does not exist, a login will be
attempted based on details in config.''')
return parser
if __name__ == '__main__':
parser = commandline_parser()
args = parser.parse_args()
csv = ExportCSV(url=args.server_url, session_id=args.session_id)
try:
write_progress(1, csv.num_steps)
validate_plugin_xml(os.path.join(__location__, 'plugin.xml'))
if args.timezone is not None:
csv.timezone = pytz.timezone(args.timezone)
csv.export(args.network_id, args.scenario_id, args.output_folder)
message = "Export complete"
except HydraPluginError as e:
message="An error has occurred"
errors = [e.message]
log.exception(e)
except Exception, e:
message="An error has occurred"
log.exception(e)