def main(args=None, return_model=False, return_instance=False):
timer = StepTimer()
if args is None:
# combine default arguments read from options.txt file with
# additional arguments specified on the command line
args = get_option_file_args(extra_args=sys.argv[1:])
# Get options needed before any modules are loaded
pre_module_options = parse_pre_module_options(args)
# turn on post-mortem debugging mode if requested
# (from http://stackoverflow.com/a/1237407 ; more options available there)
if pre_module_options.debug:
def debug(type, value, tb):
import traceback
try:
from ipdb import pm
except ImportError:
from pdb import pm
traceback.print_exception(type, value, tb)
pm()
sys.excepthook = debug
# Write output to a log file if logging option is specified
if pre_module_options.log_run_to_file:
logs_dir = pre_module_options.logs_dir
else:
logs_dir = None # disables logging
with LogOutput(logs_dir):
# Look out for outdated inputs. This has to happen before modules.txt is
# parsed to avoid errors from incompatible files.
parser = _ArgumentParser(allow_abbrev=False, add_help=False)
add_module_args(parser)
module_options = parser.parse_known_args(args=args)[0]
if(os.path.exists(module_options.inputs_dir) and
do_inputs_need_upgrade(module_options.inputs_dir)):
do_upgrade = query_yes_no(
"Warning! Your inputs directory needs to be upgraded. "
"Do you want to auto-upgrade now? We'll keep a backup of "
"this current version."
)
if do_upgrade:
upgrade_inputs(module_options.inputs_dir)
else:
print("Inputs need upgrade. Consider `switch upgrade --help`. Exiting.")
stop_logging_output()
return -1
# build a module list based on configuration options, and add
# the current module (to register define_arguments callback)
modules = get_module_list(args)
# Patch pyomo if needed, to allow reconstruction of expressions.
# This must be done before the model is constructed.
patch_pyomo()
# Define the model
model = create_model(modules, args=args)
# Add any suffixes specified on the command line (usually only iis)
add_extra_suffixes(model)
# return the model as-is if requested
if return_model and not return_instance:
return model
if model.options.reload_prior_solution:
# TODO: allow a directory to be specified after --reload-prior-solution,
# otherwise use outputs_dir.
if not os.path.isdir(model.options.outputs_dir):
raise IOError("Directory specified for prior solution does not exist.")
# get a list of modules to iterate through
iterate_modules = get_iteration_list(model)
if model.options.verbose:
print("\n=======================================================================")
print("Switch {}, http://switch-model.org".format(switch_model.__version__))
print("=======================================================================")
print("Arguments:")
print(", ".join(k+"="+repr(v) for k, v in model.options.__dict__.items() if v))
print("Modules:\n"+", ".join(m for m in modules))
if iterate_modules:
print("Iteration modules:", iterate_modules)
print("=======================================================================\n")
print("Model created in {:.2f} s.".format(timer.step_time()))
print("Loading inputs...")
# create an instance (also reports time spent reading data and loading into model)
instance = model.load_inputs()
#### Below here, we refer to instance instead of model ####
instance.pre_solve()
if instance.options.verbose:
print("Total time spent constructing model: {:.2f} s.\n".format(timer.step_time()))
# return the instance as-is if requested
if return_instance:
if return_model:
return (model, instance)
else:
return instance
# make sure the outputs_dir exists (used by some modules during iterate)
# use a race-safe approach in case this code is run in parallel
try:
os.makedirs(instance.options.outputs_dir)
except OSError:
# directory probably exists already, but double-check
if not os.path.isdir(instance.options.outputs_dir):
raise
if instance.options.reload_prior_solution:
print('Loading prior solution...')
reload_prior_solution_from_pickle(instance, instance.options.outputs_dir)
if instance.options.verbose:
print(
'Loaded previous results into model instance in {:.2f} s.'
.format(timer.step_time())
)
else:
# solve the model (reports time for each step as it goes)
if iterate_modules:
if instance.options.verbose:
print("Iterating model...")
iterate(instance, iterate_modules)
else:
results = solve(instance)
if instance.options.verbose:
print("")
print("Optimization termination condition was {}.".format(
results.solver.termination_condition))
if str(results.solver.message) != '<undefined>':
print('Solver message: {}'.format(results.solver.message))
print("")
if instance.options.verbose:
timer.step_time() # restart counter for next step
if not instance.options.no_save_solution:
save_results(instance, instance.options.outputs_dir)
if instance.options.verbose:
print('Saved results in {:.2f} s.'.format(timer.step_time()))
# report results
# (repeated if model is reloaded, to automatically run any new export code)
if not instance.options.no_post_solve:
if instance.options.verbose:
print("Executing post solve functions...")
instance.post_solve()
if instance.options.verbose:
print("Post solve processing completed in {:.2f} s.".format(timer.step_time()))
# end of LogOutput block
if instance.options.interact:
m = instance # present the solved model as 'm' for convenience
banner = (
"\n"
"=======================================================================\n"
"Entering interactive Python shell.\n"
"Abstract model is in 'model' variable; \n"
"Solved instance is in 'instance' and 'm' variables.\n"
"Type ctrl-d or exit() to exit shell.\n"
"=======================================================================\n"
)
import code
code.interact(banner=banner, local=dict(list(globals().items()) + list(locals().items())))
########################################################### import switch_model.utilities as utilities import switch_model.financials as financials import switch_model.solve import sys, os from pyomo.environ import * print "loading model..." # Ideally, we would use the main codebase to generate the model, but the # mandatory switch argument parser is interferring with pysp's command line tools #model = switch_model.solve.main(return_model=True) module_list = switch_model.solve.get_module_list(args=None) model = utilities.create_model(module_list, args=[]) # The following code augments the model object with Expressions for the # Stage costs, which both runef and runph scripts need in order to build # the stochastic objective function. In this particular example, only # two stages are considered: Investment and Operation. These Expression # names must match exactly the StageCostVariable parameter defined for # each Stage in the ScenarioStructure.dat file. # The following two functions are defined explicitely, because since they # are nested inside another function in the financials module, they can't # be called from this script. def calc_tp_costs_in_period(m, t): return sum( getattr(m, tp_cost)[t] * m.tp_weight_in_year[t]
def main(args=None, return_model=False, return_instance=False):
timer = StepTimer()
if args is None:
# combine default arguments read from options.txt file with
# additional arguments specified on the command line
args = get_option_file_args(extra_args=sys.argv[1:])
# Get options needed before any modules are loaded
pre_module_options = parse_pre_module_options(args)
# turn on post-mortem debugging mode if requested
# (from http://stackoverflow.com/a/1237407 ; more options available there)
if pre_module_options.debug:
def debug(type, value, tb):
import traceback, pdb
traceback.print_exception(type, value, tb)
pdb.pm()
sys.excepthook = debug
# Write output to a log file if logging option is specified
if pre_module_options.log_run_to_file:
logs_dir = pre_module_options.logs_dir
else:
logs_dir = None # disables logging
with LogOutput(logs_dir):
# Look out for outdated inputs. This has to happen before modules.txt is
# parsed to avoid errors from incompatible files.
parser = _ArgumentParser(allow_abbrev=False, add_help=False)
add_module_args(parser)
module_options = parser.parse_known_args(args=args)[0]
if(os.path.exists(module_options.inputs_dir) and
do_inputs_need_upgrade(module_options.inputs_dir)):
do_upgrade = query_yes_no(
("Warning! Your inputs directory needs to be upgraded. "
"Do you want to auto-upgrade now? We'll keep a backup of "
"this current version."))
if do_upgrade:
upgrade_inputs(module_options.inputs_dir)
else:
print "Inputs need upgrade. Consider `switch upgrade --help`. Exiting."
stop_logging_output()
return -1
# build a module list based on configuration options, and add
# the current module (to register define_arguments callback)
modules = get_module_list(args)
# Patch pyomo if needed, to allow reconstruction of expressions.
# This must be done before the model is constructed.
patch_pyomo()
# Define the model
model = create_model(modules, args=args)
# Add any suffixes specified on the command line (usually only iis)
add_extra_suffixes(model)
# return the model as-is if requested
if return_model and not return_instance:
return model
if model.options.reload_prior_solution:
# TODO: allow a directory to be specified after --reload-prior-solution,
# otherwise use outputs_dir.
if not os.path.isdir(model.options.outputs_dir):
raise IOError("Directory specified for prior solution does not exist.")
# get a list of modules to iterate through
iterate_modules = get_iteration_list(model)
if model.options.verbose:
print "\n======================================================================="
print "SWITCH model created in {:.2f} s.\nArguments:".format(timer.step_time())
print ", ".join(k+"="+repr(v) for k, v in model.options.__dict__.items() if v)
print "Modules:\n"+", ".join(m for m in modules)
if iterate_modules:
print "Iteration modules:", iterate_modules
print "=======================================================================\n"
print "Loading inputs..."
# create an instance (also reports time spent reading data and loading into model)
instance = model.load_inputs()
#### Below here, we refer to instance instead of model ####
instance.pre_solve()
if instance.options.verbose:
print "Total time spent constructing model: {:.2f} s.\n".format(timer.step_time())
# return the instance as-is if requested
if return_instance:
if return_model:
return (model, instance)
else:
return instance
# make sure the outputs_dir exists (used by some modules during iterate)
# use a race-safe approach in case this code is run in parallel
try:
os.makedirs(instance.options.outputs_dir)
except OSError:
# directory probably exists already, but double-check
if not os.path.isdir(instance.options.outputs_dir):
raise
if instance.options.reload_prior_solution:
print('Loading prior solution...')
reload_prior_solution_from_pickle(instance, instance.options.outputs_dir)
if instance.options.verbose:
print(
'Loaded previous results into model instance in {:.2f} s.'
.format(timer.step_time())
)
else:
# solve the model (reports time for each step as it goes)
if iterate_modules:
if instance.options.verbose:
print "Iterating model..."
iterate(instance, iterate_modules)
else:
results = solve(instance)
if instance.options.verbose:
print ""
print results.solver.message
print "Optimization termination condition was {}.".format(
results.solver.termination_condition)
print ""
if instance.options.verbose:
timer.step_time() # restart counter for next step
if not instance.options.no_save_solution:
save_results(instance, instance.options.outputs_dir)
if instance.options.verbose:
print('Saved results in {:.2f} s.'.format(timer.step_time()))
# report results
# (repeated if model is reloaded, to automatically run any new export code)
if not instance.options.no_post_solve:
if instance.options.verbose:
print "Executing post solve functions..."
instance.post_solve()
if instance.options.verbose:
print "Post solve processing completed in {:.2f} s.".format(timer.step_time())
# end of LogOutput block
if instance.options.interact:
m = instance # present the solved model as 'm' for convenience
banner = (
"\n"
"=======================================================================\n"
"Entering interactive Python shell.\n"
"Abstract model is in 'model' variable; \n"
"Solved instance is in 'instance' and 'm' variables.\n"
"Type ctrl-d or exit() to exit shell.\n"
"=======================================================================\n"
)
import code
code.interact(banner=banner, local=dict(globals().items() + locals().items()))
"Operation": ["DispatchGen", "GenFuelUseRate"]
}
# List of scenario names
scenario_list = ["LowFuelCosts", "MediumFuelCosts", "HighFuelCosts"]
###########################################################
import switch_model.utilities as utilities
import switch_model.solve
import sys, os
from pyomo.environ import *
print "creating model for scenario input generation..."
module_list = switch_model.solve.get_module_list(args=None)
model = utilities.create_model(module_list)
print "model successfully created..."
print "loading inputs into model..."
instance = model.load_inputs(inputs_dir=inputs_dir)
print "inputs successfully loaded..."
def save_dat_files():
if not os.path.exists(os.path.join(inputs_dir, pysp_subdir)):
os.makedirs(os.path.join(inputs_dir, pysp_subdir))
##############
# RootNode.dat
###########################################################
import switch_model.utilities as utilities
import switch_model.financials as financials
import switch_model.solve
import sys, os
from pyomo.environ import *
print "loading model..."
# Ideally, we would use the main codebase to generate the model, but the
# mandatory switch argument parser is interferring with pysp's command line tools
#model = switch_model.solve.main(return_model=True)
module_list = switch_model.solve.get_module_list(args=None)
model = utilities.create_model(module_list, args=[])
# The following code augments the model object with Expressions for the
# Stage costs, which both runef and runph scripts need in order to build
# the stochastic objective function. In this particular example, only
# two stages are considered: Investment and Operation. These Expression
# names must match exactly the StageCostVariable parameter defined for
# each Stage in the ScenarioStructure.dat file.
# The following two functions are defined explicitely, because since they
# are nested inside another function in the financials module, they can't
# be called from this script.
def calc_tp_costs_in_period(m, t):
return sum(
getattr(m, tp_cost)[t] * m.tp_weight_in_year[t]
}
# List of scenario names
scenario_list = [
"LowFuelCosts", "MediumFuelCosts", "HighFuelCosts"]
###########################################################
import switch_model.utilities as utilities
import switch_model.solve
import sys, os
from pyomo.environ import *
print "creating model for scenario input generation..."
module_list = switch_model.solve.get_module_list(args=None)
model = utilities.create_model(module_list)
print "model successfully created..."
print "loading inputs into model..."
instance = model.load_inputs(inputs_dir=inputs_dir)
print "inputs successfully loaded..."
def save_dat_files():
if not os.path.exists(os.path.join(inputs_dir, pysp_subdir)):
os.makedirs(os.path.join(inputs_dir, pysp_subdir))
##############
# RootNode.dat
def main(args=None, return_model=False, return_instance=False):
start_time = time.time()
if args is None:
# combine default arguments read from options.txt file with
# additional arguments specified on the command line
args = get_option_file_args(extra_args=sys.argv[1:])
# Get options needed before any modules are loaded
pre_module_options = parse_pre_module_options(args)
# turn on post-mortem debugging mode if requested
# (from http://stackoverflow.com/a/1237407 ; more options available there)
if pre_module_options.debug:
def debug(type, value, tb):
import traceback, pdb
traceback.print_exception(type, value, tb)
pdb.pm()
sys.excepthook = debug
# Write output to a log file if logging option is specified
stdout_copy = sys.stdout # make a copy of current sys.stdout to return to eventually
if pre_module_options.log_run_to_file:
logging = Logging(pre_module_options.logs_dir)
print "logging run to " + str(logging.log_file_path)
sys.stdout = logging # assign private class to sys.stdout
else:
pass
# Look out for outdated inputs. This has to happen before modules.txt is
# parsed to avoid errors from incompatible files.
parser = _ArgumentParser(allow_abbrev=False, add_help=False)
add_module_args(parser)
module_options = parser.parse_known_args(args=args)[0]
if(os.path.exists(module_options.inputs_dir) and
do_inputs_need_upgrade(module_options.inputs_dir)):
do_upgrade = query_yes_no(
("Warning! Your inputs directory needs to be upgraded. "
"Do you want to auto-upgrade now? We'll keep a backup of "
"this current version."))
if do_upgrade:
upgrade_inputs(module_options.inputs_dir)
else:
print "Inputs need upgrade. Consider `switch upgrade --help`. Exiting."
sys.stdout = stdout_copy
return -1
# build a module list based on configuration options, and add
# the current module (to register define_arguments callback)
modules = get_module_list(args)
# Patch pyomo if needed, to allow reconstruction of expressions.
# This must be done before the model is constructed.
patch_pyomo()
# Define the model
model = create_model(modules, args=args)
# Add any suffixes specified on the command line (usually only iis)
add_extra_suffixes(model)
# return the model as-is if requested
if return_model and not return_instance:
return model
if model.options.reload_prior_solution:
if not os.path.isdir(model.options.outputs_dir):
raise IOError("Specified outputs directory for solution exploration does not exist.")
# get a list of modules to iterate through
iterate_modules = get_iteration_list(model)
if model.options.verbose:
creation_time = time.time()
print "\n======================================================================="
print "SWITCH model created in {:.2f} s.\nArguments:".format(creation_time - start_time)
print ", ".join(k+"="+repr(v) for k, v in model.options.__dict__.items() if v)
print "Modules:\n"+", ".join(m for m in modules)
if iterate_modules:
print "Iteration modules:", iterate_modules
print "=======================================================================\n"
print "Loading inputs..."
# create an instance
instance = model.load_inputs()
instance.pre_solve()
instantiation_time = time.time()
if model.options.verbose:
print "Inputs loaded in {:.2f} s.\n".format(instantiation_time - creation_time)
# return the instance as-is if requested
if return_instance:
if return_model:
return (model, instance)
else:
return instance
if model.options.reload_prior_solution:
# read variable values from previously solved model
import csv
var_objects = [c for c in instance.component_objects()
if isinstance(c,pyomo.core.base.Var)]
def _convert_if_numeric(s):
try:
return float(s)
except ValueError:
return s
for var in var_objects:
if '{}.tab'.format(var.name) not in os.listdir(model.options.outputs_dir):
raise RuntimeError("Tab output file for variable {} cannot be found in outputs directory. Exiting.".format(var.name))
with open(os.path.join(model.options.outputs_dir, '{}.tab'.format(var.name)),'r') as f:
reader = csv.reader(f, delimiter='\t')
# skip headers
next(reader)
for row in reader:
index = (_convert_if_numeric(i) for i in row[:-1])
var[index].value = float(row[-1])
print 'Loaded variable {} values into instance.'.format(var.name)
output_loading_time = time.time()
print 'Finished loading previous results into model instance in {:.2f} s.'.format(output_loading_time - instantiation_time)
else:
# make sure the outputs_dir exists (used by some modules during iterate)
# use a race-safe approach in case this code is run in parallel
try:
os.makedirs(model.options.outputs_dir)
except OSError:
# directory probably exists already, but double-check
if not os.path.isdir(model.options.outputs_dir):
raise
# solve the model
if iterate_modules:
if model.options.verbose:
print "Iterating model..."
iterate(instance, iterate_modules)
else:
results = solve(instance)
if model.options.verbose:
print "Optimization termination condition was {}.\n".format(
results.solver.termination_condition)
# report/save results
if model.options.verbose:
post_solve_start_time = time.time()
print "Executing post solve functions..."
instance.post_solve()
if model.options.verbose:
post_solve_end_time = time.time()
print "Post solve processing completed in {:.2f} s.".format(
post_solve_end_time - post_solve_start_time)
# return stdout to original
sys.stdout = stdout_copy
if model.options.interact or model.options.reload_prior_solution:
m = instance # present the solved model as 'm' for convenience
banner = (
"\n"
"=======================================================================\n"
"Entering interactive Python shell.\n"
"Abstract model is in 'model' variable; \n"
"Solved instance is in 'instance' and 'm' variables.\n"
"Type ctrl-d or exit() to exit shell.\n"
"=======================================================================\n"
)
import code
code.interact(banner=banner, local=dict(globals().items() + locals().items()))
def main(args=None, return_model=False, return_instance=False):
start_time = time.time()
if args is None:
# combine default arguments read from options.txt file with
# additional arguments specified on the command line
args = get_option_file_args(extra_args=sys.argv[1:])
# Get options needed before any modules are loaded
pre_module_options = parse_pre_module_options(args)
# turn on post-mortem debugging mode if requested
# (from http://stackoverflow.com/a/1237407 ; more options available there)
if pre_module_options.debug:
def debug(type, value, tb):
import traceback, pdb
traceback.print_exception(type, value, tb)
pdb.pm()
sys.excepthook = debug
# Write output to a log file if logging option is specified
stdout_copy = sys.stdout # make a copy of current sys.stdout to return to eventually
if pre_module_options.log_run_to_file:
logging = Logging(pre_module_options.logs_dir)
print "logging run to " + str(logging.log_file_path)
sys.stdout = logging # assign private class to sys.stdout
else:
pass
# Look out for outdated inputs. This has to happen before modules.txt is
# parsed to avoid errors from incompatible files.
parser = _ArgumentParser(allow_abbrev=False, add_help=False)
add_module_args(parser)
module_options = parser.parse_known_args(args=args)[0]
if (os.path.exists(module_options.inputs_dir)
and do_inputs_need_upgrade(module_options.inputs_dir)):
do_upgrade = query_yes_no(
("Warning! Your inputs directory needs to be upgraded. "
"Do you want to auto-upgrade now? We'll keep a backup of "
"this current version."))
if do_upgrade:
upgrade_inputs(module_options.inputs_dir)
else:
print "Inputs need upgrade. Consider `switch upgrade --help`. Exiting."
sys.stdout = stdout_copy
return -1
# build a module list based on configuration options, and add
# the current module (to register define_arguments callback)
modules = get_module_list(args)
# Patch pyomo if needed, to allow reconstruction of expressions.
# This must be done before the model is constructed.
patch_pyomo()
# Define the model
model = create_model(modules, args=args)
# Add any suffixes specified on the command line (usually only iis)
add_extra_suffixes(model)
# return the model as-is if requested
if return_model and not return_instance:
return model
if model.options.reload_prior_solution:
if not os.path.isdir(model.options.outputs_dir):
raise IOError(
"Specified outputs directory for solution exploration does not exist."
)
# get a list of modules to iterate through
iterate_modules = get_iteration_list(model)
if model.options.verbose:
creation_time = time.time()
print "\n======================================================================="
print "SWITCH model created in {:.2f} s.\nArguments:".format(
creation_time - start_time)
print ", ".join(k + "=" + repr(v)
for k, v in model.options.__dict__.items() if v)
print "Modules:\n" + ", ".join(m for m in modules)
if iterate_modules:
print "Iteration modules:", iterate_modules
print "=======================================================================\n"
print "Loading inputs..."
# create an instance
instance = model.load_inputs()
instance.pre_solve()
instantiation_time = time.time()
if model.options.verbose:
print "Inputs loaded in {:.2f} s.\n".format(instantiation_time -
creation_time)
#Paty's addition for debugging:
#embed()
# return the instance as-is if requested
if return_instance:
if return_model:
return (model, instance)
else:
return instance
if model.options.reload_prior_solution:
# read variable values from previously solved model
import csv
var_objects = [
c for c in instance.component_objects()
if isinstance(c, pyomo.core.base.Var)
]
def _convert_if_numeric(s):
try:
return float(s)
except ValueError:
return s
for var in var_objects:
if '{}.tab'.format(var.name) not in os.listdir(
model.options.outputs_dir):
raise RuntimeError(
"Tab output file for variable {} cannot be found in outputs directory. Exiting."
.format(var.name))
with open(
os.path.join(model.options.outputs_dir,
'{}.tab'.format(var.name)), 'r') as f:
reader = csv.reader(f, delimiter='\t')
# skip headers
next(reader)
for row in reader:
index = (_convert_if_numeric(i) for i in row[:-1])
var[index].value = float(row[-1])
print 'Loaded variable {} values into instance.'.format(var.name)
output_loading_time = time.time()
print 'Finished loading previous results into model instance in {:.2f} s.'.format(
output_loading_time - instantiation_time)
else:
# make sure the outputs_dir exists (used by some modules during iterate)
# use a race-safe approach in case this code is run in parallel
try:
os.makedirs(model.options.outputs_dir)
except OSError:
# directory probably exists already, but double-check
if not os.path.isdir(model.options.outputs_dir):
raise
# solve the model
if iterate_modules:
if model.options.verbose:
print "Iterating model..."
iterate(instance, iterate_modules)
else:
results = solve(instance)
if model.options.verbose:
print "Optimization termination condition was {}.\n".format(
results.solver.termination_condition)
#Paty's addition for debugging:
#embed()
# report/save results
if model.options.verbose:
post_solve_start_time = time.time()
print "Executing post solve functions..."
instance.post_solve()
if model.options.verbose:
post_solve_end_time = time.time()
print "Post solve processing completed in {:.2f} s.".format(
post_solve_end_time - post_solve_start_time)
# return stdout to original
sys.stdout = stdout_copy
if model.options.interact or model.options.reload_prior_solution:
m = instance # present the solved model as 'm' for convenience
banner = (
"\n"
"=======================================================================\n"
"Entering interactive Python shell.\n"
"Abstract model is in 'model' variable; \n"
"Solved instance is in 'instance' and 'm' variables.\n"
"Type ctrl-d or exit() to exit shell.\n"
"=======================================================================\n"
)
import code
code.interact(banner=banner,
local=dict(globals().items() + locals().items()))