def test_upgrade():
temp_dir = tempfile.mkdtemp(prefix='switch_test_')
example_name = os.path.basename(os.path.normpath(example_dir))
upgrade_dir = os.path.join(temp_dir, example_name)
shutil.copytree(example_dir, upgrade_dir, ignore=shutil.ignore_patterns('outputs'))
upgrade_dir_inputs = os.path.join(upgrade_dir, 'inputs')
upgrade_dir_outputs = os.path.join(upgrade_dir, 'outputs')
switch_model.upgrade.manager.set_verbose(False)
try:
# Custom python modules may be in the example's working directory
upgrade_inputs(upgrade_dir_inputs)
sys.path.append(upgrade_dir)
switch_model.solve.main([
'--inputs-dir', upgrade_dir_inputs,
'--outputs-dir', upgrade_dir_outputs])
total_cost = read_file(os.path.join(upgrade_dir_outputs, 'total_cost.txt'))
finally:
if upgrade_dir in sys.path: # code above may have failed before appending
sys.path.remove(upgrade_dir)
_remove_temp_dir(temp_dir)
expectation_file = get_expectation_path(example_dir)
if UPDATE_EXPECTATIONS:
write_file(expectation_file, total_cost)
else:
expected = float(read_file(expectation_file))
actual = float(total_cost)
if not switch_model.utilities.approx_equal(expected, actual,
tolerance=0.0001):
raise AssertionError(
'Mismatch for total_cost (the objective function value):\n'
'Expected value: {}\n'
'Actual value: {}\n'
'Run "python -m tests.upgrade_test --update" to '
'update the expectations if this change is expected.'
.format(expected, actual))
def main(args=None, return_model=False, return_instance=False):
timer = StepTimer()
if args is None:
# combine default arguments read from options.txt file with
# additional arguments specified on the command line
args = get_option_file_args(extra_args=sys.argv[1:])
# Get options needed before any modules are loaded
pre_module_options = parse_pre_module_options(args)
# turn on post-mortem debugging mode if requested
# (from http://stackoverflow.com/a/1237407 ; more options available there)
if pre_module_options.debug:
def debug(type, value, tb):
import traceback
try:
from ipdb import pm
except ImportError:
from pdb import pm
traceback.print_exception(type, value, tb)
pm()
sys.excepthook = debug
# Write output to a log file if logging option is specified
if pre_module_options.log_run_to_file:
logs_dir = pre_module_options.logs_dir
else:
logs_dir = None # disables logging
with LogOutput(logs_dir):
# Look out for outdated inputs. This has to happen before modules.txt is
# parsed to avoid errors from incompatible files.
parser = _ArgumentParser(allow_abbrev=False, add_help=False)
add_module_args(parser)
module_options = parser.parse_known_args(args=args)[0]
if(os.path.exists(module_options.inputs_dir) and
do_inputs_need_upgrade(module_options.inputs_dir)):
do_upgrade = query_yes_no(
"Warning! Your inputs directory needs to be upgraded. "
"Do you want to auto-upgrade now? We'll keep a backup of "
"this current version."
)
if do_upgrade:
upgrade_inputs(module_options.inputs_dir)
else:
print("Inputs need upgrade. Consider `switch upgrade --help`. Exiting.")
stop_logging_output()
return -1
# build a module list based on configuration options, and add
# the current module (to register define_arguments callback)
modules = get_module_list(args)
# Patch pyomo if needed, to allow reconstruction of expressions.
# This must be done before the model is constructed.
patch_pyomo()
# Define the model
model = create_model(modules, args=args)
# Add any suffixes specified on the command line (usually only iis)
add_extra_suffixes(model)
# return the model as-is if requested
if return_model and not return_instance:
return model
if model.options.reload_prior_solution:
# TODO: allow a directory to be specified after --reload-prior-solution,
# otherwise use outputs_dir.
if not os.path.isdir(model.options.outputs_dir):
raise IOError("Directory specified for prior solution does not exist.")
# get a list of modules to iterate through
iterate_modules = get_iteration_list(model)
if model.options.verbose:
print("\n=======================================================================")
print("Switch {}, http://switch-model.org".format(switch_model.__version__))
print("=======================================================================")
print("Arguments:")
print(", ".join(k+"="+repr(v) for k, v in model.options.__dict__.items() if v))
print("Modules:\n"+", ".join(m for m in modules))
if iterate_modules:
print("Iteration modules:", iterate_modules)
print("=======================================================================\n")
print("Model created in {:.2f} s.".format(timer.step_time()))
print("Loading inputs...")
# create an instance (also reports time spent reading data and loading into model)
instance = model.load_inputs()
#### Below here, we refer to instance instead of model ####
instance.pre_solve()
if instance.options.verbose:
print("Total time spent constructing model: {:.2f} s.\n".format(timer.step_time()))
# return the instance as-is if requested
if return_instance:
if return_model:
return (model, instance)
else:
return instance
# make sure the outputs_dir exists (used by some modules during iterate)
# use a race-safe approach in case this code is run in parallel
try:
os.makedirs(instance.options.outputs_dir)
except OSError:
# directory probably exists already, but double-check
if not os.path.isdir(instance.options.outputs_dir):
raise
if instance.options.reload_prior_solution:
print('Loading prior solution...')
reload_prior_solution_from_pickle(instance, instance.options.outputs_dir)
if instance.options.verbose:
print(
'Loaded previous results into model instance in {:.2f} s.'
.format(timer.step_time())
)
else:
# solve the model (reports time for each step as it goes)
if iterate_modules:
if instance.options.verbose:
print("Iterating model...")
iterate(instance, iterate_modules)
else:
results = solve(instance)
if instance.options.verbose:
print("")
print("Optimization termination condition was {}.".format(
results.solver.termination_condition))
if str(results.solver.message) != '<undefined>':
print('Solver message: {}'.format(results.solver.message))
print("")
if instance.options.verbose:
timer.step_time() # restart counter for next step
if not instance.options.no_save_solution:
save_results(instance, instance.options.outputs_dir)
if instance.options.verbose:
print('Saved results in {:.2f} s.'.format(timer.step_time()))
# report results
# (repeated if model is reloaded, to automatically run any new export code)
if not instance.options.no_post_solve:
if instance.options.verbose:
print("Executing post solve functions...")
instance.post_solve()
if instance.options.verbose:
print("Post solve processing completed in {:.2f} s.".format(timer.step_time()))
# end of LogOutput block
if instance.options.interact:
m = instance # present the solved model as 'm' for convenience
banner = (
"\n"
"=======================================================================\n"
"Entering interactive Python shell.\n"
"Abstract model is in 'model' variable; \n"
"Solved instance is in 'instance' and 'm' variables.\n"
"Type ctrl-d or exit() to exit shell.\n"
"=======================================================================\n"
)
import code
code.interact(banner=banner, local=dict(list(globals().items()) + list(locals().items())))
def main(args=None, return_model=False, return_instance=False):
timer = StepTimer()
if args is None:
# combine default arguments read from options.txt file with
# additional arguments specified on the command line
args = get_option_file_args(extra_args=sys.argv[1:])
# Get options needed before any modules are loaded
pre_module_options = parse_pre_module_options(args)
# turn on post-mortem debugging mode if requested
# (from http://stackoverflow.com/a/1237407 ; more options available there)
if pre_module_options.debug:
def debug(type, value, tb):
import traceback, pdb
traceback.print_exception(type, value, tb)
pdb.pm()
sys.excepthook = debug
# Write output to a log file if logging option is specified
if pre_module_options.log_run_to_file:
logs_dir = pre_module_options.logs_dir
else:
logs_dir = None # disables logging
with LogOutput(logs_dir):
# Look out for outdated inputs. This has to happen before modules.txt is
# parsed to avoid errors from incompatible files.
parser = _ArgumentParser(allow_abbrev=False, add_help=False)
add_module_args(parser)
module_options = parser.parse_known_args(args=args)[0]
if(os.path.exists(module_options.inputs_dir) and
do_inputs_need_upgrade(module_options.inputs_dir)):
do_upgrade = query_yes_no(
("Warning! Your inputs directory needs to be upgraded. "
"Do you want to auto-upgrade now? We'll keep a backup of "
"this current version."))
if do_upgrade:
upgrade_inputs(module_options.inputs_dir)
else:
print "Inputs need upgrade. Consider `switch upgrade --help`. Exiting."
stop_logging_output()
return -1
# build a module list based on configuration options, and add
# the current module (to register define_arguments callback)
modules = get_module_list(args)
# Patch pyomo if needed, to allow reconstruction of expressions.
# This must be done before the model is constructed.
patch_pyomo()
# Define the model
model = create_model(modules, args=args)
# Add any suffixes specified on the command line (usually only iis)
add_extra_suffixes(model)
# return the model as-is if requested
if return_model and not return_instance:
return model
if model.options.reload_prior_solution:
# TODO: allow a directory to be specified after --reload-prior-solution,
# otherwise use outputs_dir.
if not os.path.isdir(model.options.outputs_dir):
raise IOError("Directory specified for prior solution does not exist.")
# get a list of modules to iterate through
iterate_modules = get_iteration_list(model)
if model.options.verbose:
print "\n======================================================================="
print "SWITCH model created in {:.2f} s.\nArguments:".format(timer.step_time())
print ", ".join(k+"="+repr(v) for k, v in model.options.__dict__.items() if v)
print "Modules:\n"+", ".join(m for m in modules)
if iterate_modules:
print "Iteration modules:", iterate_modules
print "=======================================================================\n"
print "Loading inputs..."
# create an instance (also reports time spent reading data and loading into model)
instance = model.load_inputs()
#### Below here, we refer to instance instead of model ####
instance.pre_solve()
if instance.options.verbose:
print "Total time spent constructing model: {:.2f} s.\n".format(timer.step_time())
# return the instance as-is if requested
if return_instance:
if return_model:
return (model, instance)
else:
return instance
# make sure the outputs_dir exists (used by some modules during iterate)
# use a race-safe approach in case this code is run in parallel
try:
os.makedirs(instance.options.outputs_dir)
except OSError:
# directory probably exists already, but double-check
if not os.path.isdir(instance.options.outputs_dir):
raise
if instance.options.reload_prior_solution:
print('Loading prior solution...')
reload_prior_solution_from_pickle(instance, instance.options.outputs_dir)
if instance.options.verbose:
print(
'Loaded previous results into model instance in {:.2f} s.'
.format(timer.step_time())
)
else:
# solve the model (reports time for each step as it goes)
if iterate_modules:
if instance.options.verbose:
print "Iterating model..."
iterate(instance, iterate_modules)
else:
results = solve(instance)
if instance.options.verbose:
print ""
print results.solver.message
print "Optimization termination condition was {}.".format(
results.solver.termination_condition)
print ""
if instance.options.verbose:
timer.step_time() # restart counter for next step
if not instance.options.no_save_solution:
save_results(instance, instance.options.outputs_dir)
if instance.options.verbose:
print('Saved results in {:.2f} s.'.format(timer.step_time()))
# report results
# (repeated if model is reloaded, to automatically run any new export code)
if not instance.options.no_post_solve:
if instance.options.verbose:
print "Executing post solve functions..."
instance.post_solve()
if instance.options.verbose:
print "Post solve processing completed in {:.2f} s.".format(timer.step_time())
# end of LogOutput block
if instance.options.interact:
m = instance # present the solved model as 'm' for convenience
banner = (
"\n"
"=======================================================================\n"
"Entering interactive Python shell.\n"
"Abstract model is in 'model' variable; \n"
"Solved instance is in 'instance' and 'm' variables.\n"
"Type ctrl-d or exit() to exit shell.\n"
"=======================================================================\n"
)
import code
code.interact(banner=banner, local=dict(globals().items() + locals().items()))
def main(args=None, return_model=False, return_instance=False):
start_time = time.time()
if args is None:
# combine default arguments read from options.txt file with
# additional arguments specified on the command line
args = get_option_file_args(extra_args=sys.argv[1:])
# Get options needed before any modules are loaded
pre_module_options = parse_pre_module_options(args)
# turn on post-mortem debugging mode if requested
# (from http://stackoverflow.com/a/1237407 ; more options available there)
if pre_module_options.debug:
def debug(type, value, tb):
import traceback, pdb
traceback.print_exception(type, value, tb)
pdb.pm()
sys.excepthook = debug
# Write output to a log file if logging option is specified
stdout_copy = sys.stdout # make a copy of current sys.stdout to return to eventually
if pre_module_options.log_run_to_file:
logging = Logging(pre_module_options.logs_dir)
print "logging run to " + str(logging.log_file_path)
sys.stdout = logging # assign private class to sys.stdout
else:
pass
# Look out for outdated inputs. This has to happen before modules.txt is
# parsed to avoid errors from incompatible files.
parser = _ArgumentParser(allow_abbrev=False, add_help=False)
add_module_args(parser)
module_options = parser.parse_known_args(args=args)[0]
if(os.path.exists(module_options.inputs_dir) and
do_inputs_need_upgrade(module_options.inputs_dir)):
do_upgrade = query_yes_no(
("Warning! Your inputs directory needs to be upgraded. "
"Do you want to auto-upgrade now? We'll keep a backup of "
"this current version."))
if do_upgrade:
upgrade_inputs(module_options.inputs_dir)
else:
print "Inputs need upgrade. Consider `switch upgrade --help`. Exiting."
sys.stdout = stdout_copy
return -1
# build a module list based on configuration options, and add
# the current module (to register define_arguments callback)
modules = get_module_list(args)
# Patch pyomo if needed, to allow reconstruction of expressions.
# This must be done before the model is constructed.
patch_pyomo()
# Define the model
model = create_model(modules, args=args)
# Add any suffixes specified on the command line (usually only iis)
add_extra_suffixes(model)
# return the model as-is if requested
if return_model and not return_instance:
return model
if model.options.reload_prior_solution:
if not os.path.isdir(model.options.outputs_dir):
raise IOError("Specified outputs directory for solution exploration does not exist.")
# get a list of modules to iterate through
iterate_modules = get_iteration_list(model)
if model.options.verbose:
creation_time = time.time()
print "\n======================================================================="
print "SWITCH model created in {:.2f} s.\nArguments:".format(creation_time - start_time)
print ", ".join(k+"="+repr(v) for k, v in model.options.__dict__.items() if v)
print "Modules:\n"+", ".join(m for m in modules)
if iterate_modules:
print "Iteration modules:", iterate_modules
print "=======================================================================\n"
print "Loading inputs..."
# create an instance
instance = model.load_inputs()
instance.pre_solve()
instantiation_time = time.time()
if model.options.verbose:
print "Inputs loaded in {:.2f} s.\n".format(instantiation_time - creation_time)
# return the instance as-is if requested
if return_instance:
if return_model:
return (model, instance)
else:
return instance
if model.options.reload_prior_solution:
# read variable values from previously solved model
import csv
var_objects = [c for c in instance.component_objects()
if isinstance(c,pyomo.core.base.Var)]
def _convert_if_numeric(s):
try:
return float(s)
except ValueError:
return s
for var in var_objects:
if '{}.tab'.format(var.name) not in os.listdir(model.options.outputs_dir):
raise RuntimeError("Tab output file for variable {} cannot be found in outputs directory. Exiting.".format(var.name))
with open(os.path.join(model.options.outputs_dir, '{}.tab'.format(var.name)),'r') as f:
reader = csv.reader(f, delimiter='\t')
# skip headers
next(reader)
for row in reader:
index = (_convert_if_numeric(i) for i in row[:-1])
var[index].value = float(row[-1])
print 'Loaded variable {} values into instance.'.format(var.name)
output_loading_time = time.time()
print 'Finished loading previous results into model instance in {:.2f} s.'.format(output_loading_time - instantiation_time)
else:
# make sure the outputs_dir exists (used by some modules during iterate)
# use a race-safe approach in case this code is run in parallel
try:
os.makedirs(model.options.outputs_dir)
except OSError:
# directory probably exists already, but double-check
if not os.path.isdir(model.options.outputs_dir):
raise
# solve the model
if iterate_modules:
if model.options.verbose:
print "Iterating model..."
iterate(instance, iterate_modules)
else:
results = solve(instance)
if model.options.verbose:
print "Optimization termination condition was {}.\n".format(
results.solver.termination_condition)
# report/save results
if model.options.verbose:
post_solve_start_time = time.time()
print "Executing post solve functions..."
instance.post_solve()
if model.options.verbose:
post_solve_end_time = time.time()
print "Post solve processing completed in {:.2f} s.".format(
post_solve_end_time - post_solve_start_time)
# return stdout to original
sys.stdout = stdout_copy
if model.options.interact or model.options.reload_prior_solution:
m = instance # present the solved model as 'm' for convenience
banner = (
"\n"
"=======================================================================\n"
"Entering interactive Python shell.\n"
"Abstract model is in 'model' variable; \n"
"Solved instance is in 'instance' and 'm' variables.\n"
"Type ctrl-d or exit() to exit shell.\n"
"=======================================================================\n"
)
import code
code.interact(banner=banner, local=dict(globals().items() + locals().items()))
def main(args=None, return_model=False, return_instance=False):
start_time = time.time()
if args is None:
# combine default arguments read from options.txt file with
# additional arguments specified on the command line
args = get_option_file_args(extra_args=sys.argv[1:])
# Get options needed before any modules are loaded
pre_module_options = parse_pre_module_options(args)
# turn on post-mortem debugging mode if requested
# (from http://stackoverflow.com/a/1237407 ; more options available there)
if pre_module_options.debug:
def debug(type, value, tb):
import traceback, pdb
traceback.print_exception(type, value, tb)
pdb.pm()
sys.excepthook = debug
# Write output to a log file if logging option is specified
stdout_copy = sys.stdout # make a copy of current sys.stdout to return to eventually
if pre_module_options.log_run_to_file:
logging = Logging(pre_module_options.logs_dir)
print "logging run to " + str(logging.log_file_path)
sys.stdout = logging # assign private class to sys.stdout
else:
pass
# Look out for outdated inputs. This has to happen before modules.txt is
# parsed to avoid errors from incompatible files.
parser = _ArgumentParser(allow_abbrev=False, add_help=False)
add_module_args(parser)
module_options = parser.parse_known_args(args=args)[0]
if (os.path.exists(module_options.inputs_dir)
and do_inputs_need_upgrade(module_options.inputs_dir)):
do_upgrade = query_yes_no(
("Warning! Your inputs directory needs to be upgraded. "
"Do you want to auto-upgrade now? We'll keep a backup of "
"this current version."))
if do_upgrade:
upgrade_inputs(module_options.inputs_dir)
else:
print "Inputs need upgrade. Consider `switch upgrade --help`. Exiting."
sys.stdout = stdout_copy
return -1
# build a module list based on configuration options, and add
# the current module (to register define_arguments callback)
modules = get_module_list(args)
# Patch pyomo if needed, to allow reconstruction of expressions.
# This must be done before the model is constructed.
patch_pyomo()
# Define the model
model = create_model(modules, args=args)
# Add any suffixes specified on the command line (usually only iis)
add_extra_suffixes(model)
# return the model as-is if requested
if return_model and not return_instance:
return model
if model.options.reload_prior_solution:
if not os.path.isdir(model.options.outputs_dir):
raise IOError(
"Specified outputs directory for solution exploration does not exist."
)
# get a list of modules to iterate through
iterate_modules = get_iteration_list(model)
if model.options.verbose:
creation_time = time.time()
print "\n======================================================================="
print "SWITCH model created in {:.2f} s.\nArguments:".format(
creation_time - start_time)
print ", ".join(k + "=" + repr(v)
for k, v in model.options.__dict__.items() if v)
print "Modules:\n" + ", ".join(m for m in modules)
if iterate_modules:
print "Iteration modules:", iterate_modules
print "=======================================================================\n"
print "Loading inputs..."
# create an instance
instance = model.load_inputs()
instance.pre_solve()
instantiation_time = time.time()
if model.options.verbose:
print "Inputs loaded in {:.2f} s.\n".format(instantiation_time -
creation_time)
#Paty's addition for debugging:
#embed()
# return the instance as-is if requested
if return_instance:
if return_model:
return (model, instance)
else:
return instance
if model.options.reload_prior_solution:
# read variable values from previously solved model
import csv
var_objects = [
c for c in instance.component_objects()
if isinstance(c, pyomo.core.base.Var)
]
def _convert_if_numeric(s):
try:
return float(s)
except ValueError:
return s
for var in var_objects:
if '{}.tab'.format(var.name) not in os.listdir(
model.options.outputs_dir):
raise RuntimeError(
"Tab output file for variable {} cannot be found in outputs directory. Exiting."
.format(var.name))
with open(
os.path.join(model.options.outputs_dir,
'{}.tab'.format(var.name)), 'r') as f:
reader = csv.reader(f, delimiter='\t')
# skip headers
next(reader)
for row in reader:
index = (_convert_if_numeric(i) for i in row[:-1])
var[index].value = float(row[-1])
print 'Loaded variable {} values into instance.'.format(var.name)
output_loading_time = time.time()
print 'Finished loading previous results into model instance in {:.2f} s.'.format(
output_loading_time - instantiation_time)
else:
# make sure the outputs_dir exists (used by some modules during iterate)
# use a race-safe approach in case this code is run in parallel
try:
os.makedirs(model.options.outputs_dir)
except OSError:
# directory probably exists already, but double-check
if not os.path.isdir(model.options.outputs_dir):
raise
# solve the model
if iterate_modules:
if model.options.verbose:
print "Iterating model..."
iterate(instance, iterate_modules)
else:
results = solve(instance)
if model.options.verbose:
print "Optimization termination condition was {}.\n".format(
results.solver.termination_condition)
#Paty's addition for debugging:
#embed()
# report/save results
if model.options.verbose:
post_solve_start_time = time.time()
print "Executing post solve functions..."
instance.post_solve()
if model.options.verbose:
post_solve_end_time = time.time()
print "Post solve processing completed in {:.2f} s.".format(
post_solve_end_time - post_solve_start_time)
# return stdout to original
sys.stdout = stdout_copy
if model.options.interact or model.options.reload_prior_solution:
m = instance # present the solved model as 'm' for convenience
banner = (
"\n"
"=======================================================================\n"
"Entering interactive Python shell.\n"
"Abstract model is in 'model' variable; \n"
"Solved instance is in 'instance' and 'm' variables.\n"
"Type ctrl-d or exit() to exit shell.\n"
"=======================================================================\n"
)
import code
code.interact(banner=banner,
local=dict(globals().items() + locals().items()))