#
model = AbstractModel()
model.n = Param(default=4)
model.x = Var(RangeSet(model.n), within=Binary)
def o_rule(model):
return sum_product(model.x)
model.o = Objective(rule=o_rule)
model.c = ConstraintList()
# Create two model instances
instance1 = model.create()
instance2 = model.create()
instance2.x[1] = 1
instance2.x[1].fixed = True
# send them to the solver(s)
action_handle = solver_manager.queue(instance1, opt=optsolver, warmstart=False, tee=True, verbose=False)
action_handle_map[action_handle] = "Original"
action_handle = solver_manager.queue(instance2, opt=optsolver, warmstart=False, tee=True, verbose=False)
action_handle_map[action_handle] = "One Var Fixed"
# retrieve the solutions
for i in range(2): # we know there are two instances
this_action_handle = solver_manager.wait_any()
solved_name = action_handle_map[this_action_handle]
results = solver_manager.get_results(this_action_handle)
print "Results for",solved_name
print results
class ScenarioTreeManager(object):
def __init__(self, options):
self._initialized = False
self._options = copy.deepcopy(options)
### Updated During initialize()
self._scenario_tree = None
self._solver_manager = None
self._instances = None
self._bundle_binding_instance_map = {}
self._bundle_scenario_instance_map = {}
self._objective_sense = None
# distributed worker information
self._phpyro_worker_jobs_map = {}
self._phpyro_job_worker_map = {}
###
# callback information
self._callback_function = {}
self._callback_module_key = {}
self._callback_module_name = {}
# For the users to modify as they please in the aggregate
# callback as long as the data placed on it can be serialized
# by Pyro
self._aggregate_user_data = {}
# validate that callback functions exist in specified modules
for options_attr_file, callback_name in (
("aggregate_cfgfile", "pysp_aggregategetter_callback"),
("bounds_cfgfile", "pysp_boundsetter_callback")):
module_name = getattr(self._options, options_attr_file)
if module_name is not None:
sys_modules_key, module = load_external_module(module_name)
callback = None
for oname, obj in inspect.getmembers(module):
if oname == callback_name:
callback = obj
break
if callback is None:
raise ImportError("PySP callback with name '%s' could "
"not be found in module file: %s" %
(callback_name, module_name))
self._callback_function[callback_name] = callback
self._callback_module_key[callback_name] = sys_modules_key
self._callback_module_name[callback_name] = module_name
def deactivate(self):
import pyomo.solvers.plugins.smanager.phpyro
if not self._initialized:
return
if self._solver_manager is not None:
if isinstance(self._solver_manager,
pyomo.solvers.plugins.smanager.\
phpyro.SolverManager_PHPyro):
scenariotreeserverutils.release_scenariotreeservers(self)
self._solver_manager.release_workers()
self._solver_manager.deactivate()
self._solver_manager = None
self._scenario_tree = None
self._instances = None
self._phpyro_worker_jobs_map = {}
self._phpyro_job_worker_map = {}
self._initialized = False
def release(self):
ret = (self._scenario_tree, self._solver_manager,
self._phpyro_worker_jobs_map, self._phpyro_job_worker_map)
self._solver_manager = None
self._scenario_tree = None
self._instances = None
self._phpyro_worker_jobs_map = {}
self._phpyro_job_worker_map = {}
self._initialized = False
return ret
def initialize(self, scenario_tree):
import pyomo.environ
import pyomo.solvers.plugins.smanager.phpyro
init_start_time = time.time()
print("Initializing Scenario Tree Manager")
print("")
if scenario_tree is None:
raise ValueError("A scenario tree must be supplied to the "
"ScenarioTreeManager initialize() method")
self._scenario_tree = scenario_tree
# construct the solver manager.
if self._options.verbose:
print("Constructing solver manager of type=" +
self._options.solver_manager_type)
self._solver_manager = \
SolverManagerFactory(self._options.solver_manager_type,
host=self._options.pyro_manager_hostname)
if self._solver_manager is None:
raise ValueError("Failed to create solver manager of "
"type=" + self._options.solver_manager_type)
isPHPyro = isinstance(self._solver_manager,
pyomo.solvers.plugins.\
smanager.phpyro.SolverManager_PHPyro)
if isPHPyro:
if self._scenario_tree.contains_bundles():
num_jobs = len(self._scenario_tree._scenario_bundles)
if not _OLD_OUTPUT:
print("Bundle solver jobs available: " + str(num_jobs))
else:
num_jobs = len(self._scenario_tree._scenarios)
if not _OLD_OUTPUT:
print("Scenario solver jobs available: " + str(num_jobs))
workers_expected = self._options.phpyro_required_workers
if (workers_expected is None):
workers_expected = num_jobs
timeout = self._options.phpyro_workers_timeout if \
(self._options.phpyro_required_workers is None) else \
None
self._solver_manager.acquire_workers(workers_expected, timeout)
initialization_action_handles = []
if isPHPyro:
if self._options.verbose:
print("Broadcasting requests to initialize "
"distributed scenario tree workers")
initialization_action_handles.extend(
scenariotreeserverutils.\
initialize_scenariotree_workers(self))
if self._options.verbose:
print("Distributed scenario tree initialization "
"requests successfully transmitted")
else:
build_start_time = time.time()
if not _OLD_OUTPUT:
print("Constructing scenario tree instances")
self._instances = \
self._scenario_tree._scenario_instance_factory.\
construct_instances_for_scenario_tree(
scenario_tree,
flatten_expressions=self._options.flatten_expressions,
report_timing=self._options.output_times,
preprocess=False)
if self._options.verbose or self._options.output_times:
print("Time to construct scenario instances=%.2f seconds" %
(time.time() - build_start_time))
if not _OLD_OUTPUT:
print("Linking instances into scenario tree")
build_start_time = time.time()
# with the scenario instances now available, link the
# referenced objects directly into the scenario tree.
self._scenario_tree.linkInInstances(
self._instances,
objective_sense=self._options.objective_sense,
create_variable_ids=True)
if self._options.verbose or self._options.output_times:
print("Time link scenario tree with instances=%.2f seconds" %
(time.time() - build_start_time))
if self._scenario_tree.contains_bundles():
build_start_time = time.time()
if self._options.verbose:
print("Forming binding instances for all scenario bundles")
self._bundle_binding_instance_map.clear()
self._bundle_scenario_instance_map.clear()
if not self._scenario_tree.contains_bundles():
raise RuntimeError(
"Failed to create binding instances for scenario "
"bundles - no scenario bundles are defined!")
for scenario_bundle in self._scenario_tree._scenario_bundles:
if self._options.verbose:
print(
"Creating binding instance for scenario bundle=%s"
% (scenario_bundle._name))
self._bundle_scenario_instance_map[
scenario_bundle._name] = {}
for scenario_name in scenario_bundle._scenario_names:
self._bundle_scenario_instance_map[scenario_bundle._name]\
[scenario_name] = self._instances[scenario_name]
# IMPORTANT: The bundle variable IDs must be idential to
# those in the parent scenario tree - this is
# critical for storing results, which occurs at
# the full-scale scenario tree.
scenario_bundle._scenario_tree.linkInInstances(
self._instances,
create_variable_ids=False,
master_scenario_tree=self._scenario_tree,
initialize_solution_data=False)
bundle_ef_instance = create_ef_instance(
scenario_bundle._scenario_tree,
ef_instance_name=scenario_bundle._name,
verbose_output=self._verbose)
self._bundle_binding_instance_map[scenario_bundle._name] = \
bundle_ef_instance
if self._output_times:
print("Scenario bundle construction time=%.2f seconds" %
(time.time() - build_start_time))
# If specified, run the user script to collect aggregate
# scenario data. This can slow down PH initialization as
# syncronization across all phsolverservers is required
if self._options.aggregate_cfgfile is not None:
callback_name = "pysp_aggregategetter_callback"
if isPHPyro:
# Transmit invocation to phsolverservers
print("Transmitting user aggregate callback invocations "
"to phsolverservers")
if self._scenario_tree.contains_bundles():
for scenario_bundle in self._scenario_tree._scenario_bundles:
ah = scenariotreeserverutils.\
transmit_external_function_invocation_to_worker(
self,
scenario_bundle._name,
self._callback_module_name[callback_name],
callback_name,
invocation_type=(scenariotreeserverutils.InvocationType.\
PerScenarioChainedInvocation),
return_action_handle=True,
function_args=(self._aggregate_user_data,))
while (1):
action_handle = self._solver_manager.wait_any()
if action_handle in initialization_action_handles:
initialization_action_handles.remove(
action_handle)
self._solver_manager.get_results(action_handle)
elif action_handle == ah:
result = self._solver_manager.get_results(
action_handle)
break
assert len(result) == 1
self._aggregate_user_data = result[0]
else:
for scenario in self._scenario_tree._scenarios:
ah = scenariotreeserverutils.\
transmit_external_function_invocation_to_worker(
self,
scenario._name,
self._callback_module_name[callback_name],
callback_name,
invocation_type=(scenariotreeserverutils.InvocationType.\
SingleInvocation),
return_action_handle=True,
function_args=(self._aggregate_user_data,))
while (1):
action_handle = self._solver_manager.wait_any()
if action_handle in initialization_action_handles:
initialization_action_handles.remove(
action_handle)
self._solver_manager.get_results(action_handle)
elif action_handle == ah:
result = self._solver_manager.get_results(
action_handle)
break
assert len(result) == 1
self._aggregate_user_data = result[0]
# Transmit final aggregate state to phsolverservers
print("Broadcasting final aggregate data to phsolverservers")
initialization_action_handles.extend(
scenariotreeserverutils.transmit_external_function_invocation(
self,
"pyomo.pysp.ph",
"assign_aggregate_data",
invocation_type=(scenariotreeserverutils.InvocationType.\
SingleInvocation),
return_action_handles=True,
function_args=(self._aggregate_user_data,)))
else:
print("Executing user aggregate getter callback function")
for scenario in self._scenario_tree._scenarios:
result = self._callback_function[callback_name](
self, self._scenario_tree, scenario,
self._aggregate_user_data)
assert len(result) == 1
self._aggregate_user_data = result[0]
# if specified, run the user script to initialize variable
# bounds at their whim.
if self._options.bounds_cfgfile is not None:
callback_name = "pysp_boundsetter_callback"
if isPHPyro:
# Transmit invocation to phsolverservers
print("Transmitting user bound callback invocations to "
"phsolverservers")
if self._scenario_tree.contains_bundles():
for scenario_bundle in self._scenario_tree._scenario_bundles:
initialization_action_handles.append(
scenariotreeserverutils.\
transmit_external_function_invocation_to_worker(
self,
scenario_bundle._name,
self._callback_module_name[callback_name],
callback_name,
invocation_type=(scenariotreeserverutils.InvocationType.\
PerScenarioInvocation),
return_action_handle=True))
else:
for scenario in self._scenario_tree._scenarios:
initialization_action_handles.append(
scenariotreeserverutils.\
transmit_external_function_invocation_to_worker(
self,
scenario._name,
self._callback_module_name[callback_name],
callback_name,
invocation_type=(scenariotreeserverutils.InvocationType.\
SingleInvocation),
return_action_handle=True))
else:
print("Executing user bound setter callback function")
for scenario in self._scenario_tree._scenarios:
self._callback_function[callback_name](self,
self._scenario_tree,
scenario)
# gather scenario tree data if not local
if isPHPyro:
if self._options.verbose:
print("Broadcasting requests to collect scenario tree "
"instance data from PH solver servers")
scenariotreeserverutils.\
gather_scenario_tree_data(self,
initialization_action_handles)
assert len(initialization_action_handles) == 0
if self._options.verbose:
print("Scenario tree instance data successfully collected")
if self._options.verbose:
print("Broadcasting scenario tree id mapping"
"to PH solver servers")
scenariotreeserverutils.transmit_scenario_tree_ids(self)
if self._options.verbose:
print("Scenario tree ids successfully sent")
self._objective_sense = \
self._scenario_tree._scenarios[0]._objective_sense
if self._options.verbose:
print("Scenario tree manager is successfully initialized")
if self._options.output_times:
print("Overall initialization time=%.2f seconds" %
(time.time() - init_start_time))
# gather and report memory statistics (for leak
# detection purposes) if specified.
if (guppy_available) and (self._options.profile_memory >= 1):
print(hpy().heap())
# indicate that we're ready to run.
self._initialized = True
class ScenarioTreeManager(object):
def __init__(self, options):
self._initialized = False
self._options = copy.deepcopy(options)
### Updated During initialize()
self._scenario_tree = None
self._solver_manager = None
self._instances = None
self._bundle_binding_instance_map = {}
self._bundle_scenario_instance_map = {}
self._objective_sense = None
# distributed worker information
self._phpyro_worker_jobs_map = {}
self._phpyro_job_worker_map = {}
###
# callback information
self._callback_function = {}
self._callback_module_key = {}
self._callback_module_name = {}
# For the users to modify as they please in the aggregate
# callback as long as the data placed on it can be serialized
# by Pyro
self._aggregate_user_data = {}
# validate that callback functions exist in specified modules
for options_attr_file, callback_name in (("aggregate_cfgfile",
"pysp_aggregategetter_callback"),
("bounds_cfgfile",
"pysp_boundsetter_callback")):
module_name = getattr(self._options, options_attr_file)
if module_name is not None:
sys_modules_key, module = load_external_module(module_name)
callback = None
for oname, obj in inspect.getmembers(module):
if oname == callback_name:
callback = obj
break
if callback is None:
raise ImportError("PySP callback with name '%s' could "
"not be found in module file: %s"
% (callback_name, module_name))
self._callback_function[callback_name] = callback
self._callback_module_key[callback_name] = sys_modules_key
self._callback_module_name[callback_name] = module_name
def deactivate(self):
import pyomo.solvers.plugins.smanager.phpyro
if not self._initialized:
return
if self._solver_manager is not None:
if isinstance(self._solver_manager,
pyomo.solvers.plugins.smanager.\
phpyro.SolverManager_PHPyro):
scenariotreeserverutils.release_scenariotreeservers(self)
self._solver_manager.release_workers()
self._solver_manager.deactivate()
self._solver_manager = None
self._scenario_tree = None
self._instances = None
self._phpyro_worker_jobs_map = {}
self._phpyro_job_worker_map = {}
self._initialized = False
def release(self):
ret = (self._scenario_tree,
self._solver_manager,
self._phpyro_worker_jobs_map,
self._phpyro_job_worker_map)
self._solver_manager = None
self._scenario_tree = None
self._instances = None
self._phpyro_worker_jobs_map = {}
self._phpyro_job_worker_map = {}
self._initialized = False
return ret
def initialize(self, scenario_tree):
import pyomo.environ
import pyomo.solvers.plugins.smanager.phpyro
init_start_time = time.time()
print("Initializing Scenario Tree Manager")
print("")
if scenario_tree is None:
raise ValueError("A scenario tree must be supplied to the "
"ScenarioTreeManager initialize() method")
self._scenario_tree = scenario_tree
# construct the solver manager.
if self._options.verbose:
print("Constructing solver manager of type="
+self._options.solver_manager_type)
self._solver_manager = \
SolverManagerFactory(self._options.solver_manager_type,
host=self._options.pyro_manager_hostname)
if self._solver_manager is None:
raise ValueError("Failed to create solver manager of "
"type="+self._options.solver_manager_type)
isPHPyro = isinstance(self._solver_manager,
pyomo.solvers.plugins.\
smanager.phpyro.SolverManager_PHPyro)
if isPHPyro:
if self._scenario_tree.contains_bundles():
num_jobs = len(self._scenario_tree._scenario_bundles)
if not _OLD_OUTPUT:
print("Bundle solver jobs available: "+str(num_jobs))
else:
num_jobs = len(self._scenario_tree._scenarios)
if not _OLD_OUTPUT:
print("Scenario solver jobs available: "+str(num_jobs))
workers_expected = self._options.phpyro_required_workers
if (workers_expected is None):
workers_expected = num_jobs
timeout = self._options.phpyro_workers_timeout if \
(self._options.phpyro_required_workers is None) else \
None
self._solver_manager.acquire_workers(workers_expected,
timeout)
initialization_action_handles = []
if isPHPyro:
if self._options.verbose:
print("Broadcasting requests to initialize "
"distributed scenario tree workers")
initialization_action_handles.extend(
scenariotreeserverutils.\
initialize_scenariotree_workers(self))
if self._options.verbose:
print("Distributed scenario tree initialization "
"requests successfully transmitted")
else:
build_start_time = time.time()
if not _OLD_OUTPUT:
print("Constructing scenario tree instances")
self._instances = \
self._scenario_tree._scenario_instance_factory.\
construct_instances_for_scenario_tree(
scenario_tree,
flatten_expressions=self._options.flatten_expressions,
report_timing=self._options.output_times,
preprocess=False)
if self._options.verbose or self._options.output_times:
print("Time to construct scenario instances=%.2f seconds"
% (time.time() - build_start_time))
if not _OLD_OUTPUT:
print("Linking instances into scenario tree")
build_start_time = time.time()
# with the scenario instances now available, link the
# referenced objects directly into the scenario tree.
self._scenario_tree.linkInInstances(
self._instances,
objective_sense=self._options.objective_sense,
create_variable_ids=True)
if self._options.verbose or self._options.output_times:
print("Time link scenario tree with instances=%.2f seconds"
% (time.time() - build_start_time))
if self._scenario_tree.contains_bundles():
build_start_time = time.time()
if self._options.verbose:
print("Forming binding instances for all scenario bundles")
self._bundle_binding_instance_map.clear()
self._bundle_scenario_instance_map.clear()
if not self._scenario_tree.contains_bundles():
raise RuntimeError("Failed to create binding instances for scenario "
"bundles - no scenario bundles are defined!")
for scenario_bundle in self._scenario_tree._scenario_bundles:
if self._options.verbose:
print("Creating binding instance for scenario bundle=%s"
% (scenario_bundle._name))
self._bundle_scenario_instance_map[scenario_bundle._name] = {}
for scenario_name in scenario_bundle._scenario_names:
self._bundle_scenario_instance_map[scenario_bundle._name]\
[scenario_name] = self._instances[scenario_name]
# IMPORTANT: The bundle variable IDs must be idential to
# those in the parent scenario tree - this is
# critical for storing results, which occurs at
# the full-scale scenario tree.
scenario_bundle._scenario_tree.linkInInstances(
self._instances,
create_variable_ids=False,
master_scenario_tree=self._scenario_tree,
initialize_solution_data=False)
bundle_ef_instance = create_ef_instance(
scenario_bundle._scenario_tree,
ef_instance_name=scenario_bundle._name,
verbose_output=self._verbose)
self._bundle_binding_instance_map[scenario_bundle._name] = \
bundle_ef_instance
if self._output_times:
print("Scenario bundle construction time=%.2f seconds"
% (time.time() - build_start_time))
# If specified, run the user script to collect aggregate
# scenario data. This can slow down PH initialization as
# syncronization across all phsolverservers is required
if self._options.aggregate_cfgfile is not None:
callback_name = "pysp_aggregategetter_callback"
if isPHPyro:
# Transmit invocation to phsolverservers
print("Transmitting user aggregate callback invocations "
"to phsolverservers")
if self._scenario_tree.contains_bundles():
for scenario_bundle in self._scenario_tree._scenario_bundles:
ah = scenariotreeserverutils.\
transmit_external_function_invocation_to_worker(
self,
scenario_bundle._name,
self._callback_module_name[callback_name],
callback_name,
invocation_type=(scenariotreeserverutils.InvocationType.\
PerScenarioChainedInvocation),
return_action_handle=True,
function_args=(self._aggregate_user_data,))
while(1):
action_handle = self._solver_manager.wait_any()
if action_handle in initialization_action_handles:
initialization_action_handles.remove(action_handle)
self._solver_manager.get_results(action_handle)
elif action_handle == ah:
result = self._solver_manager.get_results(action_handle)
break
assert len(result) == 1
self._aggregate_user_data = result[0]
else:
for scenario in self._scenario_tree._scenarios:
ah = scenariotreeserverutils.\
transmit_external_function_invocation_to_worker(
self,
scenario._name,
self._callback_module_name[callback_name],
callback_name,
invocation_type=(scenariotreeserverutils.InvocationType.\
SingleInvocation),
return_action_handle=True,
function_args=(self._aggregate_user_data,))
while(1):
action_handle = self._solver_manager.wait_any()
if action_handle in initialization_action_handles:
initialization_action_handles.remove(action_handle)
self._solver_manager.get_results(action_handle)
elif action_handle == ah:
result = self._solver_manager.get_results(action_handle)
break
assert len(result) == 1
self._aggregate_user_data = result[0]
# Transmit final aggregate state to phsolverservers
print("Broadcasting final aggregate data to phsolverservers")
initialization_action_handles.extend(
scenariotreeserverutils.transmit_external_function_invocation(
self,
"pyomo.pysp.ph",
"assign_aggregate_data",
invocation_type=(scenariotreeserverutils.InvocationType.\
SingleInvocation),
return_action_handles=True,
function_args=(self._aggregate_user_data,)))
else:
print("Executing user aggregate getter callback function")
for scenario in self._scenario_tree._scenarios:
result = self._callback_function[callback_name](
self,
self._scenario_tree,
scenario,
self._aggregate_user_data)
assert len(result) == 1
self._aggregate_user_data = result[0]
# if specified, run the user script to initialize variable
# bounds at their whim.
if self._options.bounds_cfgfile is not None:
callback_name = "pysp_boundsetter_callback"
if isPHPyro:
# Transmit invocation to phsolverservers
print("Transmitting user bound callback invocations to "
"phsolverservers")
if self._scenario_tree.contains_bundles():
for scenario_bundle in self._scenario_tree._scenario_bundles:
initialization_action_handles.append(
scenariotreeserverutils.\
transmit_external_function_invocation_to_worker(
self,
scenario_bundle._name,
self._callback_module_name[callback_name],
callback_name,
invocation_type=(scenariotreeserverutils.InvocationType.\
PerScenarioInvocation),
return_action_handle=True))
else:
for scenario in self._scenario_tree._scenarios:
initialization_action_handles.append(
scenariotreeserverutils.\
transmit_external_function_invocation_to_worker(
self,
scenario._name,
self._callback_module_name[callback_name],
callback_name,
invocation_type=(scenariotreeserverutils.InvocationType.\
SingleInvocation),
return_action_handle=True))
else:
print("Executing user bound setter callback function")
for scenario in self._scenario_tree._scenarios:
self._callback_function[callback_name](
self,
self._scenario_tree,
scenario)
# gather scenario tree data if not local
if isPHPyro:
if self._options.verbose:
print("Broadcasting requests to collect scenario tree "
"instance data from PH solver servers")
scenariotreeserverutils.\
gather_scenario_tree_data(self,
initialization_action_handles)
assert len(initialization_action_handles) == 0
if self._options.verbose:
print("Scenario tree instance data successfully collected")
if self._options.verbose:
print("Broadcasting scenario tree id mapping"
"to PH solver servers")
scenariotreeserverutils.transmit_scenario_tree_ids(self)
if self._options.verbose:
print("Scenario tree ids successfully sent")
self._objective_sense = \
self._scenario_tree._scenarios[0]._objective_sense
if self._options.verbose:
print("Scenario tree manager is successfully initialized")
if self._options.output_times:
print("Overall initialization time=%.2f seconds"
% (time.time() - init_start_time))
# gather and report memory statistics (for leak
# detection purposes) if specified.
if (guppy_available) and (self._options.profile_memory >= 1):
print(hpy().heap())
# indicate that we're ready to run.
self._initialized = True
