# model = AbstractModel() model.n = Param(default=4) model.x = Var(RangeSet(model.n), within=Binary) def o_rule(model): return sum_product(model.x) model.o = Objective(rule=o_rule) model.c = ConstraintList() # Create two model instances instance1 = model.create() instance2 = model.create() instance2.x[1] = 1 instance2.x[1].fixed = True # send them to the solver(s) action_handle = solver_manager.queue(instance1, opt=optsolver, warmstart=False, tee=True, verbose=False) action_handle_map[action_handle] = "Original" action_handle = solver_manager.queue(instance2, opt=optsolver, warmstart=False, tee=True, verbose=False) action_handle_map[action_handle] = "One Var Fixed" # retrieve the solutions for i in range(2): # we know there are two instances this_action_handle = solver_manager.wait_any() solved_name = action_handle_map[this_action_handle] results = solver_manager.get_results(this_action_handle) print "Results for",solved_name print results
class ScenarioTreeManager(object): def __init__(self, options): self._initialized = False self._options = copy.deepcopy(options) ### Updated During initialize() self._scenario_tree = None self._solver_manager = None self._instances = None self._bundle_binding_instance_map = {} self._bundle_scenario_instance_map = {} self._objective_sense = None # distributed worker information self._phpyro_worker_jobs_map = {} self._phpyro_job_worker_map = {} ### # callback information self._callback_function = {} self._callback_module_key = {} self._callback_module_name = {} # For the users to modify as they please in the aggregate # callback as long as the data placed on it can be serialized # by Pyro self._aggregate_user_data = {} # validate that callback functions exist in specified modules for options_attr_file, callback_name in ( ("aggregate_cfgfile", "pysp_aggregategetter_callback"), ("bounds_cfgfile", "pysp_boundsetter_callback")): module_name = getattr(self._options, options_attr_file) if module_name is not None: sys_modules_key, module = load_external_module(module_name) callback = None for oname, obj in inspect.getmembers(module): if oname == callback_name: callback = obj break if callback is None: raise ImportError("PySP callback with name '%s' could " "not be found in module file: %s" % (callback_name, module_name)) self._callback_function[callback_name] = callback self._callback_module_key[callback_name] = sys_modules_key self._callback_module_name[callback_name] = module_name def deactivate(self): import pyomo.solvers.plugins.smanager.phpyro if not self._initialized: return if self._solver_manager is not None: if isinstance(self._solver_manager, pyomo.solvers.plugins.smanager.\ phpyro.SolverManager_PHPyro): scenariotreeserverutils.release_scenariotreeservers(self) self._solver_manager.release_workers() self._solver_manager.deactivate() self._solver_manager = None self._scenario_tree = None self._instances = None self._phpyro_worker_jobs_map = {} self._phpyro_job_worker_map = {} self._initialized = False def release(self): ret = (self._scenario_tree, self._solver_manager, self._phpyro_worker_jobs_map, self._phpyro_job_worker_map) self._solver_manager = None self._scenario_tree = None self._instances = None self._phpyro_worker_jobs_map = {} self._phpyro_job_worker_map = {} self._initialized = False return ret def initialize(self, scenario_tree): import pyomo.environ import pyomo.solvers.plugins.smanager.phpyro init_start_time = time.time() print("Initializing Scenario Tree Manager") print("") if scenario_tree is None: raise ValueError("A scenario tree must be supplied to the " "ScenarioTreeManager initialize() method") self._scenario_tree = scenario_tree # construct the solver manager. if self._options.verbose: print("Constructing solver manager of type=" + self._options.solver_manager_type) self._solver_manager = \ SolverManagerFactory(self._options.solver_manager_type, host=self._options.pyro_manager_hostname) if self._solver_manager is None: raise ValueError("Failed to create solver manager of " "type=" + self._options.solver_manager_type) isPHPyro = isinstance(self._solver_manager, pyomo.solvers.plugins.\ smanager.phpyro.SolverManager_PHPyro) if isPHPyro: if self._scenario_tree.contains_bundles(): num_jobs = len(self._scenario_tree._scenario_bundles) if not _OLD_OUTPUT: print("Bundle solver jobs available: " + str(num_jobs)) else: num_jobs = len(self._scenario_tree._scenarios) if not _OLD_OUTPUT: print("Scenario solver jobs available: " + str(num_jobs)) workers_expected = self._options.phpyro_required_workers if (workers_expected is None): workers_expected = num_jobs timeout = self._options.phpyro_workers_timeout if \ (self._options.phpyro_required_workers is None) else \ None self._solver_manager.acquire_workers(workers_expected, timeout) initialization_action_handles = [] if isPHPyro: if self._options.verbose: print("Broadcasting requests to initialize " "distributed scenario tree workers") initialization_action_handles.extend( scenariotreeserverutils.\ initialize_scenariotree_workers(self)) if self._options.verbose: print("Distributed scenario tree initialization " "requests successfully transmitted") else: build_start_time = time.time() if not _OLD_OUTPUT: print("Constructing scenario tree instances") self._instances = \ self._scenario_tree._scenario_instance_factory.\ construct_instances_for_scenario_tree( scenario_tree, flatten_expressions=self._options.flatten_expressions, report_timing=self._options.output_times, preprocess=False) if self._options.verbose or self._options.output_times: print("Time to construct scenario instances=%.2f seconds" % (time.time() - build_start_time)) if not _OLD_OUTPUT: print("Linking instances into scenario tree") build_start_time = time.time() # with the scenario instances now available, link the # referenced objects directly into the scenario tree. self._scenario_tree.linkInInstances( self._instances, objective_sense=self._options.objective_sense, create_variable_ids=True) if self._options.verbose or self._options.output_times: print("Time link scenario tree with instances=%.2f seconds" % (time.time() - build_start_time)) if self._scenario_tree.contains_bundles(): build_start_time = time.time() if self._options.verbose: print("Forming binding instances for all scenario bundles") self._bundle_binding_instance_map.clear() self._bundle_scenario_instance_map.clear() if not self._scenario_tree.contains_bundles(): raise RuntimeError( "Failed to create binding instances for scenario " "bundles - no scenario bundles are defined!") for scenario_bundle in self._scenario_tree._scenario_bundles: if self._options.verbose: print( "Creating binding instance for scenario bundle=%s" % (scenario_bundle._name)) self._bundle_scenario_instance_map[ scenario_bundle._name] = {} for scenario_name in scenario_bundle._scenario_names: self._bundle_scenario_instance_map[scenario_bundle._name]\ [scenario_name] = self._instances[scenario_name] # IMPORTANT: The bundle variable IDs must be idential to # those in the parent scenario tree - this is # critical for storing results, which occurs at # the full-scale scenario tree. scenario_bundle._scenario_tree.linkInInstances( self._instances, create_variable_ids=False, master_scenario_tree=self._scenario_tree, initialize_solution_data=False) bundle_ef_instance = create_ef_instance( scenario_bundle._scenario_tree, ef_instance_name=scenario_bundle._name, verbose_output=self._verbose) self._bundle_binding_instance_map[scenario_bundle._name] = \ bundle_ef_instance if self._output_times: print("Scenario bundle construction time=%.2f seconds" % (time.time() - build_start_time)) # If specified, run the user script to collect aggregate # scenario data. This can slow down PH initialization as # syncronization across all phsolverservers is required if self._options.aggregate_cfgfile is not None: callback_name = "pysp_aggregategetter_callback" if isPHPyro: # Transmit invocation to phsolverservers print("Transmitting user aggregate callback invocations " "to phsolverservers") if self._scenario_tree.contains_bundles(): for scenario_bundle in self._scenario_tree._scenario_bundles: ah = scenariotreeserverutils.\ transmit_external_function_invocation_to_worker( self, scenario_bundle._name, self._callback_module_name[callback_name], callback_name, invocation_type=(scenariotreeserverutils.InvocationType.\ PerScenarioChainedInvocation), return_action_handle=True, function_args=(self._aggregate_user_data,)) while (1): action_handle = self._solver_manager.wait_any() if action_handle in initialization_action_handles: initialization_action_handles.remove( action_handle) self._solver_manager.get_results(action_handle) elif action_handle == ah: result = self._solver_manager.get_results( action_handle) break assert len(result) == 1 self._aggregate_user_data = result[0] else: for scenario in self._scenario_tree._scenarios: ah = scenariotreeserverutils.\ transmit_external_function_invocation_to_worker( self, scenario._name, self._callback_module_name[callback_name], callback_name, invocation_type=(scenariotreeserverutils.InvocationType.\ SingleInvocation), return_action_handle=True, function_args=(self._aggregate_user_data,)) while (1): action_handle = self._solver_manager.wait_any() if action_handle in initialization_action_handles: initialization_action_handles.remove( action_handle) self._solver_manager.get_results(action_handle) elif action_handle == ah: result = self._solver_manager.get_results( action_handle) break assert len(result) == 1 self._aggregate_user_data = result[0] # Transmit final aggregate state to phsolverservers print("Broadcasting final aggregate data to phsolverservers") initialization_action_handles.extend( scenariotreeserverutils.transmit_external_function_invocation( self, "pyomo.pysp.ph", "assign_aggregate_data", invocation_type=(scenariotreeserverutils.InvocationType.\ SingleInvocation), return_action_handles=True, function_args=(self._aggregate_user_data,))) else: print("Executing user aggregate getter callback function") for scenario in self._scenario_tree._scenarios: result = self._callback_function[callback_name]( self, self._scenario_tree, scenario, self._aggregate_user_data) assert len(result) == 1 self._aggregate_user_data = result[0] # if specified, run the user script to initialize variable # bounds at their whim. if self._options.bounds_cfgfile is not None: callback_name = "pysp_boundsetter_callback" if isPHPyro: # Transmit invocation to phsolverservers print("Transmitting user bound callback invocations to " "phsolverservers") if self._scenario_tree.contains_bundles(): for scenario_bundle in self._scenario_tree._scenario_bundles: initialization_action_handles.append( scenariotreeserverutils.\ transmit_external_function_invocation_to_worker( self, scenario_bundle._name, self._callback_module_name[callback_name], callback_name, invocation_type=(scenariotreeserverutils.InvocationType.\ PerScenarioInvocation), return_action_handle=True)) else: for scenario in self._scenario_tree._scenarios: initialization_action_handles.append( scenariotreeserverutils.\ transmit_external_function_invocation_to_worker( self, scenario._name, self._callback_module_name[callback_name], callback_name, invocation_type=(scenariotreeserverutils.InvocationType.\ SingleInvocation), return_action_handle=True)) else: print("Executing user bound setter callback function") for scenario in self._scenario_tree._scenarios: self._callback_function[callback_name](self, self._scenario_tree, scenario) # gather scenario tree data if not local if isPHPyro: if self._options.verbose: print("Broadcasting requests to collect scenario tree " "instance data from PH solver servers") scenariotreeserverutils.\ gather_scenario_tree_data(self, initialization_action_handles) assert len(initialization_action_handles) == 0 if self._options.verbose: print("Scenario tree instance data successfully collected") if self._options.verbose: print("Broadcasting scenario tree id mapping" "to PH solver servers") scenariotreeserverutils.transmit_scenario_tree_ids(self) if self._options.verbose: print("Scenario tree ids successfully sent") self._objective_sense = \ self._scenario_tree._scenarios[0]._objective_sense if self._options.verbose: print("Scenario tree manager is successfully initialized") if self._options.output_times: print("Overall initialization time=%.2f seconds" % (time.time() - init_start_time)) # gather and report memory statistics (for leak # detection purposes) if specified. if (guppy_available) and (self._options.profile_memory >= 1): print(hpy().heap()) # indicate that we're ready to run. self._initialized = True
class ScenarioTreeManager(object): def __init__(self, options): self._initialized = False self._options = copy.deepcopy(options) ### Updated During initialize() self._scenario_tree = None self._solver_manager = None self._instances = None self._bundle_binding_instance_map = {} self._bundle_scenario_instance_map = {} self._objective_sense = None # distributed worker information self._phpyro_worker_jobs_map = {} self._phpyro_job_worker_map = {} ### # callback information self._callback_function = {} self._callback_module_key = {} self._callback_module_name = {} # For the users to modify as they please in the aggregate # callback as long as the data placed on it can be serialized # by Pyro self._aggregate_user_data = {} # validate that callback functions exist in specified modules for options_attr_file, callback_name in (("aggregate_cfgfile", "pysp_aggregategetter_callback"), ("bounds_cfgfile", "pysp_boundsetter_callback")): module_name = getattr(self._options, options_attr_file) if module_name is not None: sys_modules_key, module = load_external_module(module_name) callback = None for oname, obj in inspect.getmembers(module): if oname == callback_name: callback = obj break if callback is None: raise ImportError("PySP callback with name '%s' could " "not be found in module file: %s" % (callback_name, module_name)) self._callback_function[callback_name] = callback self._callback_module_key[callback_name] = sys_modules_key self._callback_module_name[callback_name] = module_name def deactivate(self): import pyomo.solvers.plugins.smanager.phpyro if not self._initialized: return if self._solver_manager is not None: if isinstance(self._solver_manager, pyomo.solvers.plugins.smanager.\ phpyro.SolverManager_PHPyro): scenariotreeserverutils.release_scenariotreeservers(self) self._solver_manager.release_workers() self._solver_manager.deactivate() self._solver_manager = None self._scenario_tree = None self._instances = None self._phpyro_worker_jobs_map = {} self._phpyro_job_worker_map = {} self._initialized = False def release(self): ret = (self._scenario_tree, self._solver_manager, self._phpyro_worker_jobs_map, self._phpyro_job_worker_map) self._solver_manager = None self._scenario_tree = None self._instances = None self._phpyro_worker_jobs_map = {} self._phpyro_job_worker_map = {} self._initialized = False return ret def initialize(self, scenario_tree): import pyomo.environ import pyomo.solvers.plugins.smanager.phpyro init_start_time = time.time() print("Initializing Scenario Tree Manager") print("") if scenario_tree is None: raise ValueError("A scenario tree must be supplied to the " "ScenarioTreeManager initialize() method") self._scenario_tree = scenario_tree # construct the solver manager. if self._options.verbose: print("Constructing solver manager of type=" +self._options.solver_manager_type) self._solver_manager = \ SolverManagerFactory(self._options.solver_manager_type, host=self._options.pyro_manager_hostname) if self._solver_manager is None: raise ValueError("Failed to create solver manager of " "type="+self._options.solver_manager_type) isPHPyro = isinstance(self._solver_manager, pyomo.solvers.plugins.\ smanager.phpyro.SolverManager_PHPyro) if isPHPyro: if self._scenario_tree.contains_bundles(): num_jobs = len(self._scenario_tree._scenario_bundles) if not _OLD_OUTPUT: print("Bundle solver jobs available: "+str(num_jobs)) else: num_jobs = len(self._scenario_tree._scenarios) if not _OLD_OUTPUT: print("Scenario solver jobs available: "+str(num_jobs)) workers_expected = self._options.phpyro_required_workers if (workers_expected is None): workers_expected = num_jobs timeout = self._options.phpyro_workers_timeout if \ (self._options.phpyro_required_workers is None) else \ None self._solver_manager.acquire_workers(workers_expected, timeout) initialization_action_handles = [] if isPHPyro: if self._options.verbose: print("Broadcasting requests to initialize " "distributed scenario tree workers") initialization_action_handles.extend( scenariotreeserverutils.\ initialize_scenariotree_workers(self)) if self._options.verbose: print("Distributed scenario tree initialization " "requests successfully transmitted") else: build_start_time = time.time() if not _OLD_OUTPUT: print("Constructing scenario tree instances") self._instances = \ self._scenario_tree._scenario_instance_factory.\ construct_instances_for_scenario_tree( scenario_tree, flatten_expressions=self._options.flatten_expressions, report_timing=self._options.output_times, preprocess=False) if self._options.verbose or self._options.output_times: print("Time to construct scenario instances=%.2f seconds" % (time.time() - build_start_time)) if not _OLD_OUTPUT: print("Linking instances into scenario tree") build_start_time = time.time() # with the scenario instances now available, link the # referenced objects directly into the scenario tree. self._scenario_tree.linkInInstances( self._instances, objective_sense=self._options.objective_sense, create_variable_ids=True) if self._options.verbose or self._options.output_times: print("Time link scenario tree with instances=%.2f seconds" % (time.time() - build_start_time)) if self._scenario_tree.contains_bundles(): build_start_time = time.time() if self._options.verbose: print("Forming binding instances for all scenario bundles") self._bundle_binding_instance_map.clear() self._bundle_scenario_instance_map.clear() if not self._scenario_tree.contains_bundles(): raise RuntimeError("Failed to create binding instances for scenario " "bundles - no scenario bundles are defined!") for scenario_bundle in self._scenario_tree._scenario_bundles: if self._options.verbose: print("Creating binding instance for scenario bundle=%s" % (scenario_bundle._name)) self._bundle_scenario_instance_map[scenario_bundle._name] = {} for scenario_name in scenario_bundle._scenario_names: self._bundle_scenario_instance_map[scenario_bundle._name]\ [scenario_name] = self._instances[scenario_name] # IMPORTANT: The bundle variable IDs must be idential to # those in the parent scenario tree - this is # critical for storing results, which occurs at # the full-scale scenario tree. scenario_bundle._scenario_tree.linkInInstances( self._instances, create_variable_ids=False, master_scenario_tree=self._scenario_tree, initialize_solution_data=False) bundle_ef_instance = create_ef_instance( scenario_bundle._scenario_tree, ef_instance_name=scenario_bundle._name, verbose_output=self._verbose) self._bundle_binding_instance_map[scenario_bundle._name] = \ bundle_ef_instance if self._output_times: print("Scenario bundle construction time=%.2f seconds" % (time.time() - build_start_time)) # If specified, run the user script to collect aggregate # scenario data. This can slow down PH initialization as # syncronization across all phsolverservers is required if self._options.aggregate_cfgfile is not None: callback_name = "pysp_aggregategetter_callback" if isPHPyro: # Transmit invocation to phsolverservers print("Transmitting user aggregate callback invocations " "to phsolverservers") if self._scenario_tree.contains_bundles(): for scenario_bundle in self._scenario_tree._scenario_bundles: ah = scenariotreeserverutils.\ transmit_external_function_invocation_to_worker( self, scenario_bundle._name, self._callback_module_name[callback_name], callback_name, invocation_type=(scenariotreeserverutils.InvocationType.\ PerScenarioChainedInvocation), return_action_handle=True, function_args=(self._aggregate_user_data,)) while(1): action_handle = self._solver_manager.wait_any() if action_handle in initialization_action_handles: initialization_action_handles.remove(action_handle) self._solver_manager.get_results(action_handle) elif action_handle == ah: result = self._solver_manager.get_results(action_handle) break assert len(result) == 1 self._aggregate_user_data = result[0] else: for scenario in self._scenario_tree._scenarios: ah = scenariotreeserverutils.\ transmit_external_function_invocation_to_worker( self, scenario._name, self._callback_module_name[callback_name], callback_name, invocation_type=(scenariotreeserverutils.InvocationType.\ SingleInvocation), return_action_handle=True, function_args=(self._aggregate_user_data,)) while(1): action_handle = self._solver_manager.wait_any() if action_handle in initialization_action_handles: initialization_action_handles.remove(action_handle) self._solver_manager.get_results(action_handle) elif action_handle == ah: result = self._solver_manager.get_results(action_handle) break assert len(result) == 1 self._aggregate_user_data = result[0] # Transmit final aggregate state to phsolverservers print("Broadcasting final aggregate data to phsolverservers") initialization_action_handles.extend( scenariotreeserverutils.transmit_external_function_invocation( self, "pyomo.pysp.ph", "assign_aggregate_data", invocation_type=(scenariotreeserverutils.InvocationType.\ SingleInvocation), return_action_handles=True, function_args=(self._aggregate_user_data,))) else: print("Executing user aggregate getter callback function") for scenario in self._scenario_tree._scenarios: result = self._callback_function[callback_name]( self, self._scenario_tree, scenario, self._aggregate_user_data) assert len(result) == 1 self._aggregate_user_data = result[0] # if specified, run the user script to initialize variable # bounds at their whim. if self._options.bounds_cfgfile is not None: callback_name = "pysp_boundsetter_callback" if isPHPyro: # Transmit invocation to phsolverservers print("Transmitting user bound callback invocations to " "phsolverservers") if self._scenario_tree.contains_bundles(): for scenario_bundle in self._scenario_tree._scenario_bundles: initialization_action_handles.append( scenariotreeserverutils.\ transmit_external_function_invocation_to_worker( self, scenario_bundle._name, self._callback_module_name[callback_name], callback_name, invocation_type=(scenariotreeserverutils.InvocationType.\ PerScenarioInvocation), return_action_handle=True)) else: for scenario in self._scenario_tree._scenarios: initialization_action_handles.append( scenariotreeserverutils.\ transmit_external_function_invocation_to_worker( self, scenario._name, self._callback_module_name[callback_name], callback_name, invocation_type=(scenariotreeserverutils.InvocationType.\ SingleInvocation), return_action_handle=True)) else: print("Executing user bound setter callback function") for scenario in self._scenario_tree._scenarios: self._callback_function[callback_name]( self, self._scenario_tree, scenario) # gather scenario tree data if not local if isPHPyro: if self._options.verbose: print("Broadcasting requests to collect scenario tree " "instance data from PH solver servers") scenariotreeserverutils.\ gather_scenario_tree_data(self, initialization_action_handles) assert len(initialization_action_handles) == 0 if self._options.verbose: print("Scenario tree instance data successfully collected") if self._options.verbose: print("Broadcasting scenario tree id mapping" "to PH solver servers") scenariotreeserverutils.transmit_scenario_tree_ids(self) if self._options.verbose: print("Scenario tree ids successfully sent") self._objective_sense = \ self._scenario_tree._scenarios[0]._objective_sense if self._options.verbose: print("Scenario tree manager is successfully initialized") if self._options.output_times: print("Overall initialization time=%.2f seconds" % (time.time() - init_start_time)) # gather and report memory statistics (for leak # detection purposes) if specified. if (guppy_available) and (self._options.profile_memory >= 1): print(hpy().heap()) # indicate that we're ready to run. self._initialized = True