def __init__(self, fsfile, fsfct=None, tree_model=None, phopts=None): """Initialize a StochSolver object. """ if fsfct is None: # Changed in October 2018: None implies AbstractModel args_list = _optiondict_2_list(phopts) parser = phinit.construct_ph_options_parser("") options = parser.parse_args(args_list) scenario_instance_factory = \ ScenarioTreeInstanceFactory(fsfile, tree_model) try: self.scenario_tree = \ phinit.GenerateScenarioTreeForPH(options, scenario_instance_factory) except: print("ERROR in StochSolver called from", inspect.stack()[1][3]) raise RuntimeError( "fsfct is None, so assuming", "AbstractModel but could not find all ingredients.") else: # concrete model if callable(fsfct): scen_function = fsfct else: # better be a string fsfile = fsfile.replace('.py', '') # import does not like .py # __import__ only gives the top level module # probably need to be dealing with modules installed via setup.py m = __import__(fsfile) for n in fsfile.split(".")[1:]: m = getattr(m, n) scen_function = getattr(m, fsfct) if tree_model is None: treecbname = "pysp_scenario_tree_model_callback" tree_maker = getattr(m, treecbname) tree = tree_maker() scenario_instance_factory = ScenarioTreeInstanceFactory( scen_function, tree_model) else: # DLW March 21: still not correct scenario_instance_factory = \ ScenarioTreeInstanceFactory(scen_function, tree_model) kwargs = _kwfromphopts(phopts) self.scenario_tree = \ scenario_instance_factory.generate_scenario_tree(**kwargs) #verbose = True) instances = scenario_instance_factory. \ construct_instances_for_scenario_tree(self.scenario_tree) self.scenario_tree.linkInInstances(instances)
def test_random_bundles(self): self.assertTrue("ReferenceModel" not in sys.modules) scenario_instance_factory = \ ScenarioTreeInstanceFactory(farmer_model_dir, farmer_scenariotree_dir) scenario_tree = \ scenario_instance_factory.generate_scenario_tree(random_bundles=2) self.assertEqual(scenario_tree.contains_bundles(), True) self.assertEqual(len(scenario_tree._scenario_bundles), 2) self.assertTrue("ReferenceModel" in sys.modules) del sys.modules["ReferenceModel"]
def test_random_bundles(self): self.assertTrue("ReferenceModel" not in sys.modules) scenario_instance_factory = \ ScenarioTreeInstanceFactory(farmer_model_dir, farmer_scenariotree_dir) scenario_tree = \ scenario_instance_factory.generate_scenario_tree(random_bundles=2) self.assertEqual(scenario_tree.contains_bundles(), True) self.assertEqual(len(scenario_tree._scenario_bundles), 2) self.assertTrue("ReferenceModel" in sys.modules) del sys.modules["ReferenceModel"]
def __init__(self, fsfile, fsfct="pysp_instance_creation_callback", tree_model=None, phopts=None): """Initialize a StochSolver object. """ fsfile = fsfile.replace('.py', '') # import does not like .py # __import__ only gives the top level module # probably need to be dealing with modules installed via setup.py m = __import__(fsfile) for n in fsfile.split(".")[1:]: m = getattr(m, n) scen_function = getattr(m, fsfct) if tree_model is None: tree_maker = getattr(m, \ "pysp_scenario_tree_model_callback") tree = tree_maker() # tree_model = tree.as_concrete_model() tree_model = tree # DLW March 21: still not correct scenario_instance_factory = \ ScenarioTreeInstanceFactory(scen_function, tree_model) # ScenarioTreeInstanceFactory("ReferenceModel.py", tree_model) else: # DLW March 21: still not correct scenario_instance_factory = \ ScenarioTreeInstanceFactory(scen_function, tree_model) kwargs = _kwfromphopts(phopts) self.scenario_tree = \ scenario_instance_factory.generate_scenario_tree(**kwargs) # verbose = True) instances = scenario_instance_factory. \ construct_instances_for_scenario_tree(self.scenario_tree) self.scenario_tree.linkInInstances(instances)
def run(args=None): AllInOne = False # The value of AllInOne will be set to True for the "old" # computeconf (with fraction_for_solve) and will stay False for # the "new" computeconf (with MRP_directory_basename) try: conf_options_parser = construct_ph_options_parser( "computeconf [options]") conf_options_parser.add_argument( "--fraction-scenarios-for-solve", help= "The fraction of scenarios that are allocated to finding a solution. Default is None.", action="store", dest="fraction_for_solve", type=float, default=None) conf_options_parser.add_argument( "--number-samples-for-confidence-interval", help= "The number of samples of scenarios that are allocated to the confidence inteval (n_g). Default is None.", action="store", dest="n_g", type=int, default=None) conf_options_parser.add_argument( "--confidence-interval-alpha", help="The alpha level for the confidence interval. Default is 0.05", action="store", dest="confidence_interval_alpha", type=float, default=0.05) conf_options_parser.add_argument( "--solve-xhat-with-ph", help= "Perform xhat solve via PH rather than an EF solve. Default is False", action="store_true", dest="solve_xhat_with_ph", default=False) conf_options_parser.add_argument( "--random-seed", help= "Seed the random number generator used to select samples. Defaults to 0, indicating time seed will be used.", action="store", dest="random_seed", type=int, default=0) conf_options_parser.add_argument( "--append-file", help= "File to which summary run information is appended, for output tracking purposes.", action="store", dest="append_file", type=str, default=None) conf_options_parser.add_argument( "--write-xhat-solution", help= "Write xhat solutions (first stage variables only) to the append file? Defaults to False.", action="store_true", dest="write_xhat_solution", default=False) conf_options_parser.add_argument( "--generate-weighted-cvar", help="Add a weighted CVaR term to the primary objective", action="store_true", dest="generate_weighted_cvar", default=False) conf_options_parser.add_argument( "--cvar-weight", help= "The weight associated with the CVaR term in the risk-weighted objective formulation. Default is 1.0. If the weight is 0, then *only* a non-weighted CVaR cost will appear in the EF objective - the expected cost component will be dropped.", action="store", dest="cvar_weight", type=float, default=1.0) conf_options_parser.add_argument( "--risk-alpha", help= "The probability threshold associated with cvar (or any future) risk-oriented performance metrics. Default is 0.95.", action="store", dest="risk_alpha", type=float, default=0.95) conf_options_parser.add_argument( "--MRP-directory-basename", help= "The basename for the replicate directories. It will be appended by the number of the group (loop over n_g). Default is None", action="store", dest="MRP_directory_basename", type=str, default=None) options = conf_options_parser.parse_args(args=args) # temporary hack options._ef_options = conf_options_parser._ef_options options._ef_options.import_argparse(options) except SystemExit as _exc: # the parser throws a system exit if "-h" is specified - catch # it to exit gracefully. return _exc.code # seed the generator if a user-supplied seed is # provided. otherwise, python will seed from the current system # time. if options.random_seed > 0: random.seed(options.random_seed) start_time = time.time() if options.verbose: print("Importing model and scenario tree files") scenario_instance_factory = \ ScenarioTreeInstanceFactory(options.model_directory, options.instance_directory, options.verbose) if _OLD_OUTPUT: print("Loading reference model and scenario tree") if options.verbose or options.output_times: print("Time to import model and scenario " "tree structure files=%.2f seconds" % (time.time() - start_time)) try: scenario_tree = \ scenario_instance_factory.generate_scenario_tree( downsample_fraction=options.scenario_tree_downsample_fraction, bundles=options.scenario_bundle_specification, random_bundles=options.create_random_bundles, random_seed=options.scenario_tree_random_seed) # # print the input tree for validation/information purposes. # if options.verbose: scenario_tree.pprint() # # validate the tree prior to doing anything serious # if not scenario_tree.validate(): raise RuntimeError("Scenario tree is invalid") else: if options.verbose: print("Scenario tree is valid!") index_list, num_scenarios_for_solution, num_scenarios_per_sample = \ partition_scenario_space(scenario_tree, options) #index_list = [0,3,5,7,1,4,6,8,2,9] #for ndx in index_list: # print("%d: %s" % (ndx, scenario_tree._scenarios[ndx]._name)) xhat_ph = find_candidate(scenario_instance_factory, index_list, num_scenarios_for_solution, scenario_tree, options) run_conf(scenario_instance_factory, index_list, num_scenarios_for_solution, num_scenarios_per_sample, scenario_tree, xhat_ph, options) finally: # delete temporary unarchived directories if scenario_instance_factory is not None: scenario_instance_factory.close()
def run(args=None): AllInOne = False # The value of AllInOne will be set to True for the "old" # computeconf (with fraction_for_solve) and will stay False for # the "new" computeconf (with MRP_directory_basename) try: conf_options_parser = construct_ph_options_parser("computeconf [options]") conf_options_parser.add_argument( "--fraction-scenarios-for-solve", help="The fraction of scenarios that are allocated to finding a solution. Default is None.", action="store", dest="fraction_for_solve", type=float, default=None, ) conf_options_parser.add_argument( "--number-samples-for-confidence-interval", help="The number of samples of scenarios that are allocated to the confidence inteval (n_g). Default is None.", action="store", dest="n_g", type=int, default=None, ) conf_options_parser.add_argument( "--confidence-interval-alpha", help="The alpha level for the confidence interval. Default is 0.05", action="store", dest="confidence_interval_alpha", type=float, default=0.05, ) conf_options_parser.add_argument( "--solve-xhat-with-ph", help="Perform xhat solve via PH rather than an EF solve. Default is False", action="store_true", dest="solve_xhat_with_ph", default=False, ) conf_options_parser.add_argument( "--random-seed", help="Seed the random number generator used to select samples. Defaults to 0, indicating time seed will be used.", action="store", dest="random_seed", type=int, default=0, ) conf_options_parser.add_argument( "--append-file", help="File to which summary run information is appended, for output tracking purposes.", action="store", dest="append_file", type=str, default=None, ) conf_options_parser.add_argument( "--write-xhat-solution", help="Write xhat solutions (first stage variables only) to the append file? Defaults to False.", action="store_true", dest="write_xhat_solution", default=False, ) conf_options_parser.add_argument( "--generate-weighted-cvar", help="Add a weighted CVaR term to the primary objective", action="store_true", dest="generate_weighted_cvar", default=False, ) conf_options_parser.add_argument( "--cvar-weight", help="The weight associated with the CVaR term in the risk-weighted objective formulation. Default is 1.0. If the weight is 0, then *only* a non-weighted CVaR cost will appear in the EF objective - the expected cost component will be dropped.", action="store", dest="cvar_weight", type=float, default=1.0, ) conf_options_parser.add_argument( "--risk-alpha", help="The probability threshold associated with cvar (or any future) risk-oriented performance metrics. Default is 0.95.", action="store", dest="risk_alpha", type=float, default=0.95, ) conf_options_parser.add_argument( "--MRP-directory-basename", help="The basename for the replicate directories. It will be appended by the number of the group (loop over n_g). Default is None", action="store", dest="MRP_directory_basename", type=str, default=None, ) options = conf_options_parser.parse_args(args=args) # temporary hack options._ef_options = conf_options_parser._ef_options options._ef_options.import_argparse(options) except SystemExit as _exc: # the parser throws a system exit if "-h" is specified - catch # it to exit gracefully. return _exc.code # seed the generator if a user-supplied seed is # provided. otherwise, python will seed from the current system # time. if options.random_seed > 0: random.seed(options.random_seed) start_time = time.time() if options.verbose: print("Importing model and scenario tree files") scenario_instance_factory = ScenarioTreeInstanceFactory(options.model_directory, options.instance_directory) if _OLD_OUTPUT: print("Loading reference model and scenario tree") if options.verbose or options.output_times: print("Time to import model and scenario " "tree structure files=%.2f seconds" % (time.time() - start_time)) try: scenario_tree = scenario_instance_factory.generate_scenario_tree( downsample_fraction=options.scenario_tree_downsample_fraction, bundles=options.scenario_bundle_specification, random_bundles=options.create_random_bundles, random_seed=options.scenario_tree_random_seed, verbose=options.verbose, ) # # print the input tree for validation/information purposes. # if options.verbose: scenario_tree.pprint() # # validate the tree prior to doing anything serious # if not scenario_tree.validate(): raise RuntimeError("Scenario tree is invalid") else: if options.verbose: print("Scenario tree is valid!") index_list, num_scenarios_for_solution, num_scenarios_per_sample = partition_scenario_space( scenario_tree, options ) # index_list = [0,3,5,7,1,4,6,8,2,9] # for ndx in index_list: # print("%d: %s" % (ndx, scenario_tree._scenarios[ndx]._name)) xhat_ph = find_candidate( scenario_instance_factory, index_list, num_scenarios_for_solution, scenario_tree, options ) run_conf( scenario_instance_factory, index_list, num_scenarios_for_solution, num_scenarios_per_sample, scenario_tree, xhat_ph, options, ) finally: # delete temporary unarchived directories if scenario_instance_factory is not None: scenario_instance_factory.close()
class ScenarioTreeServerPyro(pyu_pyro.TaskWorker): # Maps name to a registered worker class to instantiate _registered_workers = {} @classmethod def get_registered_worker_type(cls, name): if name in cls._registered_workers: return cls._registered_workers[name] raise KeyError("No worker type has been registered under the name " "'%s' for ScenarioTreeServerPyro" % (name)) def __init__(self, *args, **kwds): mpi = kwds.pop('mpi', None) # add for purposes of diagnostic output. kwds["name"] = ("ScenarioTreeServerPyro_%d@%s" % (os.getpid(), socket.gethostname())) if mpi is not None: assert len(mpi) == 2 kwds["name"] += "_MPIRank_" + str(mpi[1].rank) kwds["caller_name"] = kwds["name"] self._modules_imported = kwds.pop('modules_imported', {}) pyu_pyro.TaskWorker.__init__(self, **kwds) assert hasattr(self, "_bulk_task_collection") self._bulk_task_collection = True self._contiguous_task_processing = False self.type = self.WORKERNAME self.block = True self.timeout = None self._worker_map = {} self._init_verbose = self._verbose # A reference to the mpi4py.MPI namespace self.MPI = None # The communicator and group associated with all processors self.mpi_comm_world = None self.mpi_group_world = None # The communicator associated with the workers assigned # to the current current client self.mpi_comm_workers = None if mpi is not None: assert len(mpi) == 2 self.MPI = mpi[0] self.mpi_comm_world = mpi[1] self.mpi_group_world = self.mpi_comm_world.Get_group() # # These will be used by all subsequent workers created # by this server. Their creation can eat up a nontrivial # amount of initialization time when a large number of # workers are created on this server, so we only create # them once. # self._scenario_instance_factory = None self._full_scenario_tree = None def reset(self): if self._scenario_instance_factory is not None: self._scenario_instance_factory.close() self._scenario_instance_factory = None self._full_scenario_tree = None for worker_name in list(self._worker_map): self.remove_worker(worker_name) if self.mpi_comm_workers is not None: self.mpi_comm_workers.Free() self.mpi_comm_workers = None self._verbose = self._init_verbose def remove_worker(self, name): self._worker_map[name].close() del self._worker_map[name] def process(self, data): self._worker_task_return_queue = self._current_task_client try: # The only reason we are go through this much # effort to deal with the serpent serializer # is because it is the default in Pyro4. if pyu_pyro.using_pyro4 and \ (Pyro4.config.SERIALIZER == 'serpent'): if six.PY3: assert type(data) is dict assert data['encoding'] == 'base64' data = base64.b64decode(data['data']) else: assert type(data) is unicode data = str(data) return pickle.dumps(self._process(pickle.loads(data))) except: logger.error("Scenario tree server %s caught an exception of type " "%s while processing a task. Going idle." % (self.WORKERNAME, sys.exc_info()[0].__name__)) traceback.print_exception(*sys.exc_info()) self._worker_error = True return pickle.dumps( pyu_pyro.TaskProcessingError(traceback.format_exc())) def _process(self, data): data = Bunch(**data) result = None if not data.action.startswith('ScenarioTreeServerPyro_'): #with PauseGC() as pgc: result = getattr(self._worker_map[data.worker_name], data.action)\ (*data.args, **data.kwds) elif data.action == 'ScenarioTreeServerPyro_setup': model_input = data.options.pop('model', None) if model_input is None: model_input = data.options.pop('model_callback') assert dill_available model_input = dill.loads(model_input) scenario_tree_input = data.options.pop('scenario_tree') data_input = data.options.pop('data') mpi_group = data.options.pop("mpi_group", None) verbose = data.options.pop("verbose", False) assert len(data.options) == 0 self._verbose |= verbose assert self._scenario_instance_factory is None assert self._full_scenario_tree is None if self._verbose: print("Server %s received setup request." % (self.WORKERNAME)) # Make sure these are not archives assert (not isinstance(model_input, six.string_types)) or \ os.path.exists(model_input) assert isinstance(scenario_tree_input, ScenarioTree) self._scenario_instance_factory = \ ScenarioTreeInstanceFactory( model_input, scenario_tree_input, data=data_input) # # Try to prevent unnecessarily re-importing the model module # if other callbacks are in the same location. Doing so might # have serious consequences. # if self._scenario_instance_factory._model_module is not None: self._modules_imported[self._scenario_instance_factory.\ _model_filename] = \ self._scenario_instance_factory._model_module assert self._scenario_instance_factory._scenario_tree_module is None self._full_scenario_tree = \ self._scenario_instance_factory.generate_scenario_tree() assert self.mpi_comm_workers is None if self.mpi_comm_world is not None: assert self.mpi_group_world is not None assert mpi_group is not None mpi_group = self.mpi_group_world.Incl(mpi_group) self.mpi_comm_workers = \ self.mpi_comm_world.Create_group(mpi_group) else: assert mpi_group is None if self._full_scenario_tree is None: raise RuntimeError("Unable to launch scenario tree worker - " "scenario tree construction failed.") result = True elif data.action == "ScenarioTreeServerPyro_initialize": worker_name = data.worker_name if self._verbose: print("Server %s received request to initialize " "scenario tree worker with name %s." % (self.WORKERNAME, worker_name)) assert self._scenario_instance_factory is not None assert self._full_scenario_tree is not None if worker_name in self._worker_map: raise RuntimeError( "Server %s Cannot initialize worker with name '%s' " "because a worker already exists with that name." % (self.WORKERNAME, worker_name)) worker_type = self._registered_workers[data.worker_type] self._worker_map[worker_name] = worker_type( self, worker_name, *data.init_args, **data.init_kwds) result = True elif data.action == "ScenarioTreeServerPyro_release": if self._verbose: print("Server %s releasing worker: %s" % (self.WORKERNAME, data.worker_name)) self.remove_worker(data.worker_name) result = True elif data.action == "ScenarioTreeServerPyro_reset": if self._verbose: print("Server %s received reset request" % (self.WORKERNAME)) self.reset() result = True elif data.action == "ScenarioTreeServerPyro_shutdown": if self._verbose: print("Server %s received shutdown request" % (self.WORKERNAME)) self.reset() self._worker_shutdown = True result = True else: raise ValueError("Server %s: Invalid command: %s" % (self.WORKERNAME, data.action)) return result
class ScenarioTreeServerPyro(TaskWorker, PySPConfiguredObject): # Maps name to a registered worker class to instantiate _registered_workers = {} _declared_options = \ PySPConfigBlock("Options declared for the " "ScenarioTreeServerPyro class") # # scenario instance construction # safe_declare_common_option(_declared_options, "model_location") safe_declare_common_option(_declared_options, "scenario_tree_location") # # scenario tree generation # safe_declare_common_option(_declared_options, "scenario_tree_random_seed") safe_declare_common_option(_declared_options, "scenario_tree_downsample_fraction") # # various # safe_declare_common_option(_declared_options, "verbose") @classmethod def get_registered_worker_type(cls, name): if name in cls._registered_workers: return cls._registered_workers[name] raise KeyError("No worker type has been registered under the name " "'%s' for ScenarioTreeServerPyro" % (name)) def __init__(self, *args, **kwds): # add for purposes of diagnostic output. kwds["name"] = ("ScenarioTreeServerPyro_%d@%s" % (os.getpid(), socket.gethostname())) kwds["caller_name"] = kwds["name"] self._modules_imported = kwds.pop('modules_imported', {}) TaskWorker.__init__(self, **kwds) # This classes options get updated during the "setup" phase options = self.register_options() PySPConfiguredObject.__init__(self, options) self.type = self.WORKERNAME self.block = True self.timeout = None self._worker_map = {} # # These will be used by all subsequent workers created # by this server. Their creation can eat up a nontrivial # amount of initialization time when a large number of # workers are created on this server, so we only create # them once. # self._scenario_instance_factory = None self._full_scenario_tree = None def reset(self): if self._scenario_instance_factory is not None: self._scenario_instance_factory.close() self._scenario_instance_factory = None self._full_scenario_tree = None for worker_name in list(self._worker_map): self.remove_worker(worker_name) def remove_worker(self, name): self._worker_map[name].close() del self._worker_map[name] def process(self, data): self._worker_task_return_queue = self._current_task_client try: return self._process(data) except: logger.error( "Scenario tree server %s caught an exception of type " "%s while processing a task. Going idle." % (self.WORKERNAME, sys.exc_info()[0].__name__)) traceback.print_exception(*sys.exc_info()) self._worker_error = True return TaskProcessingError(traceback.format_exc()) def _process(self, data): data = pyutilib.misc.Bunch(**data) result = None if not data.action.startswith('ScenarioTreeServerPyro_'): #with PauseGC() as pgc: result = getattr(self._worker_map[data.worker_name], data.action)\ (*data.args, **data.kwds) elif data.action == 'ScenarioTreeServerPyro_setup': options = self.register_options() for name, val in iteritems(data.options): options.get(name).set_value(val) self.set_options(options) self._options.verbose = self._options.verbose | self._verbose assert self._scenario_instance_factory is None assert self._full_scenario_tree is None if self._options.verbose: print("Server %s received setup request." % (self.WORKERNAME)) print("Options:") self.display_options() # Make sure these are not archives assert os.path.exists(self._options.model_location) assert (self._options.scenario_tree_location is None) or \ os.path.exists(self._options.scenario_tree_location) self._scenario_instance_factory = \ ScenarioTreeInstanceFactory( self._options.model_location, scenario_tree_location=self._options.scenario_tree_location, verbose=self._options.verbose) # # Try prevent unnecessarily re-imported the model module # if other callbacks are in the same location # self._modules_imported[ self._scenario_instance_factory._model_location] = \ self._scenario_instance_factory._model_module self._modules_imported[ self._scenario_instance_factory._model_filename] = \ self._scenario_instance_factory._model_module self._full_scenario_tree = \ self._scenario_instance_factory.generate_scenario_tree( downsample_fraction=self._options.scenario_tree_downsample_fraction, random_seed=self._options.scenario_tree_random_seed) if self._full_scenario_tree is None: raise RuntimeError("Unable to launch scenario tree worker - " "scenario tree construction failed.") result = True elif data.action == "ScenarioTreeServerPyro_initialize": worker_name = data.worker_name if self._options.verbose: print("Server %s received request to initialize " "scenario tree worker with name %s." % (self.WORKERNAME, worker_name)) assert self._scenario_instance_factory is not None assert self._full_scenario_tree is not None if worker_name in self._worker_map: raise RuntimeError( "Server %s Cannot initialize worker with name '%s' " "because a worker already exists with that name." % (self.WORKERNAME, worker_name)) worker_type = self._registered_workers[data.worker_type] options = worker_type.register_options() for name, val in iteritems(data.options): options.get(name).set_value(val) # # Depending on the Pyro serializer, the namedtuple # may be been converted to a tuple # if not isinstance(data.worker_init, WorkerInit): assert type(data.worker_init) is tuple data.worker_init = WorkerInit(type_=data.worker_init[0], names=data.worker_init[1], data=data.worker_init[2]) # replace enum string representation with the actual enum # object now that we've unserialized the Pyro data worker_init = WorkerInit(type_=getattr(WorkerInitType, data.worker_init.type_), names=data.worker_init.names, data=data.worker_init.data) self._worker_map[worker_name] = worker_type(options) self._worker_map[worker_name].initialize( self.WORKERNAME, self._full_scenario_tree, worker_name, worker_init, self._modules_imported) result = True elif data.action == "ScenarioTreeServerPyro_release": if self._options.verbose: print("Server %s releasing worker: %s" % (self.WORKERNAME, data.worker_name)) self.remove_worker(data.worker_name) result = True elif data.action == "ScenarioTreeServerPyro_reset": if self._options.verbose: print("Server %s received reset request" % (self.WORKERNAME)) self.reset() result = True elif data.action == "ScenarioTreeServerPyro_shutdown": if self._options.verbose: print("Server %s received shutdown request" % (self.WORKERNAME)) self.reset() self._worker_shutdown = True result = True else: raise ValueError("Server %s: Invalid command: %s" % (self.WORKERNAME, data.action)) return result
class ScenarioTreeServerPyro(TaskWorker, PySPConfiguredObject): # Maps name to a registered worker class to instantiate _registered_workers = {} _declared_options = \ PySPConfigBlock("Options declared for the " "ScenarioTreeServerPyro class") # # scenario instance construction # safe_declare_common_option(_declared_options, "model_location") safe_declare_common_option(_declared_options, "scenario_tree_location") # # scenario tree generation # safe_declare_common_option(_declared_options, "scenario_tree_random_seed") safe_declare_common_option(_declared_options, "scenario_tree_downsample_fraction") # # various # safe_declare_common_option(_declared_options, "verbose") @classmethod def get_registered_worker_type(cls, name): if name in cls._registered_workers: return cls._registered_workers[name] raise KeyError("No worker type has been registered under the name " "'%s' for ScenarioTreeServerPyro" % (name)) def __init__(self, *args, **kwds): # add for purposes of diagnostic output. kwds["name"] = ("ScenarioTreeServerPyro_%d@%s" % (os.getpid(), socket.gethostname())) kwds["caller_name"] = kwds["name"] self._modules_imported = kwds.pop('modules_imported', {}) TaskWorker.__init__(self, **kwds) # This classes options get updated during the "setup" phase options = self.register_options() PySPConfiguredObject.__init__(self, options) self.type = self.WORKERNAME self.block = True self.timeout = None self._worker_map = {} # # These will be used by all subsequent workers created # by this server. Their creation can eat up a nontrivial # amount of initialization time when a large number of # workers are created on this server, so we only create # them once. # self._scenario_instance_factory = None self._full_scenario_tree = None def reset(self): if self._scenario_instance_factory is not None: self._scenario_instance_factory.close() self._scenario_instance_factory = None self._full_scenario_tree = None for worker_name in list(self._worker_map): self.remove_worker(worker_name) def remove_worker(self, name): self._worker_map[name].close() del self._worker_map[name] def process(self, data): self._worker_task_return_queue = self._current_task_client try: return self._process(data) except: logger.error("Scenario tree server %s caught an exception of type " "%s while processing a task. Going idle." % (self.WORKERNAME, sys.exc_info()[0].__name__)) traceback.print_exception(*sys.exc_info()) self._worker_error = True return TaskProcessingError(traceback.format_exc()) def _process(self, data): data = pyutilib.misc.Bunch(**data) result = None if not data.action.startswith('ScenarioTreeServerPyro_'): #with PauseGC() as pgc: result = getattr(self._worker_map[data.worker_name], data.action)\ (*data.args, **data.kwds) elif data.action == 'ScenarioTreeServerPyro_setup': options = self.register_options() for name, val in iteritems(data.options): options.get(name).set_value(val) self.set_options(options) self._options.verbose = self._options.verbose | self._verbose assert self._scenario_instance_factory is None assert self._full_scenario_tree is None if self._options.verbose: print("Server %s received setup request." % (self.WORKERNAME)) print("Options:") self.display_options() # Make sure these are not archives assert os.path.exists(self._options.model_location) assert (self._options.scenario_tree_location is None) or \ os.path.exists(self._options.scenario_tree_location) self._scenario_instance_factory = \ ScenarioTreeInstanceFactory( self._options.model_location, scenario_tree_location=self._options.scenario_tree_location, verbose=self._options.verbose) # # Try prevent unnecessarily re-imported the model module # if other callbacks are in the same location # self._modules_imported[ self._scenario_instance_factory._model_location] = \ self._scenario_instance_factory._model_module self._modules_imported[ self._scenario_instance_factory._model_filename] = \ self._scenario_instance_factory._model_module self._full_scenario_tree = \ self._scenario_instance_factory.generate_scenario_tree( downsample_fraction=self._options.scenario_tree_downsample_fraction, random_seed=self._options.scenario_tree_random_seed) if self._full_scenario_tree is None: raise RuntimeError("Unable to launch scenario tree worker - " "scenario tree construction failed.") result = True elif data.action == "ScenarioTreeServerPyro_initialize": worker_name = data.worker_name if self._options.verbose: print("Server %s received request to initialize " "scenario tree worker with name %s." % (self.WORKERNAME, worker_name)) assert self._scenario_instance_factory is not None assert self._full_scenario_tree is not None if worker_name in self._worker_map: raise RuntimeError( "Server %s Cannot initialize worker with name '%s' " "because a worker already exists with that name." % (self.WORKERNAME, worker_name)) worker_type = self._registered_workers[data.worker_type] options = worker_type.register_options() for name, val in iteritems(data.options): options.get(name).set_value(val) # # Depending on the Pyro serializer, the namedtuple # may be been converted to a tuple # if not isinstance(data.worker_init, WorkerInit): assert type(data.worker_init) is tuple data.worker_init = WorkerInit(type_=data.worker_init[0], names=data.worker_init[1], data=data.worker_init[2]) # replace enum string representation with the actual enum # object now that we've unserialized the Pyro data worker_init = WorkerInit(type_=getattr(WorkerInitType, data.worker_init.type_), names=data.worker_init.names, data=data.worker_init.data) self._worker_map[worker_name] = worker_type(options) self._worker_map[worker_name].initialize(self.WORKERNAME, self._full_scenario_tree, worker_name, worker_init, self._modules_imported) result = True elif data.action == "ScenarioTreeServerPyro_release": if self._options.verbose: print("Server %s releasing worker: %s" % (self.WORKERNAME, data.worker_name)) self.remove_worker(data.worker_name) result = True elif data.action == "ScenarioTreeServerPyro_reset": if self._options.verbose: print("Server %s received reset request" % (self.WORKERNAME)) self.reset() result = True elif data.action == "ScenarioTreeServerPyro_shutdown": if self._options.verbose: print("Server %s received shutdown request" % (self.WORKERNAME)) self.reset() self._worker_shutdown = True result = True else: raise ValueError("Server %s: Invalid command: %s" % (self.WORKERNAME, data.action)) return result
class ScenarioTreeServerPyro(TaskWorker): # Maps name to a registered worker class to instantiate _registered_workers = {} @classmethod def get_registered_worker_type(cls, name): if name in cls._registered_workers: return cls._registered_workers[name] raise KeyError("No worker type has been registered under the name " "'%s' for ScenarioTreeServerPyro" % (name)) def __init__(self, *args, **kwds): mpi = kwds.pop('mpi', None) # add for purposes of diagnostic output. kwds["name"] = ("ScenarioTreeServerPyro_%d@%s" % (os.getpid(), socket.gethostname())) if mpi is not None: assert len(mpi) == 2 kwds["name"] += "_MPIRank_"+str(mpi[1].rank) kwds["caller_name"] = kwds["name"] self._modules_imported = kwds.pop('modules_imported', {}) TaskWorker.__init__(self, **kwds) assert hasattr(self, "_bulk_task_collection") self._bulk_task_collection = True self._contiguous_task_processing = False self.type = self.WORKERNAME self.block = True self.timeout = None self._worker_map = {} self._init_verbose = self._verbose # A reference to the mpi4py.MPI namespace self.MPI = None # The communicator and group associated with all processors self.mpi_comm_world = None self.mpi_group_world = None # The communicator associated with the workers assigned # to the current current client self.mpi_comm_workers = None if mpi is not None: assert len(mpi) == 2 self.MPI = mpi[0] self.mpi_comm_world = mpi[1] self.mpi_group_world = self.mpi_comm_world.Get_group() # # These will be used by all subsequent workers created # by this server. Their creation can eat up a nontrivial # amount of initialization time when a large number of # workers are created on this server, so we only create # them once. # self._scenario_instance_factory = None self._full_scenario_tree = None def reset(self): if self._scenario_instance_factory is not None: self._scenario_instance_factory.close() self._scenario_instance_factory = None self._full_scenario_tree = None for worker_name in list(self._worker_map): self.remove_worker(worker_name) if self.mpi_comm_workers is not None: self.mpi_comm_workers.Free() self.mpi_comm_workers = None self._verbose = self._init_verbose def remove_worker(self, name): self._worker_map[name].close() del self._worker_map[name] def process(self, data): self._worker_task_return_queue = self._current_task_client try: # The only reason we are go through this much # effort to deal with the serpent serializer # is because it is the default in Pyro4. if using_pyro4 and \ (Pyro4.config.SERIALIZER == 'serpent'): if six.PY3: assert type(data) is dict assert data['encoding'] == 'base64' data = base64.b64decode(data['data']) else: assert type(data) is unicode data = str(data) return pickle.dumps(self._process(pickle.loads(data))) except: logger.error( "Scenario tree server %s caught an exception of type " "%s while processing a task. Going idle." % (self.WORKERNAME, sys.exc_info()[0].__name__)) traceback.print_exception(*sys.exc_info()) self._worker_error = True return pickle.dumps(TaskProcessingError(traceback.format_exc())) def _process(self, data): data = pyutilib.misc.Bunch(**data) result = None if not data.action.startswith('ScenarioTreeServerPyro_'): #with PauseGC() as pgc: result = getattr(self._worker_map[data.worker_name], data.action)\ (*data.args, **data.kwds) elif data.action == 'ScenarioTreeServerPyro_setup': model_input = data.options.pop('model', None) if model_input is None: model_input = data.options.pop('model_callback') assert dill_available model_input = dill.loads(model_input) scenario_tree_input = data.options.pop('scenario_tree') data_input = data.options.pop('data') mpi_group = data.options.pop("mpi_group",None) verbose = data.options.pop("verbose", False) assert len(data.options) == 0 self._verbose |= verbose assert self._scenario_instance_factory is None assert self._full_scenario_tree is None if self._verbose: print("Server %s received setup request." % (self.WORKERNAME)) # Make sure these are not archives assert (not isinstance(model_input, six.string_types)) or \ os.path.exists(model_input) assert isinstance(scenario_tree_input, ScenarioTree) self._scenario_instance_factory = \ ScenarioTreeInstanceFactory( model_input, scenario_tree_input, data=data_input) # # Try to prevent unnecessarily re-importing the model module # if other callbacks are in the same location. Doing so might # have serious consequences. # if self._scenario_instance_factory._model_module is not None: self._modules_imported[self._scenario_instance_factory.\ _model_filename] = \ self._scenario_instance_factory._model_module assert self._scenario_instance_factory._scenario_tree_module is None self._full_scenario_tree = \ self._scenario_instance_factory.generate_scenario_tree() assert self.mpi_comm_workers is None if self.mpi_comm_world is not None: assert self.mpi_group_world is not None assert mpi_group is not None mpi_group = self.mpi_group_world.Incl(mpi_group) self.mpi_comm_workers = \ self.mpi_comm_world.Create_group(mpi_group) else: assert mpi_group is None if self._full_scenario_tree is None: raise RuntimeError("Unable to launch scenario tree worker - " "scenario tree construction failed.") result = True elif data.action == "ScenarioTreeServerPyro_initialize": worker_name = data.worker_name if self._verbose: print("Server %s received request to initialize " "scenario tree worker with name %s." % (self.WORKERNAME, worker_name)) assert self._scenario_instance_factory is not None assert self._full_scenario_tree is not None if worker_name in self._worker_map: raise RuntimeError( "Server %s Cannot initialize worker with name '%s' " "because a worker already exists with that name." % (self.WORKERNAME, worker_name)) worker_type = self._registered_workers[data.worker_type] self._worker_map[worker_name] = worker_type( self, worker_name, *data.init_args, **data.init_kwds) result = True elif data.action == "ScenarioTreeServerPyro_release": if self._verbose: print("Server %s releasing worker: %s" % (self.WORKERNAME, data.worker_name)) self.remove_worker(data.worker_name) result = True elif data.action == "ScenarioTreeServerPyro_reset": if self._verbose: print("Server %s received reset request" % (self.WORKERNAME)) self.reset() result = True elif data.action == "ScenarioTreeServerPyro_shutdown": if self._verbose: print("Server %s received shutdown request" % (self.WORKERNAME)) self.reset() self._worker_shutdown = True result = True else: raise ValueError("Server %s: Invalid command: %s" % (self.WORKERNAME, data.action)) return result
def __init__(self, fsfile, fsfct = None, tree_model = None, phopts = None): """Initialize a StochSolver object. """ if fsfct is None: # Changed in October 2018: None implies AbstractModel args_list = _optiondict_2_list(phopts) parser = phinit.construct_ph_options_parser("") options = parser.parse_args(args_list) scenario_instance_factory = \ ScenarioTreeInstanceFactory(fsfile, tree_model) try: self.scenario_tree = \ phinit.GenerateScenarioTreeForPH(options, scenario_instance_factory) except: print ("ERROR in StochSolver called from",inspect.stack()[1][3]) raise RuntimeError("fsfct is None, so assuming", "AbstractModel but could not find all ingredients.") else: # concrete model if callable(fsfct): scen_function = fsfct else: # better be a string fsfile = fsfile.replace('.py','') # import does not like .py # __import__ only gives the top level module # probably need to be dealing with modules installed via setup.py m = __import__(fsfile) for n in fsfile.split(".")[1:]: m = getattr(m, n) scen_function = getattr(m, fsfct) if tree_model is None: treecbname = "pysp_scenario_tree_model_callback" tree_maker = getattr(m, treecbname) tree = tree_maker() if isinstance(tree, Pyo.ConcreteModel): tree_model = tree else: raise RuntimeError("The tree returned by",treecbname, "must be a ConcreteModel") scenario_instance_factory = ScenarioTreeInstanceFactory(scen_function, tree_model) else: # DLW March 21: still not correct scenario_instance_factory = \ ScenarioTreeInstanceFactory(scen_function, tree_model) kwargs = _kwfromphopts(phopts) self.scenario_tree = \ scenario_instance_factory.generate_scenario_tree(**kwargs) #verbose = True) instances = scenario_instance_factory. \ construct_instances_for_scenario_tree(self.scenario_tree) self.scenario_tree.linkInInstances(instances)