Ejemplo n.º 1
0
def pysp_instance_creation_callback(scenario_name, node_names, cb_data):

    #print("Building instance for scenario =", scenario_name)
    scennum = sputils.extract_num(scenario_name)

    uc_model_params = pdp.get_uc_model()

    # Now using cb_data
    ##path = os.environ[UC_NUMSCENS_ENV_VAR] + "scenarios_r1"
    path = cb_data["path"]

    scenario_data = DataPortal(model=uc_model_params)
    scenario_data.load(filename=path + os.sep + "RootNode.dat")
    scenario_data.load(filename=path + os.sep + "Node" + str(scennum) + ".dat")

    scenario_params = uc_model_params.create_instance(scenario_data,
                                                      report_timing=False,
                                                      name=scenario_name)

    scenario_md = md.ModelData(
        pdp.create_model_data_dict_params(scenario_params, keep_names=True))

    ## TODO: use the "power_balance_constraints" for now. In the future, networks should be
    ##       handled with a custom callback -- also consider other base models
    scenario_instance = uc.create_tight_unit_commitment_model(
        scenario_md, network_constraints='power_balance_constraints')

    # hold over string attribute from Egret,
    # causes warning wth LShaped/Benders
    del scenario_instance.objective

    return scenario_instance
Ejemplo n.º 2
0
def pysp_instance_creation_callback(scenario_name,
                                    path=None,
                                    scenario_count=None):
    """
    Notes:
    - The uc_cylinders.py code has a `scenario_count` kwarg that gets passed to
      the spokes, but it seems to be unused here...
    """
    #print("Building instance for scenario =", scenario_name)
    scennum = sputils.extract_num(scenario_name)

    uc_model_params = pdp.get_uc_model()

    scenario_data = DataPortal(model=uc_model_params)
    scenario_data.load(filename=path + os.sep + "RootNode.dat")
    scenario_data.load(filename=path + os.sep + "Node" + str(scennum) + ".dat")

    scenario_params = uc_model_params.create_instance(scenario_data,
                                                      report_timing=False,
                                                      name=scenario_name)

    scenario_md = md.ModelData(
        pdp.create_model_data_dict_params(scenario_params, keep_names=True))

    ## TODO: use the "power_balance_constraints" for now. In the future, networks should be
    ##       handled with a custom callback -- also consider other base models
    scenario_instance = uc.create_tight_unit_commitment_model(
        scenario_md, network_constraints='power_balance_constraints')

    # hold over string attribute from Egret,
    # causes warning wth LShaped/Benders
    del scenario_instance.objective

    return scenario_instance
Ejemplo n.º 3
0
def create_model(data):
    """
    Create instance of Pyomo model.

    Return:
        model:      Model object.
        instance:   Problem instance.
        symbol_map: Symbol map created when writing model to a file.
        filename:    Filename that a model instance was written to.
    """
    #
    if not data.options.runtime.logging == 'quiet':
        sys.stdout.write('[%8.2f] Creating model\n' %
                         (time.time() - start_time))
        sys.stdout.flush()
    #
    if data.options.runtime.profile_memory >= 1 and pympler_available:
        global memory_data
        mem_used = pympler.muppy.get_size(pympler.muppy.get_objects())
        data.local.max_memory = mem_used
        print("   Total memory = %d bytes prior to model construction" %
              mem_used)
    #
    # Find the Model objects
    #
    _models = {}
    _model_IDS = set()
    for _name, _obj in data.local.usermodel.__dict__.items():
        if isinstance(_obj, Model) and id(_obj) not in _model_IDS:
            _models[_name] = _obj
            _model_IDS.add(id(_obj))
    model_name = data.options.model.object_name
    if len(_models) == 1:
        _name = list(_models.keys())[0]
        if model_name is None:
            model_name = _name
        elif model_name != _name:
            msg = "Model '%s' is not defined in file '%s'!"
            raise SystemExit(msg % (model_name, data.options.model.filename))
    elif len(_models) > 1:
        if model_name is None:
            msg = "Multiple models defined in file '%s'!"
            raise SystemExit(msg % data.options.model.filename)
        elif not model_name in _models:
            msg = "Unknown model '%s' in file '%s'!"
            raise SystemExit(msg % (model_name, data.options.model.filename))

    ep = ExtensionPoint(IPyomoScriptCreateModel)

    if model_name is None:
        if len(ep) == 0:
            msg = "A model is not defined and the 'pyomo_create_model' is not "\
                  "provided in module %s"
            raise SystemExit(msg % data.options.model.filename)
        elif len(ep) > 1:
            msg = 'Multiple model construction plugins have been registered in module %s!'
            raise SystemExit(msg % data.options.model.filename)
        else:
            model_options = data.options.model.options.value()
            tick = time.time()
            model = ep.service().apply(options=Bunch(*data.options),
                                       model_options=Bunch(*model_options))
            if data.options.runtime.report_timing is True:
                print("      %6.2f seconds required to construct instance" %
                      (time.time() - tick))
                data.local.time_initial_import = None
                tick = time.time()
    else:
        if model_name not in _models:
            msg = "Model '%s' is not defined in file '%s'!"
            raise SystemExit(msg % (model_name, data.options.model.filename))
        model = _models[model_name]
        if model is None:
            msg = "'%s' object is 'None' in module %s"
            raise SystemExit(msg % (model_name, data.options.model.filename))
        elif len(ep) > 0:
            msg = "Model construction function 'create_model' defined in "    \
                  "file '%s', but model is already constructed!"
            raise SystemExit(msg % data.options.model.filename)

    #
    # Print model
    #
    for ep in ExtensionPoint(IPyomoScriptPrintModel):
        ep.apply(options=data.options, model=model)

    #
    # Create Problem Instance
    #
    ep = ExtensionPoint(IPyomoScriptCreateDataPortal)
    if len(ep) > 1:
        msg = 'Multiple model data construction plugins have been registered!'
        raise SystemExit(msg)

    if len(ep) == 1:
        modeldata = ep.service().apply(options=data.options, model=model)
    else:
        modeldata = DataPortal()

    if model._constructed:
        #
        # TODO: use a better test for ConcreteModel
        #
        instance = model
        if data.options.runtime.report_timing is True and not data.local.time_initial_import is None:
            print("      %6.2f seconds required to construct instance" %
                  (data.local.time_initial_import))
    else:
        tick = time.time()
        if len(data.options.data.files) > 1:
            #
            # Load a list of *.dat files
            #
            for file in data.options.data.files:
                suffix = (file).split(".")[-1]
                if suffix != "dat":
                    msg = 'When specifiying multiple data files, they must all '  \
                          'be *.dat files.  File specified: %s'
                    raise SystemExit(msg % str(file))

                modeldata.load(filename=file, model=model)

            instance = model.create_instance(
                modeldata,
                namespaces=data.options.data.namespaces,
                profile_memory=data.options.runtime.profile_memory,
                report_timing=data.options.runtime.report_timing)

        elif len(data.options.data.files) == 1:
            #
            # Load a *.dat file or process a *.py data file
            #
            suffix = (data.options.data.files[0]).split(".")[-1].lower()
            if suffix == "dat":
                instance = model.create_instance(
                    data.options.data.files[0],
                    namespaces=data.options.data.namespaces,
                    profile_memory=data.options.runtime.profile_memory,
                    report_timing=data.options.runtime.report_timing)
            elif suffix == "py":
                userdata = import_file(data.options.data.files[0],
                                       clear_cache=True)
                if "modeldata" in dir(userdata):
                    if len(ep) == 1:
                        msg = "Cannot apply 'pyomo_create_modeldata' and use the" \
                              " 'modeldata' object that is provided in the model"
                        raise SystemExit(msg)

                    if userdata.modeldata is None:
                        msg = "'modeldata' object is 'None' in module %s"
                        raise SystemExit(msg % str(data.options.data.files[0]))

                    modeldata = userdata.modeldata

                else:
                    if len(ep) == 0:
                        msg = "Neither 'modeldata' nor 'pyomo_create_dataportal' "  \
                              'is defined in module %s'
                        raise SystemExit(msg % str(data.options.data.files[0]))

                modeldata.read(model)
                instance = model.create_instance(
                    modeldata,
                    namespaces=data.options.data.namespaces,
                    profile_memory=data.options.runtime.profile_memory,
                    report_timing=data.options.runtime.report_timing)
            elif suffix == "yml" or suffix == 'yaml':
                modeldata = yaml.load(open(data.options.data.files[0]),
                                      **yaml_load_args)
                instance = model.create_instance(
                    modeldata,
                    namespaces=data.options.data.namespaces,
                    profile_memory=data.options.runtime.profile_memory,
                    report_timing=data.options.runtime.report_timing)
            else:
                raise ValueError("Unknown data file type: " +
                                 data.options.data.files[0])
        else:
            instance = model.create_instance(
                modeldata,
                namespaces=data.options.data.namespaces,
                profile_memory=data.options.runtime.profile_memory,
                report_timing=data.options.runtime.report_timing)
        if data.options.runtime.report_timing is True:
            print("      %6.2f seconds required to construct instance" %
                  (time.time() - tick))

    #
    modify_start_time = time.time()
    for ep in ExtensionPoint(IPyomoScriptModifyInstance):
        if data.options.runtime.report_timing is True:
            tick = time.time()
        ep.apply(options=data.options, model=model, instance=instance)
        if data.options.runtime.report_timing is True:
            print("      %6.2f seconds to apply %s" %
                  (time.time() - tick, type(ep)))
            tick = time.time()
    #
    for transformation in data.options.transform:
        with TransformationFactory(transformation) as xfrm:
            instance = xfrm.create_using(instance)
            if instance is None:
                raise SystemExit("Unexpected error while applying "
                                 "transformation '%s'" % transformation)
    #
    if data.options.runtime.report_timing is True:
        total_time = time.time() - modify_start_time
        print("      %6.2f seconds required for problem transformations" %
              total_time)

    if is_debug_set(logger):
        print("MODEL INSTANCE")
        instance.pprint()
        print("")

    for ep in ExtensionPoint(IPyomoScriptPrintInstance):
        ep.apply(options=data.options, instance=instance)

    fname = None
    smap_id = None
    if not data.options.model.save_file is None:

        if data.options.runtime.report_timing is True:
            write_start_time = time.time()

        if data.options.model.save_file == True:
            if data.local.model_format in (ProblemFormat.cpxlp,
                                           ProblemFormat.lpxlp):
                fname = (data.options.data.files[0])[:-3] + 'lp'
            else:
                fname = (data.options.data.files[0])[:-3] + str(
                    data.local.model_format)
            format = data.local.model_format
        else:
            fname = data.options.model.save_file
            format = data.options.model.save_format

        io_options = {}
        if data.options.model.symbolic_solver_labels:
            io_options['symbolic_solver_labels'] = True
        if data.options.model.file_determinism != 1:
            io_options[
                'file_determinism'] = data.options.model.file_determinism
        (fname, smap_id) = instance.write(filename=fname,
                                          format=format,
                                          io_options=io_options)

        if not data.options.runtime.logging == 'quiet':
            if not os.path.exists(fname):
                print("ERROR: file " + fname + " has not been created!")
            else:
                print("Model written to file '" + str(fname) + "'")

        if data.options.runtime.report_timing is True:
            total_time = time.time() - write_start_time
            print("      %6.2f seconds required to write file" % total_time)

        if data.options.runtime.profile_memory >= 2 and pympler_available:
            print("")
            print("      Summary of objects following file output")
            post_file_output_summary = pympler.summary.summarize(
                pympler.muppy.get_objects())
            pympler.summary.print_(post_file_output_summary, limit=100)

            print("")

    for ep in ExtensionPoint(IPyomoScriptSaveInstance):
        ep.apply(options=data.options, instance=instance)

    if data.options.runtime.profile_memory >= 1 and pympler_available:
        mem_used = pympler.muppy.get_size(pympler.muppy.get_objects())
        if mem_used > data.local.max_memory:
            data.local.max_memory = mem_used
        print("   Total memory = %d bytes following Pyomo instance creation" %
              mem_used)

    return Bunch(model=model,
                 instance=instance,
                 smap_id=smap_id,
                 filename=fname,
                 local=data.local)
Ejemplo n.º 4
0
    def construct_scenario_instance(self,
                                    scenario_name,
                                    scenario_tree,
                                    profile_memory=False,
                                    output_instance_construction_time=False,
                                    compile_instance=False,
                                    verbose=False):
        assert not self._closed
        if not scenario_tree.contains_scenario(scenario_name):
            raise ValueError("ScenarioTree does not contain scenario "
                             "with name %s." % (scenario_name))

        scenario = scenario_tree.get_scenario(scenario_name)
        node_name_list = [n._name for n in scenario._node_list]

        if verbose:
            print("Creating instance for scenario=%s" % (scenario_name))

        scenario_instance = None

        try:

            if self._model_callback is not None:

                assert self._model_object is None
                try:
                    _scenario_tree_arg = None
                    # new callback signature
                    if (self._scenario_tree_filename is not None) and \
                       self._scenario_tree_filename.endswith('.dat'):
                        # we started with a .dat file, so
                        # send the PySP scenario tree
                        _scenario_tree_arg = scenario_tree
                    elif self._scenario_tree_model is not None:
                        # We started from a Pyomo
                        # scenario tree model instance, or a
                        # networkx tree.
                        _scenario_tree_arg = self._scenario_tree_model
                    else:
                        # send the PySP scenario tree
                        _scenario_tree_arg = scenario_tree
                    scenario_instance = self._model_callback(
                        _scenario_tree_arg, scenario_name, node_name_list)
                except TypeError:
                    # old callback signature
                    # TODO:
                    #logger.warning(
                    #    "DEPRECATED: The 'pysp_instance_creation_callback' function "
                    #    "signature has changed. An additional argument should be "
                    #    "added to the beginning of the arguments list that will be "
                    #    "set to the user provided scenario tree object when called "
                    #    "by PySP (e.g., a Pyomo scenario tree model instance, "
                    #    "a networkx tree, or a PySP ScenarioTree object.")
                    scenario_instance = self._model_callback(
                        scenario_name, node_name_list)

            elif self._model_object is not None:

                if (not isinstance(self._model_object, AbstractModel)) or \
                   (self._model_object.is_constructed()):
                    scenario_instance = self._model_object.clone()
                elif scenario_tree._scenario_based_data:
                    assert self.data_directory() is not None
                    scenario_data_filename = \
                        os.path.join(self.data_directory(),
                                     str(scenario_name))
                    # JPW: The following is a hack to support
                    #      initialization of block instances, which
                    #      don't work with .dat files at the
                    #      moment. Actually, it's not that bad of a
                    #      hack - it just needs to be extended a bit,
                    #      and expanded into the node-based data read
                    #      logic (where yaml is completely ignored at
                    #      the moment.
                    if os.path.exists(scenario_data_filename + '.dat'):
                        scenario_data_filename = \
                            scenario_data_filename + ".dat"
                        data = None
                    elif os.path.exists(scenario_data_filename + '.yaml'):
                        if not yaml_available:
                            raise ValueError(
                                "Found yaml data file for scenario '%s' "
                                "but he PyYAML module is not available" %
                                (scenario_name))
                        scenario_data_filename = \
                            scenario_data_filename+".yaml"
                        with open(scenario_data_filename) as f:
                            data = yaml.load(f, **yaml_load_args)
                    else:
                        raise RuntimeError(
                            "Cannot find a data file for scenario '%s' "
                            "in directory: %s\nRecognized formats: .dat, "
                            ".yaml" % (scenario_name, self.data_directory()))
                    if verbose:
                        print("Data for scenario=%s loads from file=%s" %
                              (scenario_name, scenario_data_filename))
                    if data is None:
                        scenario_instance = \
                            self._model_object.create_instance(
                                filename=scenario_data_filename,
                                profile_memory=profile_memory,
                                report_timing=output_instance_construction_time)
                    else:
                        scenario_instance = \
                            self._model_object.create_instance(
                                data,
                                profile_memory=profile_memory,
                                report_timing=output_instance_construction_time)
                else:
                    assert self.data_directory() is not None
                    data_files = []
                    for node_name in node_name_list:
                        node_data_filename = \
                            os.path.join(self.data_directory(),
                                         str(node_name)+".dat")
                        if not os.path.exists(node_data_filename):
                            raise RuntimeError(
                                "Cannot find a data file for scenario tree "
                                "node '%s' in directory: %s\nRecognized "
                                "formats: .dat" %
                                (node_name, self.data_directory()))
                        data_files.append(node_data_filename)

                    scenario_data = DataPortal(model=self._model_object)
                    for data_file in data_files:
                        if verbose:
                            print("Node data for scenario=%s partially "
                                  "loading from file=%s" %
                                  (scenario_name, data_file))
                        scenario_data.load(filename=data_file)

                    scenario_instance = self._model_object.create_instance(
                        scenario_data,
                        profile_memory=profile_memory,
                        report_timing=output_instance_construction_time)
            else:
                raise RuntimeError("Unable to construct scenario instance. "
                                   "Neither a reference model or callback "
                                   "is defined.")

            # name each instance with the scenario name
            scenario_instance._name = scenario_name

            # apply each of the post-instance creation plugins. this
            # really shouldn't be associated (in terms of naming) with the
            # pyomo script - this should be rectified with a workflow
            # re-work. it is unclear how this interacts, or doesn't, with
            # the preprocessors.
            ep = ExtensionPoint(IPyomoScriptModifyInstance)
            for ep in ExtensionPoint(IPyomoScriptModifyInstance):
                logger.warning(
                    "DEPRECATED: IPyomoScriptModifyInstance extension "
                    "point callbacks will be ignored by PySP in the future")
                ep.apply(options=None,
                         model=reference_model,
                         instance=scenario_instance)

            if compile_instance:
                from pyomo.repn.beta.matrix import \
                    compile_block_linear_constraints
                compile_block_linear_constraints(
                    scenario_instance,
                    "_PySP_compiled_linear_constraints",
                    verbose=verbose)

        except:
            logger.error("Failed to create model instance for scenario=%s" %
                         (scenario_name))
            raise

        return scenario_instance
Ejemplo n.º 5
0
    def construct_scenario_instance(self,
                                    scenario_name,
                                    scenario_tree,
                                    profile_memory=False,
                                    output_instance_construction_time=False,
                                    compile_instance=False,
                                    verbose=False):
        assert not self._closed
        if not scenario_tree.contains_scenario(scenario_name):
            raise ValueError("ScenarioTree does not contain scenario "
                             "with name %s." % (scenario_name))

        scenario = scenario_tree.get_scenario(scenario_name)
        node_name_list = [n._name for n in scenario._node_list]

        if verbose:
            print("Creating instance for scenario=%s" % (scenario_name))

        scenario_instance = None

        try:

            if self._model_callback is not None:

                assert self._model_object is None
                try:
                    _scenario_tree_arg = None
                    # new callback signature
                    if (self._scenario_tree_filename is not None) and \
                       self._scenario_tree_filename.endswith('.dat'):
                        # we started with a .dat file, so
                        # send the PySP scenario tree
                        _scenario_tree_arg = scenario_tree
                    elif self._scenario_tree_model is not None:
                        # We started from a Pyomo
                        # scenario tree model instance, or a
                        # networkx tree.
                        _scenario_tree_arg = self._scenario_tree_model
                    else:
                        # send the PySP scenario tree
                        _scenario_tree_arg = scenario_tree
                    scenario_instance = self._model_callback(_scenario_tree_arg,
                                                             scenario_name,
                                                             node_name_list)
                except TypeError:
                    # old callback signature
                    # TODO:
                    #logger.warning(
                    #    "DEPRECATED: The 'pysp_instance_creation_callback' function "
                    #    "signature has changed. An additional argument should be "
                    #    "added to the beginning of the arguments list that will be "
                    #    "set to the user provided scenario tree object when called "
                    #    "by PySP (e.g., a Pyomo scenario tree model instance, "
                    #    "a networkx tree, or a PySP ScenarioTree object.")
                    scenario_instance = self._model_callback(scenario_name,
                                                             node_name_list)

            elif self._model_object is not None:

                if (not isinstance(self._model_object, AbstractModel)) or \
                   (self._model_object.is_constructed()):
                    scenario_instance = self._model_object.clone()
                elif scenario_tree._scenario_based_data:
                    assert self.data_directory() is not None
                    scenario_data_filename = \
                        os.path.join(self.data_directory(),
                                     str(scenario_name))
                    # JPW: The following is a hack to support
                    #      initialization of block instances, which
                    #      don't work with .dat files at the
                    #      moment. Actually, it's not that bad of a
                    #      hack - it just needs to be extended a bit,
                    #      and expanded into the node-based data read
                    #      logic (where yaml is completely ignored at
                    #      the moment.
                    if os.path.exists(scenario_data_filename+'.dat'):
                        scenario_data_filename = \
                            scenario_data_filename + ".dat"
                        data = None
                    elif os.path.exists(scenario_data_filename+'.yaml'):
                        if not has_yaml:
                            raise ValueError(
                                "Found yaml data file for scenario '%s' "
                                "but he PyYAML module is not available"
                                % (scenario_name))
                        scenario_data_filename = \
                            scenario_data_filename+".yaml"
                        with open(scenario_data_filename) as f:
                            data = yaml.load(f)
                    else:
                        raise RuntimeError(
                            "Cannot find a data file for scenario '%s' "
                            "in directory: %s\nRecognized formats: .dat, "
                            ".yaml" % (scenario_name, self.data_directory()))
                    if verbose:
                        print("Data for scenario=%s loads from file=%s"
                              % (scenario_name, scenario_data_filename))
                    if data is None:
                        scenario_instance = \
                            self._model_object.create_instance(
                                filename=scenario_data_filename,
                                profile_memory=profile_memory,
                                report_timing=output_instance_construction_time)
                    else:
                        scenario_instance = \
                            self._model_object.create_instance(
                                data,
                                profile_memory=profile_memory,
                                report_timing=output_instance_construction_time)
                else:
                    assert self.data_directory() is not None
                    data_files = []
                    for node_name in node_name_list:
                        node_data_filename = \
                            os.path.join(self.data_directory(),
                                         str(node_name)+".dat")
                        if not os.path.exists(node_data_filename):
                            raise RuntimeError(
                                "Cannot find a data file for scenario tree "
                                "node '%s' in directory: %s\nRecognized "
                                "formats: .dat" % (node_name,
                                                   self.data_directory()))
                        data_files.append(node_data_filename)

                    scenario_data = DataPortal(model=self._model_object)
                    for data_file in data_files:
                        if verbose:
                            print("Node data for scenario=%s partially "
                                  "loading from file=%s"
                                  % (scenario_name, data_file))
                        scenario_data.load(filename=data_file)

                    scenario_instance = self._model_object.create_instance(
                        scenario_data,
                        profile_memory=profile_memory,
                        report_timing=output_instance_construction_time)
            else:
                raise RuntimeError("Unable to construct scenario instance. "
                                   "Neither a reference model or callback "
                                   "is defined.")

            # name each instance with the scenario name
            scenario_instance._name = scenario_name

            # apply each of the post-instance creation plugins. this
            # really shouldn't be associated (in terms of naming) with the
            # pyomo script - this should be rectified with a workflow
            # re-work. it is unclear how this interacts, or doesn't, with
            # the preprocessors.
            ep = ExtensionPoint(IPyomoScriptModifyInstance)
            for ep in ExtensionPoint(IPyomoScriptModifyInstance):
                logger.warning(
                    "DEPRECATED: IPyomoScriptModifyInstance extension "
                    "point callbacks will be ignored by PySP in the future")
                ep.apply(options=None,
                         model=reference_model,
                         instance=scenario_instance)

            if compile_instance:
                from pyomo.repn.beta.matrix import \
                    compile_block_linear_constraints
                compile_block_linear_constraints(
                    scenario_instance,
                    "_PySP_compiled_linear_constraints",
                    verbose=verbose)

        except:
            logger.error("Failed to create model instance for scenario=%s"
                         % (scenario_name))
            raise

        return scenario_instance