Esempio n. 1
0
 def test_locate_annotations(self):
     m = aml.ConcreteModel()
     m.a = StageCostAnnotation()
     m.b = aml.Block()
     m.b.a = StageCostAnnotation()
     self.assertEqual(locate_annotations(m, StageCostAnnotation),
                      [('a', m.a), ('a', m.b.a)])
     with self.assertRaises(ValueError):
         locate_annotations(m, StageCostAnnotation, max_allowed=1)
     m.b.deactivate()
     self.assertEqual(locate_annotations(m, StageCostAnnotation),
                      [('a', m.a)])
     self.assertEqual(locate_annotations(m, VariableStageAnnotation),
                      [])
Esempio n. 2
0
    def _generate_base_scenario_tree(self, model, variable_stage_assignments):

        stage_cost_annotation = locate_annotations(
            model,
            PySP_StageCostAnnotation,
            max_allowed=1)
        if len(stage_cost_annotation) == 0:
            raise ValueError("Reference model is missing stage cost "
                             "annotation: %s" % (PySP_StageCostAnnotation.__name__))
        else:
            assert len(stage_cost_annotation) == 1
            stage_cost_annotation = stage_cost_annotation[0][1]
        stage_cost_assignments = ComponentMap(
            stage_cost_annotation.expand_entries())

        stage1_cost = None
        stage2_cost = None
        for cdata, stagenum in stage_cost_assignments.items():
            if stagenum == 1:
                stage1_cost = cdata
            elif stagenum == 2:
                stage2_cost = cdata
        if stage1_cost is None:
            raise ValueError("Missing stage cost annotation for time stage: 1")
        if stage2_cost is None:
            raise ValueError("Missing stage cost annotation for time stage: 2")
        assert stage1_cost != stage2_cost

        #
        # Create a dummy 1-scenario scenario tree
        #

        stm = CreateAbstractScenarioTreeModel()
        stm.Stages.add('Stage1')
        stm.Stages.add('Stage2')
        stm.Nodes.add('RootNode')
        stm.Nodes.add('LeafNode')
        stm.Scenarios.add('ReferenceScenario')
        stm = stm.create_instance()
        stm.NodeStage['RootNode'] = 'Stage1'
        stm.ConditionalProbability['RootNode'] = 1.0
        stm.NodeStage['LeafNode'] = 'Stage2'
        stm.Children['RootNode'].add('LeafNode')
        stm.Children['LeafNode'].clear()
        stm.ConditionalProbability['LeafNode'] = 1.0
        stm.ScenarioLeafNode['ReferenceScenario'] = 'LeafNode'

        stm.StageCost['Stage1'] = stage1_cost.name
        stm.StageCost['Stage2'] = stage2_cost.name
        for var, (stagenum, derived) in variable_stage_assignments.items():
            stagelabel = 'Stage'+str(stagenum)
            if not derived:
                stm.StageVariables[stagelabel].add(var.name)
            else:
                stm.StageDerivedVariables[second_stage].add(var.name)

        scenario_tree = ScenarioTree(scenariotreeinstance=stm)
        scenario_tree.linkInInstances(
            {'ReferenceScenario': self.reference_model})
        return scenario_tree
Esempio n. 3
0
def _extract_stochastic_data(model):
    stochastic_data_annotation = locate_annotations(
        model,
        StochasticDataAnnotation,
        max_allowed=1)
    if len(stochastic_data_annotation) == 0:
        raise ValueError(
            "Reference model is missing stochastic data "
            "annotation: %s" % (StochasticDataAnnotation.__name__))
    else:
        assert len(stochastic_data_annotation) == 1
        stochastic_data_annotation = stochastic_data_annotation[0][1]
    stochastic_data = ComponentMap(
        stochastic_data_annotation.expand_entries())
    if len(stochastic_data) == 0:
        raise ValueError("At least one stochastic data "
                         "entry is required.")
    for paramdata in stochastic_data:
        assert isinstance(paramdata, _ParamData)
        if paramdata.is_constant():
            raise ValueError(
                "Stochastic data entry with name '%s' is not mutable. "
                "All stochastic data parameters must be initialized "
                "with the mutable keyword set to True."
                % (paramdata.name))
    return stochastic_data
Esempio n. 4
0
    def _map_variable_stages(self, model):

        variable_stage_annotation = locate_annotations(
            model,
            PySP_VariableStageAnnotation,
        max_allowed=1)
        if len(variable_stage_annotation) == 0:
            raise ValueError(
                "Reference model is missing variable stage "
                "annotation: %s" % (PySP_VariableStageAnnotation.__name__))
        else:
            assert len(variable_stage_annotation) == 1
            variable_stage_annotation = variable_stage_annotation[0][1]
        variable_stage_assignments = ComponentMap(
            variable_stage_annotation.expand_entries(
                expand_containers=False))
        if len(variable_stage_assignments) == 0:
            raise ValueError("At least one variable stage assignment is required.")

        min_stagenumber = min(variable_stage_assignments.values(),
                              key=lambda x: x[0])[0]
        max_stagenumber = max(variable_stage_assignments.values(),
                              key=lambda x: x[0])[0]
        if max_stagenumber > 2:
            for vardata, (stagenum, derived) in variable_stage_assignments.items():
                if stagenum > 2:
                    raise ValueError(
                        "Implicit stochastic programs must be two-stage, "
                        "but variable with name '%s' has been annotated with "
                        "stage number: %s" % (vardata.name, stagenum))

        stage_to_variables_map = {}
        stage_to_variables_map[1] = []
        stage_to_variables_map[2] = []
        for vardata in model.component_data_objects(
                Var,
                active=True,
                descend_into=True,
                sort=SortComponents.alphabetizeComponentAndIndex):
            stagenumber, derived = variable_stage_assignments.get(vardata, (2, False))
            if (stagenumber != 1) and (stagenumber != 2):
                raise ValueError("Invalid stage annotation for variable with "
                                 "name '%s'. Stage assignment must be 1 or 2. "
                                 "Current value: %s"
                                 % (vardata.name, stagenumber))
            if (stagenumber == 1):
                stage_to_variables_map[1].append((vardata, derived))
            else:
                assert stagenumber == 2
                stage_to_variables_map[2].append((vardata, derived))

        variable_to_stage_map = ComponentMap()
        for stagenum, stagevars in stage_to_variables_map.items():
            for vardata, derived in stagevars:
                variable_to_stage_map[vardata] = (stagenum, derived)

        return (stage_to_variables_map,
                variable_to_stage_map,
                variable_stage_assignments)
Esempio n. 5
0
    def __init__(self, reference_model):

        self.reference_model = None
        self.objective = None
        self.time_stages = None

        self.stage_to_variables_map = {}
        self.variable_to_stage_map = {}

        # the set of stochastic data objects
        # (possibly mapped to some distribution)
        self.stochastic_data = None

        # maps between variables and objectives
        self.variable_to_objectives_map = ComponentMap()
        self.objective_to_variables_map = ComponentMap()

        # maps between variables and constraints
        self.variable_to_constraints_map = ComponentMap()
        self.constraint_to_variables_map = ComponentMap()

        # maps between stochastic data and objectives
        self.stochastic_data_to_objectives_map = ComponentMap()
        self.objective_to_stochastic_data_map = ComponentMap()

        # maps between stochastic data and constraints
        self.stochastic_data_to_constraints_map = ComponentMap()
        self.constraint_to_stochastic_data_map = ComponentMap()

        # maps between stochastic data and variable lower and upper bounds
        self.stochastic_data_to_variables_lb_map = ComponentMap()
        self.variable_to_stochastic_data_lb_map = ComponentMap()

        self.stochastic_data_to_variables_ub_map = ComponentMap()
        self.variable_to_stochastic_data_ub_map = ComponentMap()

        self.variable_symbols = ComponentMap()

        if not isinstance(reference_model, Block):
            raise TypeError("reference model input must be a Pyomo model")
        self.reference_model = reference_model

        #
        # Extract stochastic parameters from the
        # StochasticDataAnnotation object
        #
        self.stochastic_data = \
            _extract_stochastic_data(self.reference_model)

        #
        # Get the variable stages from the
        # VariableStageAnnotation object
        #
        (self.stage_to_variables_map,
         self.variable_to_stage_map,
         self._variable_stage_assignments) = \
            _map_variable_stages(self.reference_model)
        self.time_stages = tuple(sorted(self.stage_to_variables_map))
        assert self.time_stages[0] == 1
        self.variable_symbols = generate_cuid_names(self.reference_model,
                                                    ctype=Var)
        # remove the parent blocks from this map
        keys_to_delete = []
        for var in self.variable_symbols:
            if var.parent_component().type() is not Var:
                keys_to_delete.append(var)
        for key in keys_to_delete:
            del self.variable_symbols[key]

        #
        # Get the stage cost components from the StageCostAnnotation
        # and generate a dummy single-scenario scenario tree
        #
        stage_cost_annotation = locate_annotations(
            self.reference_model,
            StageCostAnnotation,
            max_allowed=1)
        if len(stage_cost_annotation) == 0:
            raise ValueError(
                "Reference model is missing stage cost "
                "annotation: %s" % (StageCostAnnotation.__name__))
        else:
            assert len(stage_cost_annotation) == 1
            stage_cost_annotation = stage_cost_annotation[0][1]
        stage_cost_assignments = ComponentMap(
            stage_cost_annotation.expand_entries())

        stage1_cost = None
        stage2_cost = None
        for cdata, stagenum in stage_cost_assignments.items():
            if stagenum == 1:
                stage1_cost = cdata
            elif stagenum == 2:
                stage2_cost = cdata
        if stage1_cost is None:
            raise ValueError("Missing stage cost annotation "
                             "for time stage: 1")
        if stage2_cost is None:
            raise ValueError("Missing stage cost annotation "
                             "for time stage: 2")
        assert stage1_cost is not stage2_cost
        self._stage1_cost = stage1_cost
        self._stage2_cost = stage2_cost

        #
        # Extract the locations of variables and stochastic data
        # within the model
        #
        sto_obj = StochasticObjectiveAnnotation()
        for objcntr, obj in enumerate(
                  self.reference_model.component_data_objects(
                Objective,
                active=True,
                descend_into=True), 1):

            if objcntr > 1:
                raise ValueError(
                    "Reference model can not contain more than one "
                    "active objective")

            self.objective = obj
            self.objective_sense = obj.sense

            obj_params = tuple(
                self._collect_mutable_parameters(obj.expr).values())
            self.objective_to_stochastic_data_map[obj] = []
            for paramdata in obj_params:
                if paramdata in self.stochastic_data:
                    self.stochastic_data_to_objectives_map.\
                        setdefault(paramdata, []).append(obj)
                    self.objective_to_stochastic_data_map[obj].\
                        append(paramdata)
            if len(self.objective_to_stochastic_data_map[obj]) == 0:
                del self.objective_to_stochastic_data_map[obj]
            else:
                # TODO: Can we make this declaration sparse
                #       by identifying which variables have
                #       stochastic coefficients? How to handle
                #       non-linear expressions?
                sto_obj.declare(obj)

            obj_variables = tuple(
                self._collect_variables(obj.expr).values())
            self.objective_to_variables_map[obj] = []
            for var in obj_variables:
                self.variable_to_objectives_map.\
                    setdefault(var, []).append(obj)
                self.objective_to_variables_map[obj].append(var)
            if len(self.objective_to_variables_map[obj]) == 0:
                del self.objective_to_variables_map[obj]

        sto_conbounds = StochasticConstraintBoundsAnnotation()
        sto_conbody = StochasticConstraintBodyAnnotation()
        for con in self.reference_model.component_data_objects(
                Constraint,
                active=True,
                descend_into=True):

            lower_params = tuple(
                self._collect_mutable_parameters(con.lower).values())
            body_params = tuple(
                self._collect_mutable_parameters(con.body).values())
            upper_params = tuple(
                self._collect_mutable_parameters(con.upper).values())

            # TODO: Can we make this declaration sparse
            #       by idenfifying which variables have
            #       stochastic coefficients? How to handle
            #       non-linear expressions? Currently, this
            #       code also fails to detect that mutable
            #       "constant" expressions might fall out
            #       of the body and into the bounds.
            if len(body_params):
                sto_conbody.declare(con)
            if len(body_params) or \
               len(lower_params) or \
               len(upper_params):
                sto_conbounds.declare(con,
                                      lb=bool(len(lower_params) or len(body_params)),
                                      ub=bool(len(upper_params) or len(body_params)))

            all_stochastic_params = {}
            for param in itertools.chain(lower_params,
                                         body_params,
                                         upper_params):
                if param in self.stochastic_data:
                    all_stochastic_params[id(param)] = param

            if len(all_stochastic_params) > 0:
                self.constraint_to_stochastic_data_map[con] = []
                # no params will appear twice in this iteration
                for param in all_stochastic_params.values():
                    self.stochastic_data_to_constraints_map.\
                        setdefault(param, []).append(con)
                    self.constraint_to_stochastic_data_map[con].\
                        append(param)

            body_variables = tuple(
                self._collect_variables(con.body).values())
            self.constraint_to_variables_map[con] = []
            for var in body_variables:
                self.variable_to_constraints_map.\
                    setdefault(var, []).append(con)
                self.constraint_to_variables_map[con].append(var)

        # For now, it is okay to have SOSConstraints in the
        # representation of a problem, but the SOS
        # constraints can't have custom weights that
        # represent stochastic data
        for soscon in self.reference_model.component_data_objects(
                SOSConstraint,
                active=True,
                descend_into=True):
            for var, weight in soscon.get_items():
                weight_params = tuple(
                    self._collect_mutable_parameters(weight).values())
                if param in self.stochastic_data:
                    raise ValueError(
                        "SOSConstraints with stochastic data are currently"
                        " not supported in embedded stochastic programs. "
                        "The SOSConstraint component '%s' has a weight "
                        "term for variable '%s' that references stochastic"
                        " parameter '%s'"
                        % (soscon.name,
                           var.name,
                           param.name))
                self.variable_to_constraints_map.\
                    setdefault(var, []).append(soscon)
                self.constraint_to_variables_map.\
                    setdefault(soscon, []).append(var)

        sto_varbounds = StochasticVariableBoundsAnnotation()
        for var in self.reference_model.component_data_objects(
                Var,
                descend_into=True):

            lower_params = tuple(
                self._collect_mutable_parameters(var.lb).values())
            upper_params = tuple(
                self._collect_mutable_parameters(var.ub).values())

            if (len(lower_params) > 0) or \
               (len(upper_params) > 0):
                sto_varbounds.declare(var,
                                      lb=bool(len(lower_params) > 0),
                                      ub=bool(len(upper_params) > 0))

            self.variable_to_stochastic_data_lb_map[var] = []
            for param in lower_params:
                if param in self.stochastic_data:
                    self.stochastic_data_to_variables_lb_map.\
                        setdefault(param, []).append(var)
                    self.variable_to_stochastic_data_lb_map[var].\
                        append(param)
            if len(self.variable_to_stochastic_data_lb_map[var]) == 0:
                del self.variable_to_stochastic_data_lb_map[var]

            self.variable_to_stochastic_data_ub_map[var] = []
            for param in upper_params:
                if param in self.stochastic_data:
                    self.stochastic_data_to_variables_ub_map.\
                        setdefault(param, []).append(var)
                    self.variable_to_stochastic_data_ub_map[var].\
                        append(param)
            if len(self.variable_to_stochastic_data_ub_map[var]) == 0:
                del self.variable_to_stochastic_data_ub_map[var]

        #
        # Generate the explicit annotations
        #

        # first make sure these annotations do not already exist
        if len(locate_annotations(self.reference_model,
                                  StochasticConstraintBoundsAnnotation)) > 0:
            raise ValueError("Reference model can not contain "
                             "a StochasticConstraintBoundsAnnotation declaration.")
        if len(locate_annotations(self.reference_model,
                                  StochasticConstraintBodyAnnotation)) > 0:
            raise ValueError("Reference model can not contain "
                             "a StochasticConstraintBodyAnnotation declaration.")
        if len(locate_annotations(self.reference_model,
                                  StochasticObjectiveAnnotation)) > 0:
            raise ValueError("Reference model can not contain "
                             "a StochasticObjectiveAnnotation declaration.")

        # now add any necessary annotations
        if sto_obj.has_declarations:
            assert not hasattr(self.reference_model,
                               ".pyspembeddedsp_stochastic_objective_annotation")
            setattr(self.reference_model,
                    ".pyspembeddedsp_stochastic_objective_annotation",
                    sto_obj)
        if sto_conbody.has_declarations:
            assert not hasattr(self.reference_model,
                               ".pyspembeddedsp_stochastic_constraint_body_annotation")
            setattr(self.reference_model,
                    ".pyspembeddedsp_stochastic_constraint_body_annotation",
                    sto_conbody)
        if sto_conbounds.has_declarations:
            assert not hasattr(self.reference_model,
                               ".pyspembeddedsp_stochastic_constraint_bounds_annotation")
            setattr(self.reference_model,
                    ".pyspembeddedsp_stochastic_constraint_bounds_annotation",
                    sto_conbounds)
        if sto_varbounds.has_declarations:
            assert not hasattr(self.reference_model,
                               ".pyspembeddedsp_stochastic_variable_bounds_annotation")
            setattr(self.reference_model,
                    ".pyspembeddedsp_stochastic_variable_bounds_annotation",
                    sto_varbounds)
Esempio n. 6
0
File: ddsip.py Progetto: Pyomo/pyomo
def _convert_external_setup_without_cleanup(
        worker,
        scenario,
        output_directory,
        firststage_var_suffix,
        enforce_derived_nonanticipativity,
        io_options):
    import pyomo.environ
    assert os.path.exists(output_directory)

    io_options = dict(io_options)
    scenario_tree = worker.scenario_tree
    reference_model = scenario._instance
    rootnode = scenario_tree.findRootNode()
    firststage = scenario_tree.stages[0]
    secondstage = scenario_tree.stages[1]
    constraint_name_buffer = {}
    objective_name_buffer = {}
    variable_name_buffer = {}

    all_constraints = list(
        con for con in reference_model.component_data_objects(
            Constraint,
            active=True,
            descend_into=True))

    #
    # Check for model annotations
    #
    stochastic_rhs = locate_annotations(
        reference_model,
        StochasticConstraintBoundsAnnotation,
        max_allowed=1)
    if len(stochastic_rhs) == 0:
        stochastic_rhs = None
        stochastic_rhs_entries = {}
        empty_rhs_annotation = False
    else:
        assert len(stochastic_rhs) == 1
        stochastic_rhs = stochastic_rhs[0][1]
        if stochastic_rhs.has_declarations:
            empty_rhs_annotation = False
            stochastic_rhs_entries = stochastic_rhs.expand_entries()
            stochastic_rhs_entries.sort(
                key=lambda x: x[0].getname(True, constraint_name_buffer))
            if len(stochastic_rhs_entries) == 0:
                raise RuntimeError(
                    "The %s annotation was declared "
                    "with external entries but no active Constraint "
                    "objects were recovered from those entries."
                    % (StochasticConstraintBoundsAnnotation.__name__))
        else:
            empty_rhs_annotation = True
            stochastic_rhs_entries = tuple((con, stochastic_rhs.default)
                                           for con in all_constraints)


    stochastic_matrix = locate_annotations(
        reference_model,
        StochasticConstraintBodyAnnotation,
        max_allowed=1)
    if len(stochastic_matrix) == 0:
        stochastic_matrix = None
        stochastic_matrix_entries = {}
        empty_matrix_annotation = False
    else:
        assert len(stochastic_matrix) == 1
        stochastic_matrix = stochastic_matrix[0][1]
        if stochastic_matrix.has_declarations:
            empty_matrix_annotation = False
            stochastic_matrix_entries = stochastic_matrix.expand_entries()
            stochastic_matrix_entries.sort(
                key=lambda x: x[0].getname(True, constraint_name_buffer))
            if len(stochastic_matrix_entries) == 0:
                raise RuntimeError(
                    "The %s annotation was declared "
                    "with external entries but no active Constraint "
                    "objects were recovered from those entries."
                    % (StochasticConstraintBoundsAnnotation.__name__))
        else:
            empty_matrix_annotation = True
            stochastic_matrix_entries = tuple((con,stochastic_matrix.default)
                                              for con in all_constraints)

    stochastic_constraint_ids = set()
    stochastic_constraint_ids.update(id(con) for con,_
                                     in stochastic_rhs_entries)
    stochastic_constraint_ids.update(id(con) for con,_
                                     in stochastic_matrix_entries)

    stochastic_objective = locate_annotations(
        reference_model,
        StochasticObjectiveAnnotation,
        max_allowed=1)
    if len(stochastic_objective) == 0:
        stochastic_objective = None
    else:
        assert len(stochastic_objective) == 1
        stochastic_objective = stochastic_objective[0][1]

    stochastic_varbounds = locate_annotations(
        reference_model,
        StochasticVariableBoundsAnnotation)
    if len(stochastic_varbounds) > 0:
        raise ValueError(
            "The DDSIP writer does not currently support "
            "stochastic variable bounds. Invalid annotation type: %s"
            % (StochasticVariableBoundsAnnotation.__name__))

    if (stochastic_rhs is None) and \
       (stochastic_matrix is None) and \
       (stochastic_objective is None):
        raise RuntimeError(
            "No stochastic annotations found. DDSIP "
            "conversion requires at least one of the following "
            "annotation types:\n - %s\n - %s\n - %s"
            % (StochasticConstraintBoundsAnnotation.__name__,
               StochasticConstraintBodyAnnotation.__name__,
               StochasticObjectiveAnnotation.__name__))

    assert not hasattr(reference_model, "_repn")
    repn_cache = build_repns(reference_model)
    assert hasattr(reference_model, "_repn")
    assert not reference_model._gen_obj_repn
    assert not reference_model._gen_con_repn
    # compute values
    for block_repns in repn_cache.values():
        for repn in block_repns.values():
            repn.constant = value(repn.constant)
            repn.linear_coefs = [value(c) for c in repn.linear_coefs]
            repn.quadratic_coefs = [value(c) for c in repn.quadratic_coefs]

    #
    # Write the LP file once to obtain the symbol map
    #
    output_filename = os.path.join(output_directory,
                                   scenario.name+".lp.setup")
    with WriterFactory("lp") as writer:
        assert 'column_order' not in io_options
        assert 'row_order' not in io_options
        output_fname, symbol_map = writer(reference_model,
                                          output_filename,
                                          lambda x: True,
                                          io_options)
        assert output_fname == output_filename
    _safe_remove_file(output_filename)

    StageToVariableMap = map_variable_stages(
        scenario,
        scenario_tree,
        symbol_map,
        enforce_derived_nonanticipativity=enforce_derived_nonanticipativity)
    firststage_variable_ids = \
        set(id(var) for symbol, var, scenario_tree_id
            in StageToVariableMap[firststage.name])
    secondstage_variable_ids = \
        set(id(var) for symbol, var, scenario_tree_id
            in StageToVariableMap[secondstage.name])

    StageToConstraintMap = \
        map_constraint_stages(
            scenario,
            scenario_tree,
            symbol_map,
            stochastic_constraint_ids,
            firststage_variable_ids,
            secondstage_variable_ids)
    secondstage_constraint_ids = \
        set(id(con) for symbols, con
            in StageToConstraintMap[secondstage.name])

    assert len(scenario_tree.stages) == 2
    firststage = scenario_tree.stages[0]
    secondstage = scenario_tree.stages[1]

    #
    # Make sure the objective references all first stage variables.
    # We do this by directly modifying the _repn of the
    # objective which the LP/MPS writer will reference next time we call
    # it. In addition, make sure that the first second-stage variable
    # in our column ordering also appears in the objective so that
    # ONE_VAR_CONSTANT does not get identified as the first
    # second-stage variable.
    # ** Just do NOT preprocess again until we call the writer **
    #
    objective_object = scenario._instance_objective
    assert objective_object is not None
    objective_block = objective_object.parent_block()
    objective_repn = repn_cache[id(objective_block)][objective_object]

    #
    # Create column (variable) ordering maps for LP/MPS files
    #
    column_order = ComponentMap()
    firststage_variable_count = 0
    secondstage_variable_count = 0
    # first-stage variables
    for column_index, (symbol, var, scenario_tree_id) \
        in enumerate(StageToVariableMap[firststage.name]):
        column_order[var] = column_index
        firststage_variable_count += 1
    # second-stage variables
    for column_index, (symbol, var, scenario_tree_id) \
        in enumerate(StageToVariableMap[secondstage.name],
                     len(column_order)):
        column_order[var] = column_index
        secondstage_variable_count += 1
    # account for the ONE_VAR_CONSTANT second-stage variable
    # added by the LP writer
    secondstage_variable_count += 1

    #
    # Create row (constraint) ordering maps for LP/MPS files
    #
    firststage_constraint_count = 0
    secondstage_constraint_count = 0
    row_order = ComponentMap()
    # first-stage constraints
    for row_index, (symbols, con) \
        in enumerate(StageToConstraintMap[firststage.name]):
        row_order[con] = row_index
        firststage_constraint_count += len(symbols)
    # second-stage constraints
    for row_index, (symbols, con) \
        in enumerate(StageToConstraintMap[secondstage.name],
                     len(row_order)):
        row_order[con] = row_index
        secondstage_constraint_count += len(symbols)
    # account for the ONE_VAR_CONSTANT = 1 second-stage constraint
    # added by the LP writer
    secondstage_constraint_count += 1

    #
    # Create a custom labeler that allows DDSIP to identify
    # first-stage variables
    #
    if io_options.pop('symbolic_solver_labels', False):
        _labeler = TextLabeler()
    else:
        _labeler = NumericLabeler('x')
    labeler = lambda x: _labeler(x) + \
              (""
               if ((not isinstance(x, _VarData)) or \
                   (id(x) not in firststage_variable_ids)) else \
               firststage_var_suffix)

    #
    # Write the ordered LP/MPS file
    #
    output_filename = os.path.join(output_directory,
                                   scenario.name+".lp")
    symbols_filename = os.path.join(output_directory,
                                    scenario.name+".lp.symbols")
    with WriterFactory("lp") as writer:
        assert 'column_order' not in io_options
        assert 'row_order' not in io_options
        assert 'labeler' not in io_options
        assert 'force_objective_constant' not in io_options
        io_options['column_order'] = column_order
        io_options['row_order'] = row_order
        io_options['force_objective_constant'] = True
        io_options['labeler'] = labeler
        output_fname, symbol_map = writer(reference_model,
                                          output_filename,
                                          lambda x: True,
                                          io_options)
        assert output_fname == output_filename
        # write the lp file symbol paired with the scenario
        # tree id for each variable in the root node
        with open(symbols_filename, "w") as f:
            st_symbol_map = reference_model._ScenarioTreeSymbolMap
            lines = []
            for id_ in sorted(rootnode._variable_ids):
                var = st_symbol_map.bySymbol[id_]
                if not var.is_expression_type():
                    lp_label = symbol_map.byObject[id(var)]
                    lines.append("%s %s\n" % (lp_label, id_))
            f.writelines(lines)

    # re-generate these maps as the LP/MPS symbol map
    # is likely different
    StageToVariableMap = map_variable_stages(
        scenario,
        scenario_tree,
        symbol_map,
        enforce_derived_nonanticipativity=enforce_derived_nonanticipativity)

    StageToConstraintMap = map_constraint_stages(
        scenario,
        scenario_tree,
        symbol_map,
        stochastic_constraint_ids,
        firststage_variable_ids,
        secondstage_variable_ids)

    # generate a few data structures that are used
    # when writing the .sc files
    constraint_symbols = ComponentMap(
        (con, symbols) for stage_name in StageToConstraintMap
        for symbols, con in StageToConstraintMap[stage_name])

    #
    # Write the body of the .sc files
    #
    modified_constraint_lb = ComponentMap()
    modified_constraint_ub = ComponentMap()

    #
    # Stochastic RHS
    #
    # **NOTE: In the code that follows we assume the LP
    #         writer always moves constraint body
    #         constants to the rhs and that the lower part
    #         of any range constraints are written before
    #         the upper part.
    #
    stochastic_rhs_count = 0
    with open(os.path.join(output_directory,
                           scenario.name+".rhs.sc.struct"),'w') as f_rhs_struct:
        with open(os.path.join(output_directory,
                               scenario.name+".rhs.sc"),'w') as f_rhs:
            scenario_probability = scenario.probability
            rhs_struct_template = " %s\n"
            rhs_template = "  %.17g\n"
            f_rhs.write("scen\n%.17g\n"
                        % (_no_negative_zero(scenario_probability)))
            if stochastic_rhs is not None:
                for con, include_bound in stochastic_rhs_entries:
                    assert isinstance(con, _ConstraintData)
                    if not empty_rhs_annotation:
                        # verify that this constraint was
                        # flagged by PySP or the user as second-stage
                        if id(con) not in secondstage_constraint_ids:
                            raise RuntimeError(
                                "The constraint %s has been declared "
                                "in the %s annotation but it was not identified as "
                                "a second-stage constraint. To correct this issue, "
                                "remove the constraint from this annotation."
                                % (con.name,
                                   StochasticConstraintBoundsAnnotation.__name__))

                    constraint_repn = \
                        repn_cache[id(con.parent_block())][con]
                    if not constraint_repn.is_linear():
                        raise RuntimeError("Only linear constraints are "
                                           "accepted for conversion to DDSIP format. "
                                           "Constraint %s is not linear."
                                           % (con.name))

                    body_constant = constraint_repn.constant
                    # We are going to rewrite the core problem file
                    # with all stochastic values set to zero. This will
                    # allow an easy test for missing user annotations.
                    constraint_repn.constant = 0
                    if body_constant is None:
                        body_constant = 0.0
                    symbols = constraint_symbols[con]
                    assert len(symbols) > 0
                    for con_label in symbols:
                        if con_label.startswith('c_e_') or \
                           con_label.startswith('c_l_'):
                            assert (include_bound is True) or \
                                   (include_bound[0] is True)
                            stochastic_rhs_count += 1
                            f_rhs_struct.write(rhs_struct_template % (con_label))
                            f_rhs.write(rhs_template %
                                        (_no_negative_zero(
                                            value(con.lower) - \
                                            value(body_constant))))
                            # We are going to rewrite the core problem file
                            # with all stochastic values set to zero. This will
                            # allow an easy test for missing user annotations.
                            modified_constraint_lb[con] = con.lower
                            con._lower = _deterministic_check_constant
                            if con_label.startswith('c_e_'):
                                modified_constraint_ub[con] = con.upper
                                con._upper = _deterministic_check_constant
                        elif con_label.startswith('r_l_') :
                            if (include_bound is True) or \
                               (include_bound[0] is True):
                                stochastic_rhs_count += 1
                                f_rhs_struct.write(rhs_struct_template % (con_label))
                                f_rhs.write(rhs_template %
                                             (_no_negative_zero(
                                                 value(con.lower) - \
                                                 value(body_constant))))
                                # We are going to rewrite the core problem file
                                # with all stochastic values set to zero. This will
                                # allow an easy test for missing user annotations.
                                modified_constraint_lb[con] = con.lower
                                con._lower = _deterministic_check_constant
                        elif con_label.startswith('c_u_'):
                            assert (include_bound is True) or \
                                   (include_bound[1] is True)
                            stochastic_rhs_count += 1
                            f_rhs_struct.write(rhs_struct_template % (con_label))
                            f_rhs.write(rhs_template %
                                        (_no_negative_zero(
                                            value(con.upper) - \
                                            value(body_constant))))
                            # We are going to rewrite the core problem file
                            # with all stochastic values set to zero. This will
                            # allow an easy test for missing user annotations.
                            modified_constraint_ub[con] = con.upper
                            con._upper = _deterministic_check_constant
                        elif con_label.startswith('r_u_'):
                            if (include_bound is True) or \
                               (include_bound[1] is True):
                                stochastic_rhs_count += 1
                                f_rhs_struct.write(rhs_struct_template % (con_label))
                                f_rhs.write(rhs_template %
                                            (_no_negative_zero(
                                                value(con.upper) - \
                                                value(body_constant))))
                                # We are going to rewrite the core problem file
                                # with all stochastic values set to zero. This will
                                # allow an easy test for missing user annotations.
                                modified_constraint_ub[con] = con.upper
                                con._upper = _deterministic_check_constant
                        else:
                            assert False

    #
    # Stochastic Matrix
    #
    stochastic_matrix_count = 0
    with open(os.path.join(output_directory,
                           scenario.name+".matrix.sc.struct"),'w') as f_mat_struct:
        with open(os.path.join(output_directory,
                               scenario.name+".matrix.sc"),'w') as f_mat:
            scenario_probability = scenario.probability
            matrix_struct_template = " %s %s\n"
            matrix_template = "  %.17g\n"
            f_mat.write("scen\n")
            if stochastic_matrix is not None:
                for con, var_list in stochastic_matrix_entries:
                    assert isinstance(con, _ConstraintData)
                    if not empty_matrix_annotation:
                        # verify that this constraint was
                        # flagged by PySP or the user as second-stage
                        if id(con) not in secondstage_constraint_ids:
                            raise RuntimeError(
                                "The constraint %s has been declared "
                                "in the %s annotation but it was not identified as "
                                "a second-stage constraint. To correct this issue, "
                                "remove the constraint from this annotation."
                                % (con.name,
                                   StochasticConstraintBodyAnnotation.__name__))
                    constraint_repn = \
                        repn_cache[id(con.parent_block())][con]
                    if not constraint_repn.is_linear():
                        raise RuntimeError("Only linear constraints are "
                                           "accepted for conversion to DDSIP format. "
                                           "Constraint %s is not linear."
                                           % (con.name))
                    assert len(constraint_repn.linear_vars) > 0
                    if var_list is None:
                        var_list = constraint_repn.linear_vars
                    assert len(var_list) > 0
                    symbols = constraint_symbols[con]
                    # sort the variable list by the column ordering
                    # so that we have deterministic output
                    var_list = list(var_list)
                    var_list.sort(key=lambda _v: column_order[_v])
                    new_coefs = list(constraint_repn.linear_coefs)
                    for var in var_list:
                        assert isinstance(var, _VarData)
                        assert not var.fixed
                        var_coef = None
                        for i, (_var, coef) in enumerate(zip(constraint_repn.linear_vars,
                                                             constraint_repn.linear_coefs)):
                            if _var is var:
                                var_coef = coef
                                # We are going to rewrite with core problem file
                                # with all stochastic values set to zero. This will
                                # allow an easy test for missing user annotations.
                                new_coefs[i] = _deterministic_check_value
                                break
                        if var_coef is None:
                            raise RuntimeError(
                                "The coefficient for variable %s has "
                                "been marked as stochastic in constraint %s using "
                                "the %s annotation, but the variable does not appear"
                                " in the canonical constraint expression."
                                % (var.name,
                                   con.name,
                                   StochasticConstraintBodyAnnotation.__name__))
                        var_label = symbol_map.byObject[id(var)]
                        for con_label in symbols:
                            stochastic_matrix_count += 1
                            f_mat_struct.write(matrix_struct_template
                                              % (con_label, var_label))
                            f_mat.write(matrix_template
                                        % (_no_negative_zero(value(var_coef))))

                    constraint_repn.linear_coefs = tuple(new_coefs)

    #
    # Stochastic Objective
    #
    stochastic_cost_count = 0
    with open(os.path.join(output_directory,
                           scenario.name+".cost.sc.struct"),'w') as f_obj_struct:
        with open(os.path.join(output_directory,
                               scenario.name+".cost.sc"),'w') as f_obj:
            obj_struct_template = " %s\n"
            obj_template = "  %.17g\n"
            f_obj.write("scen\n")
            if stochastic_objective is not None:
                if stochastic_objective.has_declarations:
                    sorted_values = stochastic_objective.expand_entries()
                    assert len(sorted_values) <= 1
                    if len(sorted_values) == 0:
                        raise RuntimeError(
                            "The %s annotation was declared "
                            "with external entries but no active Objective "
                            "objects were recovered from those entries."
                            % (StochasticObjectiveAnnotation.__name__))
                    obj, (objective_variables, include_constant) = \
                        sorted_values[0]
                    assert obj is objective_object
                else:
                    objective_variables, include_constant = \
                        stochastic_objective.default

                if not objective_repn.is_linear():
                    raise RuntimeError("Only linear stochastic objectives are "
                                       "accepted for conversion to DDSIP format. "
                                       "Objective %s is not linear."
                                       % (objective_object.name))
                if objective_variables is None:
                    objective_variables = objective_repn.linear_vars
                stochastic_objective_label = symbol_map.byObject[id(objective_object)]
                # sort the variable list by the column ordering
                # so that we have deterministic output
                objective_variables = list(objective_variables)
                objective_variables.sort(key=lambda _v: column_order[_v])
                assert (len(objective_variables) > 0) or include_constant
                new_coefs = list(objective_repn.linear_coefs)
                for var in objective_variables:
                    assert isinstance(var, _VarData)
                    var_coef = None
                    for i, (_var, coef) in enumerate(zip(objective_repn.linear_vars,
                                                         objective_repn.linear_coefs)):
                        if _var is var:
                            var_coef = coef
                            # We are going to rewrite the core problem file
                            # with all stochastic values set to zero. This will
                            # allow an easy test for missing user annotations.
                            new_coefs[i] = _deterministic_check_value
                            break
                    if var_coef is None:
                        raise RuntimeError(
                            "The coefficient for variable %s has "
                            "been marked as stochastic in objective %s using "
                            "the %s annotation, but the variable does not appear"
                            " in the canonical objective expression."
                            % (var.name,
                               objective_object.name,
                               StochasticObjectiveAnnotation.__name__))
                    var_label = symbol_map.byObject[id(var)]
                    stochastic_cost_count += 1
                    f_obj_struct.write(obj_struct_template % (var_label))
                    f_obj.write(obj_template
                                % (_no_negative_zero(value(var_coef))))

                objective_repn.linear_coefs = tuple(new_coefs)
                if include_constant:
                    obj_constant = objective_repn.constant
                    # We are going to rewrite the core problem file
                    # with all stochastic values set to zero. This will
                    # allow an easy test for missing user annotations.
                    objective_repn.constant = _deterministic_check_value
                    if obj_constant is None:
                        obj_constant = 0.0
                    stochastic_cost_count += 1
                    f_obj_struct.write(obj_struct_template % ("ONE_VAR_CONSTANT"))
                    f_obj.write(obj_template % (_no_negative_zero(obj_constant)))

    #
    # Write the deterministic part of the LP/MPS-file to its own
    # file for debugging purposes
    #
    reference_model_name = reference_model.name
    reference_model._name = "ZeroStochasticData"
    det_output_filename = os.path.join(output_directory,
                                       scenario.name+".lp.det")
    with WriterFactory("lp") as writer:
        output_fname, symbol_map = writer(reference_model,
                                          det_output_filename,
                                          lambda x: True,
                                          io_options)
        assert output_fname == det_output_filename
    reference_model._name = reference_model_name

    # reset bounds on any constraints that were modified
    for con, lower in iteritems(modified_constraint_lb):
        con._lower = as_numeric(lower)
    for con, upper in iteritems(modified_constraint_ub):
        con._upper = as_numeric(upper)

    return (firststage_variable_count,
            secondstage_variable_count,
            firststage_constraint_count,
            secondstage_constraint_count,
            stochastic_cost_count,
            stochastic_rhs_count,
            stochastic_matrix_count)