class TrustRegionSolver(plugin.Plugin):
"""
A trust region filter method for black box / glass box optimizaiton
Solves nonlinear optimization problems containing external function calls
through automatic construction of reduced models (ROM), also known as
surrogate models.
Currently implements linear and quadratic reduced models.
See Eason, Biegler (2016) AIChE Journal for more details
Arguments:
"""
# + param.CONFIG.generte_yaml_template()
plugin.implements(IOptSolver)
plugin.alias(
'trustregion',
doc='Trust region filter method for black box/glass box optimization')
def available(self, exception_flag=True):
"""Check if solver is available.
TODO: For now, it is always available. However, sub-solvers may not
always be available, and so this should reflect that possibility.
"""
return True
def version(self):
"""Return a 3-tuple describing the solver version."""
return __version__
def solve(self, model, eflist, **kwds):
assert not kwds
#config = param.CONFIG(kwds)
return TRF(model, eflist) #, config)
class JSONSolutionSaverExtension(PySPConfiguredExtension, PySPConfiguredObject,
SingletonPlugin):
implements(IPySPSolutionSaverExtension)
_declared_options = \
PySPConfigBlock("Options declared for the "
"JSONSolutionSaverExtension class")
safe_declare_common_option(_declared_options, "output_name")
safe_declare_common_option(_declared_options, "save_stages")
_default_prefix = "jsonsaver_"
#
# Note: Do not try to user super() or access the
# class name inside the __init__ method when
# a class derives from a SingletonPlugin. Due to
# how Pyutilib implements its Singleton type,
# the __class__ cell will be empty.
# (See: https://stackoverflow.com/questions/
# 13126727/how-is-super-in-python-3-implemented)
#
def __init__(self):
PySPConfiguredExtension.__init__(self)
def save(self, manager):
if self.get_option("output_name") is not None:
stage_solutions = []
# Do NOT open file in 'binary' mode when dumping JSON
# (produces an error in Python3)
with open(self.get_option('output_name'), 'w') as f:
cntr = 0
for stage in manager.scenario_tree.stages:
if (self.get_option('save_stages') <= 0) or \
(cntr+1 <= self.get_option('save_stages')):
cntr += 1
node_solutions = {}
for tree_node in stage.nodes:
_node_solution = extract_node_solution(tree_node)
if _node_solution is None:
print(
"No solution appears to be stored in node with "
"name %s. No solution will be saved." %
(tree_node.name))
return False
node_solutions[tree_node.name] = _node_solution
stage_solutions.append(node_solutions)
else:
break
json.dump(stage_solutions, f, indent=2, sort_keys=True)
print("Saved scenario tree solution for %s time stages "
"to file %s" % (cntr, self.get_option('output_name')))
return True
print("No value was set for %s option 'output_name'. "
"Nothing will be saved." % (type(self).__name__))
return False
class ConnectorExpander(Plugin):
implements(IPyomoScriptModifyInstance)
def apply(self, **kwds):
instance = kwds.pop('instance')
xform = TransformationFactory('core.expand_connectors')
xform.apply_to(instance, **kwds)
return instance
class TMP(Plugin):
implements(modelapi[key], service=True)
def __init__(self):
self.fn = getattr(data.local.usermodel, key)
def apply(self, **kwds):
return self.fn(**kwds)
class ConvexHull_Transformation_Plugin(Plugin):
implements(IPyomoScriptModifyInstance, service=True)
def apply(self, **kwds):
instance = kwds.pop('instance')
xform = TransformationFactory('gdp.chull')
return xform.apply(instance, **kwds)
class BigM_Transformation_PyomoScript_Plugin(Plugin):
implements(IPyomoScriptModifyInstance, service=True)
def apply(self, **kwds):
instance = kwds.pop('instance')
# Not sure why the ModifyInstance callback started passing the
# model along with the instance. We will ignore it.
model = kwds.pop('model', None)
xform = TransformationFactory('gdp.bigm')
return xform.apply_to(instance, **kwds)
class Transformation(Plugin):
"""
Base class for all model transformations.
"""
implements(IModelTransformation, service=False)
def __init__(self, **kwds):
kwds["name"] = kwds.get("name", "transformation")
super(Transformation, self).__init__(**kwds)
@deprecated(
"Transformation.apply() has been deprecated. Please use either "
"Transformation.apply_to() for in-place transformations or "
"Transformation.create_using() for transformations that create a "
"new, independent transformed model instance.")
def apply(self, model, **kwds):
inplace = kwds.pop('inplace', True)
if inplace:
self.apply_to(model, **kwds)
else:
return self.create_using(model, **kwds)
def apply_to(self, model, **kwds):
"""
Apply the transformation to the given model.
"""
timer = TransformationTimer(self, 'in-place')
if not hasattr(model, '_transformation_data'):
model._transformation_data = TransformationData()
self._apply_to(model, **kwds)
timer.report()
def create_using(self, model, **kwds):
"""
Create a new model with this transformation
"""
timer = TransformationTimer(self, 'out-of-place')
if not hasattr(model, '_transformation_data'):
model._transformation_data = TransformationData()
new_model = self._create_using(model, **kwds)
timer.report()
return new_model
def _apply_to(self, model, **kwds):
raise RuntimeError(
"The Transformation.apply_to method is not implemented.")
def _create_using(self, model, **kwds):
instance = model.clone()
self._apply_to(instance, **kwds)
return instance
class UnknownDataManager(Plugin):
implements(IDataManager)
def __init__(self, *args, **kwds):
Plugin.__init__(self, **kwds)
#
# The 'type' is the class type of the solver instance
#
self.type = kwds["type"]
def available(self):
return False
class PyomoDataCommands(Plugin):
alias("dat", "Pyomo data command file interface")
implements(IDataManager, service=False)
def __init__(self):
self._info = []
self.options = Options()
def available(self):
return True
def initialize(self, **kwds):
self.filename = kwds.pop('filename')
self.add_options(**kwds)
def add_options(self, **kwds):
self.options.update(kwds)
def open(self):
if self.filename is None: #pragma:nocover
raise IOError("No filename specified")
if not os.path.exists(self.filename): #pragma:nocover
raise IOError("Cannot find file '%s'" % self.filename)
def close(self):
pass
def read(self):
"""
This function does nothing, since executing Pyomo data commands
both reads and processes the data all at once.
"""
pass
def write(self, data): #pragma:nocover
"""
This function does nothing, because we cannot write to a *.dat file.
"""
pass
def process(self, model, data, default):
"""
Read Pyomo data commands and process the data.
"""
_process_include(['include', self.filename], model, data, default,
self.options)
def clear(self):
self._info = []
class ConvexHull_Transformation_PyomoScript_Plugin(Plugin):
"""Plugin to automatically call the GDP Convex Hull relaxation within
the Pyomo script.
"""
implements(IPyomoScriptModifyInstance, service=True)
def apply(self, **kwds):
instance = kwds.pop('instance')
# Not sure why the ModifyInstance callback started passing the
# model along with the instance. We will ignore it.
model = kwds.pop('model', None)
xform = TransformationFactory('gdp.chull')
return xform.apply_to(instance, **kwds)
class ExpressionRegistration(Plugin):
implements(IPyomoExpression, service=False)
def __init__(self, type, cls, swap=False):
self._type = type
self._cls = cls
self._swap = swap
def type(self):
return self._type
def create(self, args):
if self._swap:
args = list(args)
args.reverse()
return self._cls(args)
class OptimalityPHExtension(SingletonPlugin):
implements(phextension.IPHExtension)
def reset(self, ph):
pass
def pre_ph_initialization(self, ph):
print "Adding OptimalityGapConvergence to list of convergers, with target={}.".format(
convergence_threshold)
# code here is similar to _converger-related code in pyomo.pysp.ph
# note: this is applied in addition to the other convergers, so those
# should be set to strict values if we want this to be the main criterion
# TODO: find a way to specify the threshold on the command-line
# (a crude way would be to write several different versions of this file
# with different hard-coded thresholds.)
ph._convergers.append(
OptimalityGapConvergence(
convergence_threshold=convergence_threshold))
def post_instance_creation(self, ph):
pass
def post_ph_initialization(self, ph):
pass
def post_iteration_0_solves(self, ph):
pass
def post_iteration_0(self, ph):
pass
def pre_iteration_k_solves(self, ph):
pass
def post_iteration_k_solves(self, ph):
pass
def post_iteration_k(self, ph):
pass
def post_ph_execution(self, ph):
pass
class TMP(Plugin):
implements(IModelComponent, service=False)
alias(cls.__name__, description)
component = cls
class TableData(Plugin):
"""
An object that imports data from a table in an external data source.
"""
implements(IDataManager, service=False)
def __init__(self):
"""
Constructor
"""
self._info = None
self._data = None
self.options = Options()
self.options.ncolumns = 1
def available(self):
return True
def initialize(self, **kwds):
self.filename = kwds.pop('filename')
self.add_options(**kwds)
def add_options(self, **kwds):
self.options.update(kwds)
def open(self): #pragma:nocover
"""
Open the table
"""
pass
def read(self): #pragma:nocover
"""
Read data from the table
"""
return False
def write(self, data): #pragma:nocover
"""
Write data from the table
"""
return False
def close(self): #pragma:nocover
"""
Close the table
"""
pass
def process(self, model, data, default):
"""
Return the data that was extracted from this table
"""
if model is None:
model = self.options.model
if not self.options.namespace in data:
data[self.options.namespace] = {}
return _process_data(self._info,
model,
data[self.options.namespace],
default,
self.filename,
index=self.options.index,
set=self.options.set,
param=self.options.param,
ncolumns=self.options.ncolumns)
def clear(self):
"""
Clear the data that was extracted from this table
"""
self._info = None
def _set_data(self, headers, rows):
header_index = []
if self.options.select is None:
for i in xrange(len(headers)):
header_index.append(i)
else:
for i in self.options.select:
header_index.append(headers.index(str(i)))
self.options.ncolumns = len(headers)
if not self.options.param is None:
if not type(self.options.param) in (list, tuple):
self.options.param = (self.options.param, )
_params = []
for p in self.options.param:
if isinstance(p, Param):
self.options.model = p.model()
_params.append(p.name)
else:
_params.append(p)
self.options.param = tuple(_params)
if isinstance(self.options.set, Set):
self.options.model = self.options.set.model()
self.options.set = self.options.set.name
if isinstance(self.options.index, Set):
self.options.model = self.options.index.model()
self.options.index = self.options.index.name
if self.options.format is None:
if not self.options.set is None:
self.options.format = 'set'
elif not self.options.param is None:
self.options.format = 'table'
if self.options.format is None:
raise ValueError("Unspecified format and data option")
elif self.options.set is None and self.options.param is None:
msg = "Must specify the set or parameter option for data"
raise IOError(msg)
if self.options.format == 'set':
if not self.options.index is None:
msg = "Cannot specify index for data with the 'set' format: %s"
raise IOError(msg % str(self.options.index))
self._info = ["set", self.options.set, ":="]
for row in rows:
if self.options.ncolumns > 1:
self._info.append(tuple(row))
else:
self._info.extend(row)
elif self.options.format == 'set_array':
if not self.options.index is None:
msg = "Cannot specify index for data with the 'set_array' " \
'format: %s'
raise IOError(msg % str(self.options.index))
self._info = ["set", self.options.set, ":"]
self._info.extend(headers[1:])
self._info.append(":=")
for row in rows:
self._info.extend(row)
elif self.options.format == 'transposed_array':
self._info = ["param", self.options.param[0], "(tr)", ":"]
self._info.extend(headers[1:])
self._info.append(":=")
for row in rows:
self._info.extend(row)
elif self.options.format == 'array':
self._info = ["param", self.options.param[0], ":"]
self._info.extend(headers[1:])
self._info.append(":=")
for row in rows:
self._info.extend(row)
elif self.options.format == 'table':
if self.options.index is not None:
self._info = ["param", ":", self.options.index, ":"]
else:
self._info = ["param", ":"]
for param in self.options.param:
self._info.append(param)
self._info.append(":=")
for row in rows:
for i in header_index:
self._info.append(row[i])
self.options.ncolumns = len(header_index)
else:
msg = "Unknown parameter format: '%s'"
raise ValueError(msg % self.options.format)
def get_table(self):
tmp = []
if not self.options.columns is None:
tmp.append(self.options.columns)
if not self.options.set is None:
# Create column names
if self.options.columns is None:
cols = []
for i in xrange(self.options.set.dimen):
cols.append(self.options.set.name + str(i))
tmp.append(cols)
# Get rows
for data in self.options.set:
if self.options.set.dimen > 1:
tmp.append(list(data))
else:
tmp.append([data])
elif not self.options.param is None:
if type(self.options.param) in (list, tuple):
_param = self.options.param
else:
_param = [self.options.param]
tmp = []
# Collect data
for index in _param[0]:
if index is None:
row = []
elif type(index) in (list, tuple):
row = list(index)
else:
row = [index]
for param in _param:
row.append(value(param[index]))
tmp.append(row)
# Create column names
if self.options.columns is None:
cols = []
for i in xrange(len(tmp[0]) - len(_param)):
cols.append('I' + str(i))
for param in _param:
cols.append(param)
tmp = [cols] + tmp
return tmp
class ConnectorExpander(Plugin):
implements(IPyomoScriptModifyInstance)
def apply(self, **kwds):
if __debug__ and logger.isEnabledFor(logging.DEBUG): #pragma:nocover
logger.debug("Calling ConnectorExpander")
instance = kwds['instance']
blockList = list(instance.block_data_objects(active=True))
noConnectors = True
for b in blockList:
if b.component_map(Connector):
noConnectors = False
break
if noConnectors:
return
if __debug__ and logger.isEnabledFor(logging.DEBUG): #pragma:nocover
logger.debug(" Connectors found!")
#
# At this point, there are connectors in the model, so we must
# look for constraints that involve connectors and expand them.
#
#options = kwds['options']
#model = kwds['model']
# In general, blocks should be relatively self-contained, so we
# should build the connectors from the "bottom up":
blockList.reverse()
# Expand each constraint involving a connector
for block in blockList:
if __debug__ and logger.isEnabledFor(logging.DEBUG): #pragma:nocover
logger.debug(" block: " + block.name)
CCC = {}
for name, constraint in itertools.chain\
( iteritems(block.component_map(Constraint)),
iteritems(block.component_map(ConstraintList)) ):
cList = []
CCC[name+'.expanded'] = cList
for idx, c in iteritems(constraint._data):
if __debug__ and logger.isEnabledFor(logging.DEBUG): #pragma:nocover
logger.debug(" (looking at constraint %s[%s])", name, idx)
connectors = []
self._gather_connectors(c.body, connectors)
if len(connectors) == 0:
continue
if __debug__ and logger.isEnabledFor(logging.DEBUG): #pragma:nocover
logger.debug(" (found connectors in constraint)")
# Validate that all connectors match
errors, ref, skip = self._validate_connectors(connectors)
if errors:
logger.error(
( "Connector mismatch: errors detected when "
"constructing constraint %s\n " %
(name + (idx and '[%s]' % idx or '')) ) +
'\n '.join(reversed(errors)) )
raise ValueError(
"Connector mismatch in constraint %s" % \
name + (idx and '[%s]' % idx or ''))
if __debug__ and logger.isEnabledFor(logging.DEBUG): #pragma:nocover
logger.debug(" (connectors valid)")
# Fill in any empty connectors
for conn in connectors:
if conn.vars:
continue
for var in ref.vars:
if var in skip:
continue
v = Var()
block.add_component(conn.local_name + '.auto.' + var, v)
conn.vars[var] = v
v.construct()
# OK - expand this constraint
self._expand_constraint(block, name, idx, c, ref, skip, cList)
# Now deactivate the original constraint
c.deactivate()
for name, exprs in iteritems(CCC):
cList = ConstraintList()
block.add_component( name, cList )
cList.construct()
for expr in exprs:
cList.add(expr)
# Now, go back and implement VarList aggregators
for block in blockList:
for conn in itervalues(block.component_map(Connector)):
for var, aggregator in iteritems(conn.aggregators):
c = Constraint(expr=aggregator(block, var))
block.add_component(
conn.local_name + '.' + var.local_name + '.aggregate', c)
c.construct()
def _gather_connectors(self, expr, connectors):
if expr.is_expression():
if expr.__class__ is _ProductExpression:
for e in expr._numerator:
self._gather_connectors(e, connectors)
for e in expr._denominator:
self._gather_connectors(e, connectors)
else:
for e in expr._args:
self._gather_connectors(e, connectors)
elif isinstance(expr, _ConnectorValue):
connectors.append(expr)
def _validate_connectors(self, connectors):
errors = []
ref = None
skip = set()
for idx in xrange(len(connectors)):
if connectors[idx].vars.keys():
ref = connectors.pop(idx)
break
if ref is None:
errors.append(
"Cannot identify a reference connector: no connectors "
"have assigned variables" )
return errors, ref, skip
a = set(ref.vars.keys())
for key, val in iteritems(ref.vars):
if val is None:
skip.add(key)
for tmp in connectors:
b = set(tmp.vars.keys())
if not b:
continue
for key, val in iteritems(tmp.vars):
if val is None:
skip.add(key)
for var in a - b:
# TODO: add a fq_name so we can easily get
# the full model.block.connector name
errors.append(
"Connector '%s' missing variable '%s' "
"(appearing in reference connector '%s')" %
( tmp.name, var, ref.name ) )
for var in b - a:
errors.append(
"Reference connector '%s' missing variable '%s' "
"(appearing in connector '%s')" %
( ref.name, var, tmp.name ) )
return errors, ref, skip
def _expand_constraint(self, block, name, idx, constraint, ref, skip, cList):
def _substitute_var(arg, var):
if arg.is_expression():
if arg.__class__ is _ProductExpression:
_substitute_vars(arg._numerator, var)
_substitute_vars(arg._denominator, var)
else:
_substitute_vars(arg._args, var)
return arg
elif isinstance(arg, _ConnectorValue):
v = arg.vars[var]
if v.is_expression():
v = v.clone()
return _substitute_var(v, var)
elif isinstance(arg, VarList):
return arg.add()
return arg
def _substitute_vars(args, var):
for idx, arg in enumerate(args):
if arg.is_expression():
if arg.__class__ is _ProductExpression:
_substitute_vars(arg._numerator, var)
_substitute_vars(arg._denominator, var)
else:
_substitute_vars(arg._args, var)
elif isinstance(arg, _ConnectorValue):
v = arg.vars[var]
if v.is_expression():
v = v.clone()
args[idx] = _substitute_var(v, var)
elif isinstance(arg, VarList):
args[idx] = arg.add()
for var in ref.vars.iterkeys():
if var in skip:
continue
if constraint.body.is_expression():
c = _substitute_var(constraint.body.clone(), var)
else:
c = _substitute_var(constraint.body, var)
if constraint.equality:
cList.append( ( c, constraint.upper ) )
else:
cList.append( ( constraint.lower, c, constraint.upper ) )
class InterScenarioPlugin(SingletonPlugin):
implements(phextension.IPHExtension)
def __init__(self):
self.enableRhoUpdates = True
self.enableFeasibilityCuts = True
self.enableIncumbentCuts = True
self.epsilon = 1e-7
self.cut_scale = 0 #1e-4
self.allow_variable_slack = False
# Force this plugin to run every N iterations
self.iterationInterval = 100
# Alternative methods to trigger the plugin:
#
# If the convergence metric degrades by either a relative or
# absolute amount
self.convergenceRelativeDegredation = 10.33
self.convergenceAbsoluteDegredation = 10.001
# If at least recutThreshold fraction of all-to-all scenario
# tests produced feasibility cuts
self.recutThreshold = 0.33
# If at least this fraction of unique solutions are preserved
# from one iteration to the next
self.repeated_solution_threshhold = 0.90
# multiplier on computed rho values
self.rhoScale = 0.75
# How quickly rho moves to new values [0..1]
# 0: no damping (jump to calculated rho)
# 1: complete damping (do not change current value of rho)
self.rhoDamping = 0.1
# Minimum difference in objective to include a cut, and minimum
# difference in variable values to include that term in a cut
self.cutThreshold_minDiff = 0.0001
# Fraction of the cut library to use for cross-scenario
# (all-to-all) cuts
self.cutThreshold_crossCut = 0
# Force the InterScenario plugin to re-run while the improvement
# in the Lagrangean bound is at least this much:
self.iteration0RecutBoundImprovement = 0.0025
def reset(self, ph):
self.incumbent = None
self.rho = None
self.x_deviation = None
self.lastConvergenceMetric = None
self.feasibility_cuts = []
self.incumbent_cuts = []
self.lastRun = 0
self.average_solution = None
self.converger = NormalizedTermDiffConvergence()
self.unique_scenario_solutions = []
def pre_ph_initialization(self, ph):
self.reset(ph)
pass
def post_instance_creation(self, ph):
if self.enableRhoUpdates:
rootNode = ph._scenario_tree.findRootNode()
for v in rootNode._xbars:
ph.setRhoAllScenarios(rootNode, v, 0)
pass
def post_ph_initialization(self, ph):
if len(ph._scenario_tree._stages) > 2:
raise RuntimeError(
"InterScenario plugin only works with 2-stage problems")
self._sense_to_min = 1 if ph._objective_sense == minimize else -1
# We are going to manage RHO here. So, we want to turn it off
# until we finish the initial round of interscenario feasibility
# cuts.
if self.enableRhoUpdates:
rootNode = ph._scenario_tree.findRootNode()
for v in rootNode._xbars:
ph.setRhoAllScenarios(rootNode, v, 0)
#self.rho = dict((v,ph._rho) for v in ph._scenario_tree.findRootNode()._xbars)
def post_iteration_0_solves(self, ph):
self._collect_unique_scenario_solutions(ph)
self._interscenario_plugin(ph)
count = 0
while self.rho is None and self.feasibility_cuts:
count += 1
toc("InterScenario plugin: PH iteration 0 re-solve pass %s" %
(count, ))
_stale_scenarios = []
for _id, _soln in enumerate(self.unique_scenario_solutions):
_was_cut = sum(1 for c in self.feasibility_cuts
if type(c[_id]) is tuple)
if _was_cut:
_stale_scenarios.extend(_soln[1])
self._distribute_cuts(ph, True)
toc("InterScenario plugin: distributed cuts to scenarios")
self._collect_unique_scenario_solutions(ph)
self._interscenario_plugin(ph)
self.lastRun = 0
def post_iteration_0(self, ph):
self.converger.update(ph._current_iteration, ph, ph._scenario_tree,
ph._instances)
self.lastConvergenceMetric = self.converger.lastMetric()
pass
def pre_iteration_k_solves(self, ph):
if self.feasibility_cuts or self.incumbent_cuts:
self._distribute_cuts(ph)
pass
def post_iteration_k_solves(self, ph):
self.converger.update(ph._current_iteration, ph, ph._scenario_tree,
ph._instances)
curr = self.converger.lastMetric()
last = self.lastConvergenceMetric
delta = curr - last
#print("InterScenario convergence:", last, curr, delta)
run = False
if (self._collect_unique_scenario_solutions(ph) >=
self.repeated_solution_threshhold):
print("InterScenario plugin: triggered by no change in "
"scenario solutions")
run = True
if (delta > last * self.convergenceRelativeDegredation
and delta > self.convergenceAbsoluteDegredation):
print("InterScenario plugin: triggered by convergence degredation "
"(%0.4f; %+0.4f)" % (curr, delta))
run = True
if ph._current_iteration - self.lastRun >= self.iterationInterval:
print("InterScenario plugin: triggered by iteration limit")
run = True
if self.rho is None:
print("InterScenario plugin: triggered to initialize rho")
run = True
elif self.enableRhoUpdates:
rootNode = ph._scenario_tree.findRootNode()
for _id, rho in iteritems(self.rho):
_max = rootNode._maximums[_id]
_min = rootNode._minimums[_id]
if rho < self.epsilon and _max - _min > self.epsilon:
print("InterScenario plugin: triggered by variable "
"divergence with rho==0 (%s: %s; [%s, %s])" %
(_id, rho, _max, _min))
run = True
break
if run:
self.lastRun = ph._current_iteration
self._interscenario_plugin(ph)
self.lastConvergenceMetric = curr
pass
def post_iteration_k(self, ph):
pass
def post_ph_execution(self, ph):
self._collect_unique_scenario_solutions(ph)
self._interscenario_plugin(ph)
pass
def _interscenario_plugin(self, ph):
toc("InterScenario plugin: analyzing scenario dual information")
# (1) Collect all scenario (first) stage variables
#self._collect_unique_scenario_solutions(ph)
# (2) Filter them to find a set we want to distribute
pass
# (3) Distribute (some) of the variable sets out to the
# scenarios, fix, and resolve; Collect and return the
# objectives, duals, and any cuts
partial_obj_values, dual_values, cuts, probability \
= self._solve_interscenario_solutions( ph )
# Compute the non-anticipative objective values for each
# scenario solution
self.feasible_objectives = self._compute_objective(
partial_obj_values, probability)
for _id, soln in enumerate(self.unique_scenario_solutions):
_scenarios = [ph._scenario_tree.get_scenario(x) for x in soln[1]]
print(
" Solution %2d: generated %2d cuts, "
"cut by %2d other scenarios; objective %10s, "
"scenario cost [%s], cut obj [%s] [generated by %s]" %
(_id, sum(1 for c in cuts[_id] if type(c) is tuple),
sum(1 for c in cuts if type(c[_id]) is tuple), "None"
if self.feasible_objectives[_id] is None else "%10.2f" %
self.feasible_objectives[_id], ", ".join("%10.2f" % x._cost
for x in _scenarios),
" ".join("%5.2f" % x[0] if type(x) is tuple else "%5s" % x
for x in cuts[_id]), ','.join(soln[1])))
scenarioCosts = [
ph._scenario_tree.get_scenario(x)._cost
for s in self.unique_scenario_solutions for x in s[1]
]
scenarioProb = [
ph._scenario_tree.get_scenario(x)._probability
for s in self.unique_scenario_solutions for x in s[1]
]
_avg = sum(scenarioProb[i] * c for i, c in enumerate(scenarioCosts))
_max = max(scenarioCosts)
_min = min(scenarioCosts)
if self.average_solution is None:
_del_avg = None
_del_avg_str = "-----%"
else:
_prev = self.average_solution
_del_avg = (_avg - _prev) / max(abs(_avg), abs(_prev))
_del_avg_str = "%+.2f%%" % (100 * _del_avg, )
self.average_solution = _avg
print(" Average scenario cost: %f (%s) Max-min: %f (%0.2f%%)" %
(_avg, _del_avg_str, _max - _min, abs(100. *
(_max - _min) / _avg)))
# (4) save any cuts for distribution before the next solve
#self.feasibility_cuts = []
#for c in cuts:
# self.feasibility_cuts.extend(
# x for x in c if type(x) is tuple and x[0] > self.cutThreshold )
#cutCount = len(self.feasibility_cuts)
if self.enableFeasibilityCuts:
self.feasibility_cuts = cuts
cutCount = sum(
sum(1 for x in c
if type(x) is tuple and x[0] > self.cutThreshold_minDiff)
for c in cuts)
subProblemCount = sum(len(c) for c in cuts)
# (5) compute and publish the new incumbent
self._update_incumbent(ph)
# (6a) If this is iteration 0, and we have feasibility cuts, and
# they are (sufficiently) helping the Lagrangean bound, then
# skip setting rho and do another round oc cuts
if ph._current_iteration == 0:
# Tell ph that we may have a good opter bound
ph._update_reported_bounds(outer=self.average_solution)
if (cutCount > self.recutThreshold * (subProblemCount - len(cuts))
and (_del_avg is None
or _del_avg > self.iteration0RecutBoundImprovement)):
# Bypass RHO updates and check for more cuts
#self.lastRun = ph._current_iteration - self.iterationInterval
return
# (6b) compute updated rho estimates
new_rho, loginfo = self._process_dual_information(
ph, dual_values, probability)
_scale = self.rhoScale
if self.rho is None:
print("InterScenario plugin: initializing rho")
self.rho = {}
for v, r in iteritems(new_rho):
self.rho[v] = _scale * r
else:
_damping = self.rhoDamping
for v, r in iteritems(new_rho):
if self.rho[v]:
self.rho[v] += (1 - _damping) * (_scale * r - self.rho[v])
#self.rho[v] = max(_scale*r, self.rho[v]) - \
# _damping*abs(_scale*r - self.rho[v])
else:
self.rho[v] = _scale * r
for v, l in sorted(iteritems(loginfo)):
if v is None:
print(l)
else:
print(l % (self.rho[v], ))
#print("SETTING SELF.RHO", self.rho)
rootNode = ph._scenario_tree.findRootNode()
if self.enableRhoUpdates:
for v, r in iteritems(self.rho):
ph.setRhoAllScenarios(rootNode, v, r)
def _collect_unique_scenario_solutions(self, ph):
# list of (varmap, scenario_list) tuples
_old_unique_scenario_solutions = self.unique_scenario_solutions
self.unique_scenario_solutions = []
# See ph.py:update_variable_statistics for a multistage version...
rootNode = ph._scenario_tree.findRootNode()
for scenario in rootNode._scenarios:
_this_sol = dict(scenario._x[rootNode._name])
for _id, _val in iteritems(scenario._x[rootNode._name]):
#if rootNode.is_variable_fixed(_id):
# continue
if rootNode.is_variable_binary(_id) or \
rootNode.is_variable_integer(_id):
_this_sol[_id] = int(round(_val))
found = False
# Note: because we are looking for unique variable values,
# then if the user is bundling, this will implicitly re-form
# the bundles
for _sol in self.unique_scenario_solutions:
if _this_sol == _sol[0]:
_sol[1].append(scenario._name)
found = True
break
if not found:
self.unique_scenario_solutions.append(
(_this_sol, [scenario._name]))
_unchanged = 0
for _old_soln, _old_scen in _old_unique_scenario_solutions:
for _soln, _scen in self.unique_scenario_solutions:
if _old_soln == _soln:
_unchanged += 1
break
print("Interscenario plugin: %s unchanged scenario solutions "
"(out of %s)" %
(_unchanged, len(self.unique_scenario_solutions)))
return float(_unchanged) / len(self.unique_scenario_solutions)
def _solve_interscenario_solutions(self, ph):
results = (
[],
[],
[],
)
probability = []
#cutlist = []
distributed = isinstance(ph._solver_manager, SolverManager_PHPyro)
action_handles = []
if ph._scenario_tree.contains_bundles():
subproblems = ph._scenario_tree._scenario_bundles
else:
subproblems = ph._scenario_tree._scenarios
for problem in subproblems:
probability.append(problem._probability)
options = (self.unique_scenario_solutions, )
kwd_options = {
'epsilon': self.epsilon,
'cut_scale': self.cut_scale,
'allow_slack': self.allow_variable_slack,
'enable_rho': self.enableRhoUpdates,
'enable_cuts': self.enableFeasibilityCuts
}
if distributed:
action_handles.append(
ph._solver_manager.queue(
action="invoke_external_function",
name=problem._name,
queue_name=ph._phpyro_job_worker_map[problem._name],
invocation_type=InvocationType.SingleInvocation.key,
generateResponse=True,
module_name='pyomo.pysp.plugins.interscenario',
function_name='solve_fixed_scenario_solutions',
function_kwds=kwd_options,
function_args=options,
))
else:
_tmp = solve_fixed_scenario_solutions(ph, ph._scenario_tree,
problem, *options,
**kwd_options)
for i, r in enumerate(results):
r.append(_tmp[i])
#cutlist.extend(_tmp[-1])
if distributed:
num_results_so_far = 0
num_results = len(action_handles)
for r in results:
r.extend([None] * num_results)
while (num_results_so_far < num_results):
_ah = ph._solver_manager.wait_any()
_ah_id = action_handles.index(_ah)
_tmp = ph._solver_manager.get_results(_ah)
for i, r in enumerate(results):
r[_ah_id] = _tmp[i]
#cutlist.extend(_tmp[-1])
num_results_so_far += 1
return results + (probability, ) # + (cutlist,)
def _distribute_cuts(self, ph, resolve=False):
totalCuts = 0
cutObj = sorted(
c[0] for x in self.feasibility_cuts for c in x
if type(c) is tuple and c[0] > self.cutThreshold_minDiff)
if cutObj:
allCutThreshold = cutObj[min(
int((1 - self.cutThreshold_crossCut) * len(cutObj)),
len(cutObj) - 1)]
else:
allCutThreshold = 1
distributed = isinstance(ph._solver_manager, SolverManager_PHPyro)
if ph._scenario_tree.contains_bundles():
subproblems = ph._scenario_tree._scenario_bundles
get_scenarios = lambda x: x._scenario_names
else:
subproblems = ph._scenario_tree._scenarios
get_scenarios = lambda x: [x]
resolves = []
for problem in subproblems:
cuts = []
for id, (x, s) in enumerate(self.unique_scenario_solutions):
found = False
for scenario in get_scenarios(problem):
if scenario._name in s:
found = True
break
if found:
cuts.extend(c[id] for c in self.feasibility_cuts
if type(c[id]) is tuple
and c[id][0] > self.cutThreshold_minDiff)
elif self.feasible_objectives[id] is None:
# We only add cuts generated by other scenarios to
# scenarios that are not currently feasible (as
# these are feassibility cuts, they should not
# impact feasible scenarios)
cuts.extend(
c[id] for c in self.feasibility_cuts
if type(c[id]) is tuple and c[id][0] > allCutThreshold)
if not cuts and not self.incumbent_cuts:
resolves.append(None)
continue
totalCuts += len(cuts)
if distributed:
resolves.append(
ph._solver_manager.queue(
action="invoke_external_function",
name=problem._name,
queue_name=ph._phpyro_job_worker_map[problem._name],
invocation_type=InvocationType.SingleInvocation.key,
generateResponse=True,
module_name='pyomo.pysp.plugins.interscenario',
function_name='add_new_cuts',
function_kwds=None,
function_args=(cuts, self.incumbent_cuts, resolve),
))
else:
ans = add_new_cuts(ph, ph._scenario_tree, problem, cuts,
self.incumbent_cuts, resolve)
resolves.append(ans)
toc("distributed cuts to scenario %s%s" %
(problem._name,
' and resolved scenario' if resolve else ''))
toc("InterScenario plugin: added %d feasibility cuts from a "
"library of %s cuts" % (totalCuts, len(cutObj)))
self.feasibility_cuts = []
if self.incumbent_cuts:
print("InterScenario plugin: added %d incumbent cuts" %
(len(self.incumbent_cuts), ))
self.incumbent_cuts = []
if distributed:
num_results_so_far = sum(1 for x in resolves if x is None)
num_results = len(resolves)
while (num_results_so_far < num_results):
_ah = ph._solver_manager.wait_any()
_ah_idx = resolves.index(_ah)
resolves[_ah_idx] = ph._solver_manager.get_results(_ah)
num_results_so_far += 1
if resolve:
# Transfer the first stage values and cost back to PH and
# recompute xbar
rootNode = ph._scenario_tree.findRootNode()
for _id, problem in enumerate(subproblems):
ans = resolves[_id]
if ans is None:
continue
for scenario in get_scenarios(problem):
scenario._cost = ans[1]
assert (sorted(ans[0]) == sorted(
scenario._x[rootNode._name]))
scenario._x[rootNode._name] = ans[0] #[_vid] = _vval
ph.update_variable_statistics()
def _compute_objective(self, partial_obj_values, probability):
obj_values = []
for soln_id in xrange(len(self.unique_scenario_solutions)):
obj = 0.
for scen_or_bundle_id, p in enumerate(probability):
if partial_obj_values[scen_or_bundle_id][soln_id] is None:
obj = None
break
obj += p * partial_obj_values[scen_or_bundle_id][soln_id]
obj_values.append(obj)
return obj_values
def _update_incumbent(self, ph):
feasible_obj = [
o for o in enumerate(self.feasible_objectives) if o[1] is not None
]
if not feasible_obj:
print("InterScenario plugin: No scenario solutions are "
"globally feasible")
return
print("InterScenario plugin: Feasible objectives: %s" %
(sorted(o[1] for o in feasible_obj), ))
best_id, best_obj = min(
((x[0], self._sense_to_min * x[1]) for x in feasible_obj),
key=operator.itemgetter(1))
binary_vars = []
integer_vars = []
continuous_vars = []
rootNode = ph._scenario_tree.findRootNode()
for _id in rootNode._scenarios[0]._x[rootNode._name]:
if rootNode.is_variable_fixed(_id):
continue
if rootNode.is_variable_binary(_id):
binary_vars.append(_id)
elif rootNode.is_variable_integer(_id):
integer_vars.append(_id)
elif rootNode.is_variable_semicontinuous(_id):
assert False, "FIXME"
else:
# we can not add incumbent cuts for continuous domains
continuous_vars.append(_id)
if self.incumbent is None or \
self.incumbent[0] * self._sense_to_min > best_obj + self.epsilon:
# Cut the old incumbent
if self.enableIncumbentCuts and self.incumbent and not continuous_vars:
_x = self.incumbent[1][0]
self.incumbent_cuts.append((
dict((vid, round(_x[vid])) for vid in binary_vars),
dict((vid, round(_x[vid])) for vid in integer_vars),
))
# New incumbent!
self.incumbent = (best_obj * self._sense_to_min,
self.unique_scenario_solutions[best_id], best_id)
# Tell PH (that we have a good inner bound)
ph._update_reported_bounds(inner=best_obj)
msg = "InterScenario plugin: NEW incumbent: %s = %s, %s" \
% self.incumbent
print(msg)
logger.info(msg)
elif self.incumbent[0] * self._sense_to_min < best_obj - self.epsilon:
# Keep existing incumbent... so the best thing here can be cut
msg = "InterScenario plugin: incumbent: %s = %s, %s" \
% self.incumbent
print(msg)
best_id = -1
if continuous_vars or not self.enableIncumbentCuts:
return
for _id, obj in feasible_obj:
if _id == best_id:
continue
_x = self.unique_scenario_solutions[_id][0]
self.incumbent_cuts.append((
dict((vid, round(_x[vid])) for vid in binary_vars),
dict((vid, round(_x[vid])) for vid in integer_vars),
))
def _process_dual_information(self, ph, dual_values, probability):
# Notes:
# dual_values: [ [ { var_id: dual } ] ]
# - list of list of maps of variable id to dual value. The
# outer list is returned by each subproblem (corresponds to
# a bundle or scenario). The order in this list matches
# the order in the probability list. The inner list holds
# the dual values for each solution the scenario/bundle was
# asked to evaluate. This inner list is in the same order
# as the solutions list.
# probability: [ scenario/bundle probility ]
# - list of the scenario or bundle probability for the
# submodel that returned the corresponding objective/dual
# values
# unique_scenario_solutions: [ {var_id:var_value}, [ scenario_names ] ]
# - list of candidate solutions holding the 1st stage
# variable values (in a map) and the list of scenarios
# that had that solution as the optimal solution in this
# iteration
# soln_prob: the total probability of all scenarios that have
# this solution as their locally-optimal solution
soln_prob = [0.] * len(self.unique_scenario_solutions)
for soln_id, soln_info in enumerate(self.unique_scenario_solutions):
for src_scen_name in soln_info[1]:
src_scen = ph._scenario_tree.get_scenario(src_scen_name)
soln_prob[soln_id] += src_scen._probability
total_soln_prob = sum(soln_prob)
# xbar: { var_id : xbar }
# - this has the average first stage variable values. We
# should really get this from the scenario tree, as we
# cannot guarantee that we will see all the current values
# here (they can be filtered)
#xbar = dict( (
# k,
# sum(v*soln_prob[i] for i,v in enumerate(vv))/total_soln_prob )
# for k, vv in iteritems(var_info) )
xbar = ph._scenario_tree.findRootNode()._xbars
if self.x_deviation is None:
self.x_deviation = dict(
(v, max(s[0][v] for s in self.unique_scenario_solutions) -
min(s[0][v] for s in self.unique_scenario_solutions))
for v in xbar)
max_dual = dict((v, 0.) for v in xbar)
weighted_rho = dict((v, 0.) for v in xbar)
for soln_id, soln_p in enumerate(soln_prob):
x = self.unique_scenario_solutions[soln_id][0]
avg_dual = dict((v, 0.) for v in xbar)
p_total = 0.
for scen_id, p in enumerate(probability):
if dual_values[scen_id][soln_id] is None:
continue
for v, d in iteritems(dual_values[scen_id][soln_id]):
avg_dual[v] += math.copysign(d, xbar[v] - x[v]) * p
max_dual[v] = max(max_dual[v], abs(d))
p_total += p
if p_total:
for v in avg_dual:
avg_dual[v] /= p_total
#x_deviation = dict( (v, abs(xbar[v]-self.unique_scenario_solutions[soln_id][0][v]))
# for v in xbar )
for v, x_dev in iteritems(self.x_deviation):
weighted_rho[v] += soln_prob[soln_id] * avg_dual[v] / (x_dev +
1.)
if False: # MAX dual (not average)
for v, x_dev in iteritems(self.x_deviation):
weighted_rho[v] += max_dual[v] / (x_dev + 1.)
# var_info: { var_id : [ scenario values ] }
# - this has the list of all values for a single 1st stage
# variable, in the same order as the solutions list (and the
# soln_prob list)
var_info = {}
for soln_id, soln_info in enumerate(self.unique_scenario_solutions):
for k, v in iteritems(soln_info[0]):
try:
var_info[k].append(v)
except:
var_info[k] = [v]
dual_info = {}
for sid, scenario_results in enumerate(dual_values):
for solution in scenario_results:
if solution is None:
continue
for k, v in iteritems(solution):
try:
dual_info[k].append(v)
except:
dual_info[k] = [v]
# (optionally) scale the weighted rho
#for v in xbar:
# weighted_rho[v] = weighted_rho[v] / total_soln_prob
# Check for rho == 0
_min_rho = min(_rho for _rho in weighted_rho if _rho > 0)
for v in xbar:
if weighted_rho[v] <= 0:
# If there is variable disagreement, but no objective
# pressure to price the disagreement, the best thing we
# can do is guess and let later iterations sort it out.
#
#if max(var_info[v]) - min(var_info[v]) > 0:
# weighted_rho[v] = 1.
#
# Actually, we will just set all 0 rho values to the
# smallest non-zero dual
weighted_rho[v] = _min_rho
loginfo = {
None:
"%4s: %6s [%7s, %7s] %7s; "
"%6s [%6s, %6s] %6s; RHO %7s : %7s" %
('---', 'Dual', 'min', 'max', 'stdev', 'Var', 'min', 'max',
'stdev', 'computed', 'final')
}
for k, duals in iteritems(dual_info):
# DISABLE!
#break
d_min = min(duals)
d_max = max(duals)
_sum = sum(abs(x) for x in duals)
_sumsq = sum(x**2 for x in duals)
n = float(len(duals))
d_avg = _sum / n
d_stdev = math.sqrt(abs(_sumsq / n - d_avg**2))
x_min = min(var_info[k])
x_max = max(var_info[k])
_sum = sum(abs(x) for x in var_info[k])
_sumsq = sum(x**2 for x in var_info[k])
n = float(len(var_info[k]))
x_avg = _sum / n
x_stdev = math.sqrt(abs(_sumsq / n - x_avg**2 + 1e-6))
loginfo[k] = \
"%4d: %6.1f [%7.1f, %7.1f] %7.1f; " \
"%6.1f [%6.1f, %6.1f] %6.1f; RHO %7.2f : %%7.2f" % (
k,
d_avg, d_min, d_max, d_stdev,
x_avg, x_min, x_max, x_stdev,
weighted_rho[k] )
return weighted_rho, loginfo
class JSONSolutionLoaderExtension(PySPConfiguredExtension,
PySPConfiguredObject, SingletonPlugin):
implements(IPySPSolutionLoaderExtension)
_declared_options = \
PySPConfigBlock("Options declared for the "
"JSONSolutionLoaderExtension class")
safe_declare_common_option(_declared_options, "input_name")
safe_declare_common_option(_declared_options, "load_stages")
_default_prefix = "jsonloader_"
#
# Note: Do not try to user super() or access the
# class name inside the __init__ method when
# a class derives from a SingletonPlugin. Due to
# how Pyutilib implements its Singleton type,
# the __class__ cell will be empty.
# (See: https://stackoverflow.com/questions/
# 13126727/how-is-super-in-python-3-implemented)
#
def __init__(self):
PySPConfiguredExtension.__init__(self)
def load(self, manager):
if self.get_option("input_name") is not None:
stage_solutions = None
# Do NOT open file in 'binary' mode when loading JSON
# (produces an error in Python3)
with open(self.get_option("input_name"), 'r') as f:
stage_solutions = json.load(f)
cntr = 0
if self.get_option('load_stages') > len(
manager.scenario_tree.stages):
raise ValueError(
"The value of the %s option (%s) can not be greater than "
"the number of time stages in the local scenario tree (%s)"
% (self.get_full_option_name('load_stages'),
self.get_option('load_stages'),
len(manager.scenario_tree.stages)))
if self.get_option('load_stages') > len(stage_solutions):
raise ValueError(
"The value of the %s option (%s) can not be greater than "
"the number of time stages in the scenario tree solution "
"stored in %s (%s)" %
(self.get_full_option_name('load_stages'),
self.get_option('load_stages'),
self.get_option('input_name'), len(stage_solutions)))
for stage, stage_solution in zip_longest(
manager.scenario_tree.stages, stage_solutions):
if stage_solution is None:
break
if (self.get_option('load_stages') <= 0) or \
(cntr+1 <= self.get_option('load_stages')):
if stage is None:
raise RuntimeError(
"Local scenario tree has fewer stages (%s) than what is "
"held by the solution loaded from file %s. Use the "
"option %s to limit the number of stages that "
"are loaded." %
(cntr, self.get_option('input_name'),
self.get_full_option_name('load_stages')))
cntr += 1
for tree_node in stage.nodes:
try:
node_solution = stage_solution[tree_node.name]
except KeyError:
raise KeyError(
"Local scenario tree contains a tree node "
"that was not found in the solution at time"
"-stage %s: %s" % (cntr, tree_node.name))
load_node_solution(tree_node, node_solution)
else:
break
print("Loaded scenario tree solution for %s time stages "
"from file %s" % (cntr, self.get_option('input_name')))
return True
print("No value was set for %s option 'input_name'. "
"Nothing will be saved." % (type(self).__name__))
return False
class PyomoTask_tmp(with_metaclass(TaskMeta,PyomoTask)):
plugin.alias(_alias)
plugin.implements(IPyomoTask, service=False)
def __init__(self, *args, **kwargs):
kwargs['fn'] = fn
PyomoTask.__init__(self, *args, **kwargs)
if not fn is None:
if len(argspec.args) is 0:
nargs = 0
elif argspec.defaults is None:
nargs = len(argspec.args)
else:
nargs = len(argspec.args) - len(argspec.defaults)
self._kwargs = argspec.args[nargs:]
if nargs != 1 and 'data' not in self._kwargs:
logger.error("A Pyomo functor '%s' must have a 'data argument" % _alias)
if argspec.defaults is None:
_defaults = {}
else:
_defaults = dict(list(zip(argspec.args[nargs:], argspec.defaults)))
#
docinfo = parse_docstring(fn)
#
if 'data' in docinfo['optional']:
self.inputs.declare('data', doc='A container of labeled data.', optional=True)
else:
self.inputs.declare('data', doc='A container of labeled data.')
for name in argspec.args[nargs:]:
if name in docinfo['optional']:
self.inputs.declare(name, optional=True, default=_defaults[name], doc=docinfo['optional'][name])
elif name in docinfo['required']:
self.inputs.declare(name, doc=docinfo['required'][name])
elif name != 'data':
#print docinfo
logger.error("Argument '%s' is not specified in the docstring!" % name)
#
self.outputs.declare('data', doc='A container of labeled data.')
if outputs is None:
_outputs = list(docinfo['return'].keys())
else:
_outputs = outputs
for name in _outputs:
if name in docinfo['return']:
self.outputs.declare(name, doc=docinfo['return'][name])
else:
logger.error("Return value '%s' is not specified in the docstring!" % name)
#
self._nested_requirements = []
for name in docinfo['required']:
if '.' in name:
self._nested_requirements.append(name)
#
# Error check keys for docinfo
#
for name in docinfo['required']:
if '.' in name:
continue
if not name in self.inputs:
logger.error("Unexpected name '%s' in list of required inputs for functor '%s'" % (name,_alias))
for name in docinfo['optional']:
if not name in self.inputs:
logger.error("Unexpected name '%s' in list of optional inputs for functor '%s'" % (name,_alias))
for name in docinfo['return']:
if not name in self.outputs:
logger.error("Unexpected name '%s' in list of outputs for functor '%s'" % (name,_alias))
#
self.__help__ = fn.__doc__
self.__doc__ = fn.__doc__
self.__short_doc__ = docinfo['short_doc'].strip()
self.__long_doc__ = docinfo['long_doc'].strip()
self.__namespace__ = namespace
class JSONDictionary(Plugin):
alias("json", "JSON file interface")
implements(IDataManager, service=False)
def __init__(self):
self._info = {}
self.options = Options()
def available(self):
return True
def initialize(self, **kwds):
self.filename = kwds.pop('filename')
self.add_options(**kwds)
def add_options(self, **kwds):
self.options.update(kwds)
def open(self):
if self.filename is None:
raise IOError("No filename specified")
def close(self):
pass
def read(self):
"""
This function loads data from a JSON file and tuplizes the nested
dictionaries and lists of lists.
"""
if not os.path.exists(self.filename):
raise IOError("Cannot find file '%s'" % self.filename)
INPUT = open(self.filename, 'r')
jdata = json.load(INPUT)
INPUT.close()
if jdata is None or len(jdata) == 0:
raise IOError("Empty JSON data file")
self._info = {}
for k, v in jdata.items():
self._info[k] = tuplize(v)
def write(self, data):
"""
This function creates a JSON file for the specified data.
"""
with open(self.filename, 'w') as OUTPUT:
jdata = {}
if self.options.data is None:
for k, v in data.items():
jdata[k] = detuplize(v)
elif type(self.options.data) in (list, tuple):
for k in self.options.data:
jdata[k] = detuplize(data[k])
else:
k = self.options.data
jdata[k] = detuplize(data[k])
json.dump(jdata, OUTPUT)
def process(self, model, data, default):
"""
Set the data for the selected components
"""
if not self.options.namespace in data:
data[self.options.namespace] = {}
#
try:
if self.options.data is None:
for key in self._info:
self._set_data(data, self.options.namespace, key,
self._info[key])
elif type(self.options.data) in (list, tuple):
for key in self.options.data:
self._set_data(data, self.options.namespace, key,
self._info[key])
else:
key = self.options.data
self._set_data(data, self.options.namespace, key,
self._info[key])
except KeyError:
raise IOError(
"Data value for '%s' is not available in JSON file '%s'" %
(key, self.filename))
def _set_data(self, data, namespace, name, value):
if type(value) is dict:
data[namespace][name] = value
else:
data[namespace][name] = {None: value}
def clear(self):
self._info = {}
