class DataRecorder(object):
def __init__(self, data):
self._data = data
self._timer = TicTocTimer()
self._category = {}
self._timingHandler = TimingHandler()
timing_logger = logging.getLogger('pyomo.common.timing')
timing_logger.setLevel(logging.INFO)
timing_logger.addHandler(self._timingHandler)
@unittest.pytest.fixture(autouse=True)
def add_data_attribute(self, request):
# set a class attribute on the invoking test context
request.cls.testdata = OrderedDict()
self._data[request.node.nodeid] = request.cls.testdata
yield
request.cls.testdata = None
@unittest.pytest.hookimpl(hookwrapper=True)
def pytest_runtest_call(self, item):
self._timingHandler.setTest(self._data[item.nodeid])
# Trigger garbage collection (try and get a "clean" environment)
gc.collect()
gc.collect()
self._timer.tic("")
# Run the test
yield
# Collect the timing and clean up
self._data[item.nodeid]['test_time'] = self._timer.toc("")
self._timingHandler.clearTest()
def _run_test(self, model_lib, data):
timer = TicTocTimer()
if isinstance(data, six.string_types) and data.endswith('.dat'):
model = model_lib()
modeldir = os.path.dirname(model_lib.__code__.co_filename)
dat_file = os.path.join(modeldir, data)
model = model.create_instance(dat_file)
elif data is None:
model = model_lib()
elif type(data) is tuple:
model = model_lib(*data)
else:
model = model_lib(data)
if not model.is_constructed():
model = model.create_instance()
self.recordTestData('create_instance', timer.toc(''))
for fmt in ('nl', 'lp', 'bar', 'gams'):
if not getattr(self, fmt, 0):
continue
writer = WriterFactory(fmt)
fname = os.path.join(CWD, 'tmp.test.' + fmt)
self.assertFalse(os.path.exists(fname))
try:
timer.tic('')
writer(model, fname, lambda x: True, {})
_time = timer.toc('')
self.assertTrue(os.path.exists(fname))
self.recordTestData(fmt, _time)
finally:
try:
os.remove(fname)
except:
pass
class DataRecorder(nose.plugins.base.Plugin):
enabled = True
def __init__(self, data):
super(DataRecorder, self).__init__()
self._data = data
self._timingHandler = None
self._timer = TicTocTimer()
self._category = {}
def configure(self, options, conf):
super(DataRecorder, self).configure(options, conf)
self.attribPlugin = next(x for x in conf.plugins
if hasattr(x, 'validateAttrib'))
self.enabled = True
def begin(self):
# Set up the interception of the timing reports
self._timingHandler = TimingHandler()
timing_logger = logging.getLogger('pyomo.common.timing')
timing_logger.setLevel(logging.INFO)
timing_logger.addHandler(self._timingHandler)
# Determine and remember the "status" of all key categories. If
# the user did not specify any categories, then they all will
# remain enabled. However, if the user specified any
# categories, then disable the ones that the user did NOT
# request.
for cat in ('nl', 'lp', 'bar', 'gams'):
if not self.attribPlugin.enabled:
self._category[cat] = True
continue
@unittest.category(cat, 'performance', 'long', 'short', 'devel')
class tmp(unittest.TestCase):
def test(self):
pass
req = self.attribPlugin.validateAttrib(tmp.test, tmp)
self._category[cat] = req is not False
def beforeTest(self, test):
addr = test.address()
try:
addr = addr[1] + ":" + addr[2]
except:
addr = test.id()
test.test.testdata = self._data[addr] = OrderedDict()
self._timingHandler.setTest(test.test.testdata)
# disable any categories we are not interested in
for cat, req in iteritems(self._category):
if getattr(test.test, cat, 0):
setattr(test.test, cat, req)
self._timer.tic("")
def afterTest(self, test):
test.test.testdata['test_time'] = self._timer.toc("")
test.test.testdata = None
self._timingHandler.clearTest()
def _run_test(self, model_lib, data):
gc.collect()
timer = TicTocTimer()
if isinstance(data, str) and data.endswith('.dat'):
model = model_lib()
modeldir = os.path.dirname(model_lib.__code__.co_filename)
dat_file = os.path.join(modeldir, data)
model = model.create_instance(dat_file)
elif data is None:
model = model_lib()
elif type(data) is tuple:
model = model_lib(*data)
else:
model = model_lib(data)
if not model.is_constructed():
model = model.create_instance()
self.recordData('create_instance', timer.toc('create_instance'))
markers = [mark.name for mark in self.pytestmark]
for fmt in ('nl', 'lp', 'bar', 'gams'):
if fmt not in markers:
continue
writer = WriterFactory(fmt)
fname = os.path.join(CWD, 'tmp.test.'+fmt)
self.assertFalse(os.path.exists(fname))
gc.collect()
try:
timer.tic(None)
writer(model, fname, lambda x:True, {})
_time = timer.toc(fmt)
self.assertTrue(os.path.exists(fname))
self.recordData(fmt, _time)
finally:
try:
os.remove(fname)
except:
pass
import pyomo.environ as pyo
from pyomo.common.timing import TicTocTimer
from pyomo.core.expr.numeric_expr import LinearExpression
N1 = 10
N2 = 100000
m = pyo.ConcreteModel()
m.x = pyo.Var(list(range(N1)))
timer = TicTocTimer()
timer.tic()
for i in range(N2):
e = sum(i * m.x[i] for i in range(N1))
timer.toc('created expression with sum function')
for i in range(N2):
coefs = [i for i in range(N1)]
lin_vars = [m.x[i] for i in range(N1)]
e = LinearExpression(constant=0, linear_coefs=coefs, linear_vars=lin_vars)
timer.toc('created expression with LinearExpression constructor')
def test_TicTocTimer_tictoc(self):
RES = 1e-2 # resolution (seconds)
timer = TicTocTimer()
abs_time = time.time()
time.sleep(0.1)
with capture_output() as out:
start_time = time.time()
timer.tic(None)
self.assertEqual(out.getvalue(), '')
with capture_output() as out:
start_time = time.time()
timer.tic()
self.assertRegex(
out.getvalue(),
r'\[ [.0-9]+\] Resetting the tic/toc delta timer'
)
time.sleep(0.1)
with capture_output() as out:
delta = timer.toc()
self.assertAlmostEqual(time.time() - start_time, delta, delta=RES)
self.assertAlmostEqual(0, timer.toc(None), delta=RES)
self.assertRegex(
out.getvalue(),
r'\[\+ [.0-9]+\] .* in test_TicTocTimer_tictoc'
)
with capture_output() as out:
total = timer.toc(delta=False)
self.assertAlmostEqual(time.time() - abs_time, total, delta=RES)
self.assertRegex(
out.getvalue(),
r'\[ [.0-9]+\] .* in test_TicTocTimer_tictoc'
)
ref = 0
ref -= time.time()
time.sleep(0.1)
timer.stop()
ref += time.time()
cumul_stop1 = timer.toc(None)
with self.assertRaisesRegex(
RuntimeError,
'Stopping a TicTocTimer that was already stopped'):
timer.stop()
time.sleep(0.1)
cumul_stop2 = timer.toc(None)
self.assertEqual(cumul_stop1, cumul_stop2)
timer.start()
ref -= time.time()
time.sleep(0.1)
# Note: pypy on GHA frequently has timing differences of >0.02s
# for the following tests
RES = 5e-2
with capture_output() as out:
delta = timer.toc()
timer.stop()
ref += time.time()
self.assertAlmostEqual(ref, delta, delta=RES)
#self.assertAlmostEqual(0, timer.toc(None), delta=RES)
self.assertRegex(
out.getvalue(),
r'\[ [.0-9]+\| 1\] .* in test_TicTocTimer_tictoc'
)
with capture_output() as out:
# Note that delta is ignored if the timer is a cumulative timer
total = timer.toc(delta=False)
self.assertAlmostEqual(delta, total, delta=RES)
self.assertRegex(
out.getvalue(),
r'\[ [.0-9]+\| 1\] .* in test_TicTocTimer_tictoc'
)
print('Building model')
print('--------------')
m = create_warehouse_model(num_locations=200, num_customers=200)
# @:report_timing
# @report_timing_with_lin_expr:
print('Building model with LinearExpression')
print('------------------------------------')
m = create_warehouse_linear_expr(num_locations=200, num_customers=200)
# @:report_timing_with_lin_expr
report_timing(False)
# @tic_toc_timer:
timer = TicTocTimer()
timer.tic('start')
m = create_warehouse_model(num_locations=200, num_customers=200)
timer.toc('Built model')
solve_warehouse_location(m)
timer.toc('Wrote LP file and solved')
# @:tic_toc_timer
# @time_parametric:
solve_parametric()
timer.toc('Finished parameter sweep')
# @:time_parametric
# @profile_parametric:
pr = cProfile.Profile()
pr.enable()
solve_parametric()
def test_TicTocTimer_tictoc(self):
SLEEP = 0.1
RES = 0.01 # resolution (seconds): 1/10 the sleep
abs_time = time.time()
timer = TicTocTimer()
time.sleep(SLEEP)
with capture_output() as out:
start_time = time.time()
timer.tic(None)
self.assertEqual(out.getvalue(), '')
with capture_output() as out:
start_time = time.time()
timer.tic()
self.assertRegex(
out.getvalue(),
r'\[ [.0-9]+\] Resetting the tic/toc delta timer'
)
time.sleep(SLEEP)
ref = time.time()
with capture_output() as out:
delta = timer.toc()
self.assertAlmostEqual(ref - start_time, delta, delta=RES)
self.assertAlmostEqual(0, timer.toc(None), delta=RES)
self.assertRegex(
out.getvalue(),
r'\[\+ [.0-9]+\] .* in test_TicTocTimer_tictoc'
)
ref = time.time()
with capture_output() as out:
total = timer.toc(delta=False)
self.assertAlmostEqual(ref - abs_time, total, delta=RES)
self.assertRegex(
out.getvalue(),
r'\[ [.0-9]+\] .* in test_TicTocTimer_tictoc'
)
ref *= -1
time.sleep(SLEEP)
ref += time.time()
timer.stop()
cumul_stop1 = timer.toc(None)
self.assertAlmostEqual(ref, cumul_stop1, delta=RES)
with self.assertRaisesRegex(
RuntimeError,
'Stopping a TicTocTimer that was already stopped'):
timer.stop()
time.sleep(SLEEP)
cumul_stop2 = timer.toc(None)
self.assertEqual(cumul_stop1, cumul_stop2)
ref -= time.time()
timer.start()
time.sleep(SLEEP)
# Note: pypy and osx (py3.8) on GHA occasionally have timing
# differences of >0.01s for the following tests
if 'pypy_version_info' in dir(sys) or sys.platform == 'darwin':
RES *= 2
with capture_output() as out:
ref += time.time()
timer.stop()
delta = timer.toc()
self.assertAlmostEqual(ref, delta, delta=RES)
#self.assertAlmostEqual(0, timer.toc(None), delta=RES)
self.assertRegex(
out.getvalue(),
r'\[ [.0-9]+\| 1\] .* in test_TicTocTimer_tictoc'
)
with capture_output() as out:
# Note that delta is ignored if the timer is a cumulative timer
total = timer.toc(delta=False)
self.assertAlmostEqual(delta, total, delta=RES)
self.assertRegex(
out.getvalue(),
r'\[ [.0-9]+\| 1\] .* in test_TicTocTimer_tictoc'
)
def test_stochpdegas_automatic(self):
timer = TicTocTimer()
from .stochpdegas_automatic import model
instance = model.create_instance(
os.path.join(_dir, 'stochpdegas_automatic.dat'))
self.recordData('create_instance', timer.toc('create_instance'))
# discretize model
discretizer = TransformationFactory('dae.finite_difference')
discretizer.apply_to(instance,
nfe=1,
wrt=instance.DIS,
scheme='FORWARD')
discretizer.apply_to(instance,
nfe=47,
wrt=instance.TIME,
scheme='BACKWARD')
self.recordData('discretize', timer.toc('discretize'))
# What it should be to match description in paper
#discretizer.apply_to(instance,nfe=48,wrt=instance.TIME,scheme='BACKWARD')
TimeStep = instance.TIME.at(2) - instance.TIME.at(1)
def supcost_rule(m, k):
return sum(m.cs * m.s[k, j, t] * (TimeStep) for j in m.SUP
for t in m.TIME.get_finite_elements())
instance.supcost = Expression(instance.SCEN, rule=supcost_rule)
def boostcost_rule(m, k):
return sum(m.ce * m.pow[k, j, t] * (TimeStep) for j in m.LINK_A
for t in m.TIME.get_finite_elements())
instance.boostcost = Expression(instance.SCEN, rule=boostcost_rule)
def trackcost_rule(m, k):
return sum(m.cd * (m.dem[k, j, t] - m.stochd[k, j, t])**2.0
for j in m.DEM for t in m.TIME.get_finite_elements())
instance.trackcost = Expression(instance.SCEN, rule=trackcost_rule)
def sspcost_rule(m, k):
return sum(m.cT * (m.px[k, i, m.TIME.last(), j] -
m.px[k, i, m.TIME.first(), j])**2.0
for i in m.LINK for j in m.DIS)
instance.sspcost = Expression(instance.SCEN, rule=sspcost_rule)
def ssfcost_rule(m, k):
return sum(m.cT * (m.fx[k, i, m.TIME.last(), j] -
m.fx[k, i, m.TIME.first(), j])**2.0
for i in m.LINK for j in m.DIS)
instance.ssfcost = Expression(instance.SCEN, rule=ssfcost_rule)
def cost_rule(m, k):
return 1e-6 * (m.supcost[k] + m.boostcost[k] + m.trackcost[k] +
m.sspcost[k] + m.ssfcost[k])
instance.cost = Expression(instance.SCEN, rule=cost_rule)
def mcost_rule(m):
return (1.0 / m.S) * sum(m.cost[k] for k in m.SCEN)
instance.mcost = Expression(rule=mcost_rule)
def eqcvar_rule(m, k):
return m.cost[k] - m.nu <= m.phi[k]
instance.eqcvar = Constraint(instance.SCEN, rule=eqcvar_rule)
def obj_rule(m):
return (1.0 - m.cvar_lambda) * m.mcost + m.cvar_lambda * m.cvarcost
instance.obj = Objective(rule=obj_rule)
self.recordData('postprocessing', timer.toc('postprocessing'))
for fmt in ('nl', 'bar', 'gams'):
if not getattr(self, fmt, 0):
continue
writer = WriterFactory(fmt)
fname = 'tmp.test.' + fmt
self.assertFalse(os.path.exists(fname))
try:
timer.tic(None)
writer(instance, fname, lambda x: True, {})
_time = timer.toc(fmt)
self.assertTrue(os.path.exists(fname))
self.recordData(fmt, _time)
finally:
try:
os.remove(fname)
except:
pass
def test_TicTocTimer_tictoc(self):
SLEEP = 0.1
RES = 0.02 # resolution (seconds): 1/5 the sleep
# Note: pypy on GHA occasionally has timing
# differences of >0.04s
if 'pypy_version_info' in dir(sys):
RES *= 2.5
# Note: previously, OSX on GHA also had significantly nosier tests
# if sys.platform == 'darwin':
# RES *= 2
abs_time = time.perf_counter()
timer = TicTocTimer()
time.sleep(SLEEP)
with capture_output() as out:
start_time = time.perf_counter()
timer.tic(None)
self.assertEqual(out.getvalue(), '')
with capture_output() as out:
start_time = time.perf_counter()
timer.tic()
self.assertRegex(
out.getvalue(),
r'\[ [.0-9]+\] Resetting the tic/toc delta timer'
)
time.sleep(SLEEP)
ref = time.perf_counter()
with capture_output() as out:
delta = timer.toc()
self.assertAlmostEqual(ref - start_time, delta, delta=RES)
self.assertAlmostEqual(0, timer.toc(None), delta=RES)
self.assertRegex(
out.getvalue(),
r'\[\+ [.0-9]+\] .* in test_TicTocTimer_tictoc'
)
ref = time.perf_counter()
with capture_output() as out:
total = timer.toc(delta=False)
self.assertAlmostEqual(ref - abs_time, total, delta=RES)
self.assertRegex(
out.getvalue(),
r'\[ [.0-9]+\] .* in test_TicTocTimer_tictoc'
)
ref *= -1
time.sleep(SLEEP)
ref += time.perf_counter()
timer.stop()
cumul_stop1 = timer.toc(None)
self.assertAlmostEqual(ref, cumul_stop1, delta=RES)
with self.assertRaisesRegex(
RuntimeError,
'Stopping a TicTocTimer that was already stopped'):
timer.stop()
time.sleep(SLEEP)
cumul_stop2 = timer.toc(None)
self.assertEqual(cumul_stop1, cumul_stop2)
ref -= time.perf_counter()
timer.start()
time.sleep(SLEEP)
with capture_output() as out:
ref += time.perf_counter()
timer.stop()
delta = timer.toc()
self.assertAlmostEqual(ref, delta, delta=RES)
#self.assertAlmostEqual(0, timer.toc(None), delta=RES)
self.assertRegex(
out.getvalue(),
r'\[ [.0-9]+\| 1\] .* in test_TicTocTimer_tictoc'
)
with capture_output() as out:
# Note that delta is ignored if the timer is a cumulative timer
total = timer.toc(delta=False)
self.assertAlmostEqual(delta, total, delta=RES)
self.assertRegex(
out.getvalue(),
r'\[ [.0-9]+\| 1\] .* in test_TicTocTimer_tictoc'
)
def build_multiperiod_design(m,
flowsheet,
initialization=None,
unfix_dof=None,
flowsheet_options={},
initialization_options={},
unfix_dof_options={},
solver=None,
verbose=True,
stochastic=False,
multiyear=False,
multiple_days=False,
**kwargs):
"""
This function constructs multiperiod optimization model
"""
# Create timer object
timer = TicTocTimer()
timer.tic("Processing input information.")
if stochastic:
# If True, set_scenarios must either be passed as an argument,
# or it should defined as an attribute of the model
if "set_scenarios" in kwargs:
set_scenarios = kwargs["set_scenarios"]
elif hasattr(m, "set_scenarios"):
set_scenarios = m.set_scenarios
else:
raise Exception(f"stochastic option is set to True, but set_scenarios has "
f"not been defined. Either pass set_scenarios as an argument "
f"or define it as an attribute of the model.")
if multiyear:
# If True, set_years must either be passed as an argument,
# or it should defined as an attribute of the model
if "set_years" in kwargs:
set_years = kwargs["set_years"]
elif hasattr(m, "set_years"):
set_years = m.set_years
else:
raise Exception(f"multiyear option is set to True, but set_years has "
f"not been defined. Either pass set_years as an argument "
f"or define it as an attribute of the model.")
if multiple_days:
# If True, set_days must either be passed as an argument,
# or it should defined as an attribute of the model
if "set_days" in kwargs:
set_days = kwargs["set_days"]
elif hasattr(m, "set_days"):
set_days = m.set_days
else:
raise Exception(f"multiple_days option is set to True, but set_days has "
f"not been defined. Either pass set_days as an argument "
f"or define it as an attribute of the model.")
# Set of time periods
if "set_time" in kwargs:
set_time = kwargs["set_time"]
elif hasattr(m, "set_time"):
set_time = m.set_time
else:
raise Exception(f"set_time is a required option. Either pass set_time as "
f"an argument or define it as an attribute of the model.")
# Set solver object
if solver is None:
solver = get_solver()
# Construct the set of time periods
if multiyear and multiple_days:
set_period = [(t, d, y) for y in set_years for d in set_days for t in set_time]
elif multiple_days:
set_period = [(t, d) for d in set_days for t in set_time]
else:
set_period = [t for t in set_time]
"""
Period rule
"""
timer.toc("Beginning the formulation of the multiperiod problem.")
def _period_model_rule(options, verbose_flag):
def _period_model(blk):
if verbose_flag:
print("Constructing flowsheet model for ", blk.name)
flowsheet(blk, options=options)
return _period_model
def _build_scenario_model(blk):
blk.period = Block(set_period, rule=_period_model_rule(flowsheet_options, verbose))
return blk
# Construct the multiperiod model
if stochastic:
m.scenario = Block(set_scenarios, rule=_build_scenario_model)
else:
_build_scenario_model(m)
timer.toc("Completed the formulation of the multiperiod problem")
"""
Initialization routine
"""
if initialization is None:
print("*** WARNING *** Initialization function is not provided. "
"Returning the multiperiod model without initialization.")
return
b = ConcreteModel()
flowsheet(b, options=flowsheet_options)
initialization(b, options=initialization_options)
result = solver.solve(b)
try:
assert check_optimal_termination(result)
except AssertionError:
print(f"Flowsheet did not converge to optimality "
f"after fixing the degrees of freedom.")
raise
# Save the solution in json file
to_json(b, fname="temp_initialized_model.json")
timer.toc("Created an instance of the flowsheet and initialized it.")
# Initialize the multiperiod optimization model
if stochastic:
for s in set_scenarios:
for p in set_period:
from_json(m.scenario[s].period[p], fname="temp_initialized_model.json")
else:
for p in set_period:
from_json(m.period[p], fname="temp_initialized_model.json")
timer.toc("Initialized the entire multiperiod optimization model.")
"""
Unfix the degrees of freedom in each period model for optimization model
"""
if unfix_dof is None:
print("*** WARNING *** unfix_dof function is not provided. "
"Returning the model without unfixing degrees of freedom")
return
if stochastic:
for s in set_scenarios:
for p in set_period:
unfix_dof(m.scenario[s].period[p], options=unfix_dof_options)
else:
for p in set_period:
unfix_dof(m.period[p], options=unfix_dof_options)
timer.toc("Unfixed the degrees of freedom from each period model.")