def test_multiproc_doe(self):
problem = Problem()
root = problem.root = Group()
root.add('indep_var', IndepVarComp('x', val=1.0))
root.add('const', IndepVarComp('c', val=2.0))
root.add('mult', ExecComp4Test("y=c*x", nl_delay=1.0))
root.connect('indep_var.x', 'mult.x')
root.connect('const.c', 'mult.c')
num_levels = 25
problem.driver = FullFactorialDriver(num_levels=num_levels,
num_par_doe=7,
load_balance=True)
problem.driver.options['auto_add_response'] = True
problem.driver.add_desvar('indep_var.x',
lower=1.0,
upper=float(num_levels))
problem.driver.add_objective('mult.y')
problem.setup(check=False)
problem.run()
num_cases = 0
for responses, success, msg in problem.driver.get_responses():
responses = dict(responses)
num_cases += 1
self.assertEqual(responses['indep_var.x'] * 2.0,
responses['mult.y'])
self.assertEqual(num_cases, num_levels)
def test_csv_failures(self):
problem = Problem()
root = problem.root = Group()
root.add('indep_var', IndepVarComp('x', val=1.0))
root.add('const', IndepVarComp('c', val=2.0))
root.add('mult', ExecComp4Test("y=c*x", fails=[3, 4]))
root.connect('indep_var.x', 'mult.x')
root.connect('const.c', 'mult.c')
num_levels = 15
problem.driver = FullFactorialDriver(num_levels=num_levels)
problem.driver.add_desvar('indep_var.x',
lower=1.0,
upper=float(num_levels))
problem.driver.add_objective('mult.y')
problem.driver.add_recorder(self.recorder)
problem.setup(check=False)
problem.run()
lines = self.io.getvalue().split('\n')
vnames = lines[0].split(',')
succ_idx = vnames.index('success')
fails = []
for i, line in enumerate(lines[1:]):
succ = line.split(',', 1)[0]
if succ == '0':
fails.append(i)
self.assertEqual(fails, [3, 4])
def test_load_balanced_doe(self):
problem = Problem(impl=impl)
root = problem.root = Group()
root.add('indep_var', IndepVarComp('x', val=1.0))
root.add('const', IndepVarComp('c', val=2.0))
root.add('mult', Mult())
root.connect('indep_var.x', 'mult.x')
root.connect('const.c', 'mult.c')
num_levels = 25
problem.driver = FullFactorialDriver(num_levels=num_levels,
num_par_doe=self.N_PROCS,
load_balance=True)
problem.driver.add_desvar('indep_var.x',
lower=1.0,
upper=float(num_levels))
problem.driver.add_objective('mult.y')
problem.driver.add_recorder(InMemoryRecorder())
problem.setup(check=False)
problem.run()
for data in problem.driver.recorders[0].iters:
self.assertEqual(data['unknowns']['indep_var.x'] * 2.0,
data['unknowns']['mult.y'])
num_cases = len(problem.driver.recorders[0].iters)
if MPI:
lens = problem.comm.allgather(num_cases)
self.assertEqual(sum(lens), num_levels)
else:
self.assertEqual(num_cases, num_levels)
def test_doe_fail_analysis_error(self):
problem = Problem(impl=impl)
root = problem.root = Group()
root.add('indep_var', IndepVarComp('x', val=1.0))
root.add('const', IndepVarComp('c', val=2.0))
if MPI:
fail_rank = 1 # raise exception from this rank
npardoe = self.N_PROCS
else:
fail_rank = 0
npardoe = 1
root.add('mult',
ExecComp4Test("y=c*x", fail_rank=fail_rank, fails=[3, 4]))
root.connect('indep_var.x', 'mult.x')
root.connect('const.c', 'mult.c')
num_levels = 25
problem.driver = FullFactorialDriver(num_levels=num_levels,
num_par_doe=npardoe)
problem.driver.options['auto_add_response'] = True
problem.driver.add_desvar('indep_var.x',
lower=1.0,
upper=float(num_levels))
problem.driver.add_objective('mult.y')
problem.driver.add_response('mult.case_rank')
problem.setup(check=False)
problem.run()
fails_in_fail_rank = 0
nfails = 0
num_cases = 0
for responses, success, msg in problem.driver.get_responses():
responses = dict(responses)
num_cases += 1
if success:
self.assertEqual(responses['indep_var.x'] * 2.0,
responses['mult.y'])
else:
nfails += 1
if responses['mult.case_rank'] == fail_rank:
fails_in_fail_rank += 1
if MPI:
lens = problem.comm.allgather(num_cases)
self.assertEqual(sum(lens), 25)
if self.comm.rank == 0:
self.assertEqual(fails_in_fail_rank, 2)
self.assertEqual(nfails, 2)
else:
self.assertEqual(num_cases, 25)
self.assertEqual(nfails, 2)
def test_load_balanced_doe_crit_fail(self):
problem = Problem(impl=impl)
root = problem.root = Group()
root.add('indep_var', IndepVarComp('x', val=1.0))
root.add('const', IndepVarComp('c', val=2.0))
if MPI:
fail_rank = 1 # raise exception from this rank
else:
fail_rank = 0
if self.comm.rank == fail_rank:
root.add('mult', ExecComp4Test("y=c*x", fails=[3], critical=True))
else:
root.add('mult', ExecComp4Test("y=c*x"))
root.connect('indep_var.x', 'mult.x')
root.connect('const.c', 'mult.c')
num_levels = 25
problem.driver = FullFactorialDriver(num_levels=num_levels,
num_par_doe=self.N_PROCS,
load_balance=True)
problem.driver.add_desvar('indep_var.x',
lower=1.0, upper=float(num_levels))
problem.driver.add_objective('mult.y')
problem.driver.add_recorder(InMemoryRecorder())
problem.setup(check=False)
if MPI:
problem.run()
else:
try:
problem.run()
except Exception as err:
self.assertEqual(str(err), "OMG, a critical error!")
else:
self.fail("expected exception")
for data in problem.driver.recorders[0].iters:
self.assertEqual(data['unknowns']['indep_var.x']*2.0,
data['unknowns']['mult.y'])
num_cases = len(problem.driver.recorders[0].iters)
if MPI:
# in load balanced mode, we can't really predict how many cases
# will actually run before we terminate, so just check to see if
# we at least have less than the full set we'd have if nothing
# went wrong.
lens = problem.comm.allgather(num_cases)
self.assertTrue(sum(lens) < num_levels,
"Cases run (%d) should be less than total cases (%d)" %
(sum(lens), num_levels))
else:
self.assertEqual(num_cases, 3)
def test_doe_fail_critical(self):
problem = Problem(impl=impl)
root = problem.root = Group()
root.add('indep_var', IndepVarComp('x', val=1.0))
root.add('const', IndepVarComp('c', val=2.0))
if MPI:
fail_rank = 1 # raise exception from this rank
else:
fail_rank = 0
if self.comm.rank == fail_rank:
root.add('mult', ExecComp4Test("y=c*x", fails=[3], critical=True))
else:
root.add('mult', ExecComp4Test("y=c*x"))
root.connect('indep_var.x', 'mult.x')
root.connect('const.c', 'mult.c')
num_levels = 25
problem.driver = FullFactorialDriver(num_levels=num_levels,
num_par_doe=self.N_PROCS)
problem.driver.add_desvar('indep_var.x',
lower=1.0,
upper=float(num_levels))
problem.driver.add_objective('mult.y')
problem.driver.add_recorder(InMemoryRecorder())
problem.setup(check=False)
try:
problem.run()
except Exception as err:
with MultiProcFailCheck(self.comm):
if self.comm.rank == fail_rank:
self.assertEqual(str(err), "OMG, a critical error!")
else:
self.assertEqual(
str(err),
"an exception was raised by another MPI process.")
for data in problem.driver.recorders[0].iters:
self.assertEqual(data['unknowns']['indep_var.x'] * 2.0,
data['unknowns']['mult.y'])
num_cases = len(problem.driver.recorders[0].iters)
if MPI:
lens = problem.comm.allgather(num_cases)
self.assertEqual(sum(lens), 12)
else:
self.assertEqual(num_cases, 3)
def test_doe_fail_analysis_error(self):
problem = Problem(impl=impl)
root = problem.root = Group()
root.add('indep_var', IndepVarComp('x', val=1.0))
root.add('const', IndepVarComp('c', val=2.0))
fail_rank = 1 # raise exception from this rank
if self.comm.rank == fail_rank:
root.add('mult', ExecComp4Test("y=c*x", fails=[3, 4]))
else:
root.add('mult', ExecComp4Test("y=c*x"))
root.connect('indep_var.x', 'mult.x')
root.connect('const.c', 'mult.c')
num_levels = 25
problem.driver = FullFactorialDriver(num_levels=num_levels,
num_par_doe=self.N_PROCS)
problem.driver.add_desvar('indep_var.x',
lower=1.0,
upper=float(num_levels))
problem.driver.add_objective('mult.y')
problem.driver.add_recorder(InMemoryRecorder())
problem.setup(check=False)
problem.run()
for data in problem.driver.recorders[0].iters:
self.assertEqual(data['unknowns']['indep_var.x'] * 2.0,
data['unknowns']['mult.y'])
num_cases = len(problem.driver.recorders[0].iters)
if MPI:
lens = problem.comm.allgather(num_cases)
self.assertEqual(sum(lens), 25)
else:
self.assertEqual(num_cases, 25)
nfails = 0
for data in problem.driver.recorders[0].iters:
if not data['success']:
nfails += 1
if self.comm.rank == fail_rank:
self.assertEqual(nfails, 2)
else:
self.assertEqual(nfails, 0)
def test_load_balanced_doe_crit_fail(self):
problem = Problem()
root = problem.root = Group()
root.add('indep_var', IndepVarComp('x', val=1.0))
root.add('const', IndepVarComp('c', val=2.0))
root.add(
'mult',
ExecComp4Test("y=c*x",
fail_rank=(0, 1, 2, 3, 4),
fails=[3],
fail_hard=True))
root.connect('indep_var.x', 'mult.x')
root.connect('const.c', 'mult.c')
num_levels = 25
problem.driver = FullFactorialDriver(num_levels=num_levels,
num_par_doe=5,
load_balance=True)
problem.driver.options['auto_add_response'] = True
problem.driver.add_desvar('indep_var.x',
lower=1.0,
upper=float(num_levels))
problem.driver.add_objective('mult.y')
problem.setup(check=False)
problem.run()
num_cases = 0
for responses, success, msg in problem.driver.get_responses():
responses = dict(responses)
num_cases += 1
self.assertEqual(responses['indep_var.x'] * 2.0,
responses['mult.y'])
# in load balanced mode, we can't really predict how many cases
# will actually run before we terminate, so just check to see if
# we at least have less than the full set we'd have if nothing
# went wrong.
self.assertTrue(
num_cases < num_levels and num_cases >= 3,
"Cases run (%d) should be less than total cases (%d)" %
(num_cases, num_levels))
def test_load_balanced_doe(self):
problem = Problem(impl=impl)
root = problem.root = Group()
root.add('indep_var', IndepVarComp('x', val=1.0))
root.add('const', IndepVarComp('c', val=2.0))
root.add('mult', ExecComp4Test("y=c*x"))
root.connect('indep_var.x', 'mult.x')
root.connect('const.c', 'mult.c')
num_levels = 25
problem.driver = FullFactorialDriver(num_levels=num_levels,
num_par_doe=self.N_PROCS,
load_balance=True)
problem.driver.options['auto_add_response'] = True
problem.driver.add_desvar('indep_var.x',
lower=1.0,
upper=float(num_levels))
problem.driver.add_objective('mult.y')
problem.setup(check=False)
problem.run()
num_cases = 0
for responses, success, msg in problem.driver.get_responses():
responses = dict(responses)
num_cases += 1
if success:
self.assertEqual(responses['indep_var.x'] * 2.0,
responses['mult.y'])
if MPI:
lens = problem.comm.allgather(num_cases)
self.assertEqual(sum(lens), num_levels)
else:
self.assertEqual(num_cases, num_levels)
top.root.connect(
'indep2.rProp', 'subprob.indep2.rProp'
) # Each of SubProblem's IndepVarComp components has to be connected (maybe promoted works too) to a
top.root.connect('indep3.cruiseSpeed', 'subprob.indep3.cruiseSpeed'
) # IndepVarComp component in the top level.
top.root.connect(
'indep4.batteryMass', 'subprob.indep4.batteryMass'
) # Alternatively it might make more sense to output the design variables states as metrics.
top.root.connect('indep5.motorMass', 'subprob.indep5.motorMass')
top.root.connect('indep6.mtom', 'subprob.indep6.mtom')
# TopProblem: set up the parameter study
# for a parameter study, the following drivers can be used:
# UniformDriver, FullFactorialDriver, LatinHypercubeDriver, OptimizedLatinHypercubeDriver
# in this case, we will use FullFactorialDriver
top.driver = FullFactorialDriver(num_levels=1)
# TopProblem: add top.driver's design variables
top.driver.add_desvar('indep1.range', lower=100000.0, upper=110000.0)
# Data collection
recorder = SqliteRecorder('subprob')
recorder.options['record_params'] = True
recorder.options['record_metadata'] = True
top.driver.add_recorder(recorder)
# Setup
top.setup(check=False)
print("range =", top['indep1.range'])
) # This is where you designate what to expose to the outside world
# Add the 'SaveTime' and 'MeasureTime' Components to OptimizationProfiler's root Group.
OptimizationProfiler.root.add('SaveTime', SaveTime())
OptimizationProfiler.root.add('MeasureTime', MeasureTime())
# Connections
OptimizationProfiler.root.connect('p1.x_0', 'SaveTime.pass_in')
OptimizationProfiler.root.connect('p2.y_0', 'OptimizationProblem.p2.y')
OptimizationProfiler.root.connect('SaveTime.pass_out',
'OptimizationProblem.p1.x')
OptimizationProfiler.root.connect('OptimizationProblem.Paraboloid.f_xy',
'MeasureTime.finished')
# Add driver
OptimizationProfiler.driver = FullFactorialDriver(num_levels=11)
# Add design variables and objectives to the parameter study driver
OptimizationProfiler.driver.add_desvar('p1.x_0', lower=-50, upper=50)
OptimizationProfiler.driver.add_desvar('p2.y_0', lower=-50, upper=50)
OptimizationProfiler.driver.add_objective('MeasureTime.time')
OptimizationProfiler.driver.add_objective(
'OptimizationProblem.Paraboloid.f_xy')
OptimizationProfiler.driver.add_objective(
'OptimizationProblem.output1.x_f')
OptimizationProfiler.driver.add_objective(
'OptimizationProblem.output2.y_f')
# Instantiate a top-level Problem 'OptimizationProfilerRepeat'
# Instantiate a Group and add it to OptimizationProfilerRepeat
OptimizationProfilerRepeat = Problem()
def test_load_balanced_doe_soft_fail(self):
problem = Problem()
root = problem.root = Group()
root.add('indep_var', IndepVarComp('x', val=1.0))
root.add('const', IndepVarComp('c', val=2.0))
fail_rank = 1
root.add('mult',
ExecComp4Test("y=c*x", fail_rank=fail_rank, fails=[3, 4, 5]))
root.connect('indep_var.x', 'mult.x')
root.connect('const.c', 'mult.c')
num_levels = 25
num_par_doe = 5
problem.driver = FullFactorialDriver(num_levels=num_levels,
num_par_doe=num_par_doe,
load_balance=True)
problem.driver.options['auto_add_response'] = True
problem.driver.add_desvar('indep_var.x',
lower=1.0,
upper=float(num_levels))
problem.driver.add_objective('mult.y')
problem.driver.add_response('mult.case_rank')
problem.setup(check=False)
problem.run()
num_cases = len(problem.driver.recorders[0].iters)
self.assertEqual(num_cases, num_levels)
nfails = [0] * num_par_doe
nsuccs = [0] * num_par_doe
num_cases = 0
for responses, success, msg in problem.driver.get_responses():
responses = dict(responses)
num_cases += 1
rank = responses['mult.case_rank']
if success:
self.assertEqual(responses['indep_var.x'] * 2.0,
responses['mult.y'])
nsuccs[rank] += 1
else:
nfails[rank] += 1
# there's a chance that the fail rank didn't get enough
# cases to actually fail 3 times, so we need to check
# how many cases it actually got.
cases_in_fail_rank = nsuccs[fail_rank] + nfails[fail_rank]
if cases_in_fail_rank > 5:
self.assertEqual(nfails[fail_rank], 3)
elif cases_in_fail_rank > 4:
self.assertEqual(nfails[fail_rank], 2)
elif cases_in_fail_rank > 3:
self.assertEqual(nfails[fail_rank], 1)
else:
self.assertEqual(nfails[fail_rank], 0)
) # This is where you designate what to expose to the outside world
# Add the 'SaveTime' and 'MeasureTime' Components to OptimizationProfiler's root Group.
OptimizationProfiler.root.add('SaveTime', SaveTime())
OptimizationProfiler.root.add('MeasureTime', MeasureTime())
# Connections
OptimizationProfiler.root.connect('p1.x_0', 'SaveTime.pass_in')
OptimizationProfiler.root.connect('p2.y_0', 'OptimizationProblem.p2.y')
OptimizationProfiler.root.connect('SaveTime.pass_out',
'OptimizationProblem.p1.x')
OptimizationProfiler.root.connect('OptimizationProblem.Paraboloid.f_xy',
'MeasureTime.finished')
# Add driver
OptimizationProfiler.driver = FullFactorialDriver(num_levels=11)
# Add design variables and objectives to the parameter study driver
OptimizationProfiler.driver.add_desvar('p1.x_0', lower=-50, upper=50)
OptimizationProfiler.driver.add_desvar('p2.y_0', lower=-50, upper=50)
OptimizationProfiler.driver.add_objective('MeasureTime.time')
OptimizationProfiler.driver.add_objective(
'OptimizationProblem.Paraboloid.f_xy')
OptimizationProfiler.driver.add_objective(
'OptimizationProblem.output1.x_f')
OptimizationProfiler.driver.add_objective(
'OptimizationProblem.output2.y_f')
# Data collection
recorder = SqliteRecorder('record_results')
recorder.options['record_params'] = True
# Add paraboloidProblem to ParaboloidParameterStudy as a SubProblem called 'ParaboloidProblem'
# Include paraboloidProblem's Problem Inputs and Problem Outputs in 'params' and 'unknowns' fields SubProblem
ParaboloidParameterStudy.root.add(
'ParaboloidProblem',
SubProblem(paraboloidProblem,
params=['p1.x', 'p2.y'],
unknowns=['Paraboloid.f_xy'])
) # This is where you designate what to expose to the outside world
# Connect ParaboloidParameterStudy's IndepVarComps to ParaboloidProblem's params
ParaboloidParameterStudy.root.connect('p1.x', 'ParaboloidProblem.p1.x')
ParaboloidParameterStudy.root.connect('p2.y', 'ParaboloidProblem.p2.y')
# Add driver
ParaboloidParameterStudy.driver = FullFactorialDriver(num_levels=11)
# Add design variables and objectives to the parameter study driver
ParaboloidParameterStudy.driver.add_desvar('p1.x', lower=-50, upper=50)
ParaboloidParameterStudy.driver.add_desvar('p2.y', lower=-50, upper=50)
ParaboloidParameterStudy.driver.add_objective(
'ParaboloidProblem.Paraboloid.f_xy')
# Data collection
recorder = SqliteRecorder('record_results')
recorder.options['record_params'] = True
recorder.options['record_metadata'] = True
ParaboloidParameterStudy.driver.add_recorder(recorder)
# Setup
ParaboloidParameterStudy.setup(check=False)
def test_doe_fail_hard(self):
problem = Problem(impl=impl)
root = problem.root = Group()
root.add('indep_var', IndepVarComp('x', val=1.0))
root.add('const', IndepVarComp('c', val=2.0))
if MPI:
fail_rank = 1 # raise exception from this rank
npardoe = self.N_PROCS
else:
fail_rank = 0
npardoe = 1
root.add(
'mult',
ExecComp4Test("y=c*x",
fail_rank=fail_rank,
fails=[3],
fail_hard=True))
root.connect('indep_var.x', 'mult.x')
root.connect('const.c', 'mult.c')
num_levels = 25
problem.driver = FullFactorialDriver(num_levels=num_levels,
num_par_doe=npardoe)
problem.driver.options['auto_add_response'] = True
problem.driver.add_desvar('indep_var.x',
lower=1.0,
upper=float(num_levels))
problem.driver.add_objective('mult.y')
problem.setup(check=False)
try:
problem.run()
except Exception as err:
with MultiProcFailCheck(self.comm):
if self.comm.rank == fail_rank:
self.assertEqual(str(err), "OMG, a critical error!")
else:
self.assertEqual(
str(err),
"an exception was raised by another MPI process.")
else:
if MPI:
self.fail('exception expected')
nsucc = 0
nfail = 0
num_cases = 0
for responses, success, msg in problem.driver.get_responses():
responses = dict(responses)
num_cases += 1
if success:
self.assertEqual(responses['indep_var.x'] * 2.0,
responses['mult.y'])
nsucc += 1
else:
nfail += 1
self.assertEqual(nfail, 0) # hard errors aren't saved
if MPI:
lens = problem.comm.allgather(num_cases)
self.assertEqual(sum(lens), 12)
else:
self.assertTrue(num_cases < num_levels)
def test_load_balanced_doe_soft_fail(self):
problem = Problem(impl=impl)
root = problem.root = Group()
root.add('indep_var', IndepVarComp('x', val=1.0))
root.add('const', IndepVarComp('c', val=2.0))
if MPI:
fail_rank = 1 # raise exception from this rank
else:
fail_rank = 0
fail_idxs = [3, 4, 5]
root.add('mult',
ExecComp4Test("y=c*x", fail_rank=fail_rank, fails=fail_idxs))
root.connect('indep_var.x', 'mult.x')
root.connect('const.c', 'mult.c')
num_levels = 25
problem.driver = FullFactorialDriver(num_levels=num_levels,
num_par_doe=self.N_PROCS,
load_balance=True)
problem.driver.options['auto_add_response'] = True
problem.driver.add_desvar('indep_var.x',
lower=1.0,
upper=float(num_levels))
problem.driver.add_objective('mult.y')
problem.driver.add_response('mult.case_rank')
problem.setup(check=False)
problem.run()
nfails = 0
num_cases = 0
cases_in_fail_rank = 0
for responses, success, msg in problem.driver.get_responses():
responses = dict(responses)
num_cases += 1
if success:
self.assertEqual(responses['indep_var.x'] * 2.0,
responses['mult.y'])
else:
nfails += 1
if responses['mult.case_rank'] == fail_rank:
cases_in_fail_rank += 1
if MPI and self.comm.rank > 0:
self.assertEqual(num_cases, 0)
else:
self.assertEqual(num_cases, num_levels)
if self.comm.rank == 0:
# FIXME: for now, all cases get sent back to the master process (0),
# even when recorders are parallel.
# there's a chance that the fail rank didn't get enough
# cases to actually fail 3 times, so we need to check
# how many cases it actually got.
if cases_in_fail_rank > 5:
self.assertEqual(nfails, 3)
elif cases_in_fail_rank > 4:
self.assertEqual(nfails, 2)
elif cases_in_fail_rank > 3:
self.assertEqual(nfails, 1)
else:
self.assertEqual(nfails, 0)
else:
self.assertEqual(nfails, 0)
def test_load_balanced_doe_crit_fail(self):
if MPI is None:
raise unittest.SkipTest(
"Can't run this test (even in serial) without mpi4py and petsc4py"
)
problem = Problem(impl=impl)
root = problem.root = Group()
root.add('indep_var', IndepVarComp('x', val=1.0))
root.add('const', IndepVarComp('c', val=2.0))
fail_rank = 1 # raise exception from this rank
root.add(
'mult',
ExecComp4Test("y=c*x",
fail_rank=fail_rank,
fails=[1],
fail_hard=True))
root.connect('indep_var.x', 'mult.x')
root.connect('const.c', 'mult.c')
num_levels = 25
problem.driver = FullFactorialDriver(num_levels=num_levels,
num_par_doe=self.N_PROCS,
load_balance=True)
problem.driver.options['auto_add_response'] = True
problem.driver.add_desvar('indep_var.x',
lower=1.0,
upper=float(num_levels))
problem.driver.add_objective('mult.y')
problem.setup(check=False)
problem.run()
nsucc = 0
nfail = 0
num_cases = 0
for responses, success, msg in problem.driver.get_responses():
responses = dict(responses)
num_cases += 1
if success:
self.assertEqual(responses['indep_var.x'] * 2.0,
responses['mult.y'])
nsucc += 1
else:
nfail += 1
# in load balanced mode, we can't really predict how many cases
# will actually run before we terminate, so just check to see if
# we at least have less than the full set we'd have if nothing
# went wrong.
if MPI:
lens = problem.comm.allgather(num_cases)
self.assertTrue(
sum(lens) < num_levels,
"Cases run (%d) should be less than total cases (%d)" %
(sum(lens), num_levels))
else:
self.assertTrue(num_cases < num_levels)
self.assertEqual(nfail, 0) # hard failure cases are not saved
