def execute(self):
""" Run each parameter set. """
# Prepare parameters and responses.
exprs = {}
case_paths = {}
inputs = []
values = []
for path in self.get_parameters():
if isinstance(path, tuple):
for target in path:
inputs.append(target)
if not is_legal_name(target):
exprs[target] = ExprEvaluator(target)
path = path[0]
else:
inputs.append(path)
if not is_legal_name(path):
exprs[path] = ExprEvaluator(path)
path = make_legal_path(path)
values.append(self.get('case_inputs.' + path))
for path in self.get_responses():
if not is_legal_name(path):
exprs[path] = ExprEvaluator(path)
case_paths[path] = make_legal_path(path)
length = len(values[0]) if values else 0
self.init_responses(length)
# Run each parameter set.
for i in range(length):
# Set inputs.
for j, path in enumerate(inputs):
value = values[j][i]
expr = exprs.get(path)
if expr:
expr.set(value, self.parent)
else:
self.parent.set(path, value)
# Run workflow.
self.workflow.run()
# Get outputs.
for path in self.get_responses():
expr = exprs.get(path)
if expr:
value = expr.evaluate(self.parent)
else:
value = self.parent.get(path)
path = case_paths[path]
self.set('case_outputs.' + path,
value,
index=(i, ),
force=True)
def execute(self):
""" Run each parameter set. """
# Prepare parameters and responses.
exprs = {}
case_paths = {}
inputs = []
values = []
for path in self.get_parameters():
if isinstance(path, tuple):
for target in path:
inputs.append(target)
if not is_legal_name(target):
exprs[target] = ExprEvaluator(target)
path = path[0]
else:
inputs.append(path)
if not is_legal_name(path):
exprs[path] = ExprEvaluator(path)
path = make_legal_path(path)
values.append(self.get('case_inputs.'+path))
for path in self.get_responses():
if not is_legal_name(path):
exprs[path] = ExprEvaluator(path)
case_paths[path] = make_legal_path(path)
length = len(values[0]) if values else 0
self.init_responses(length)
# Run each parameter set.
for i in range(length):
# Set inputs.
for j, path in enumerate(inputs):
value = values[j][i]
expr = exprs.get(path)
if expr:
expr.set(value, self.parent)
else:
self.parent.set(path, value)
# Run workflow.
self.workflow.run()
# Get outputs.
for path in self.get_responses():
expr = exprs.get(path)
if expr:
value = expr.evaluate(self.parent)
else:
value = self.parent.get(path)
path = case_paths[path]
self.set('case_outputs.'+path, value, index=(i,), force=True)
def set_vartree_inputs(driver, cases):
""" Set ``case_inputs`` vartree on `driver` from `cases`. """
nans = [float('NaN')] * len(cases)
parameters = driver.get_parameters()
for path, value in cases[0].get_inputs():
if path not in parameters:
driver.add_parameter(path)
path = make_legal_path(path)
driver.set('case_inputs.'+path, nans)
for i, case in enumerate(cases):
for path, value in case.get_inputs():
path = make_legal_path(path)
driver.set('case_inputs.%s[%d]'%(path,i), value)
def set_vartree_inputs(driver, cases):
""" Set ``case_inputs`` vartree on `driver` from `cases`. """
nans = [float('NaN')] * len(cases)
parameters = driver.get_parameters()
for path, value in cases[0].get_inputs():
if path not in parameters:
driver.add_parameter(path)
path = make_legal_path(path)
driver.set('case_inputs.' + path, nans)
for i, case in enumerate(cases):
for path, value in case.get_inputs():
path = make_legal_path(path)
driver.set('case_inputs.%s[%d]' % (path, i), value)
def add_parameter(self, target, low=None, high=None,
scaler=None, adder=None, start=None,
fd_step=None, name=None, scope=None):
"""Adds a parameter or group of parameters to the driver."""
super(HasVarTreeParameters, self).add_parameter(
target, low, high, scaler, adder, start, fd_step, name, scope)
if name is not None:
path = name
elif isinstance(target, basestring):
path = target
elif isinstance(target, Parameter):
path = target.name or target.target
else:
path = target[0]
path = make_legal_path(path)
obj = self.parent
names = ['case_inputs'] + path.split('.')
for name in names[:-1]:
if obj.get_trait(name):
val = obj.get(name)
else:
val = VariableTree()
obj.add_trait(name, VarTree(val, iotype='in'))
obj = val
name = names[-1]
obj.add_trait(name, List(iotype='in'))
def add_parameter(self,
target,
low=None,
high=None,
scaler=None,
adder=None,
start=None,
fd_step=None,
name=None,
scope=None):
"""We need to create our special variable trees."""
super(AdaptiveSampleDriver,
self).add_parameter(target, low, high, scaler, adder, start,
fd_step, name, scope)
# Parameter gets [0] as an initial case, just for connectability.
if name is not None:
target = name
elif isinstance(target, tuple):
target = target[0] # Use first target of ParameterGroup.
path = make_legal_path(target)
desc = 'Input for adding new points during parent iteration.'
self._add_vartrees('adaptive_inputs', path, desc)
desc = 'Holds all inputs processed by this driver.'
self._add_vartrees('all_case_inputs', path, desc)
desc = 'Holds just the DOE inputs.'
self._add_vartrees('DOE_inputs', path, desc)
def add_parameter(self, target, low=None, high=None,
scaler=None, adder=None, start=None,
fd_step=None, name=None, scope=None):
"""Adds a parameter or group of parameters to the driver."""
super(HasVarTreeParameters, self).add_parameter(
target, low, high, scaler, adder, start, fd_step, name, scope)
if name is not None:
path = name
elif isinstance(target, basestring):
path = target
elif isinstance(target, Parameter):
path = target.name or target.target
else:
path = target[0]
path = make_legal_path(path)
obj = self.parent
names = ['case_inputs'] + path.split('.')
for name in names[:-1]:
if obj.get_trait(name):
val = obj.get(name)
else:
val = VariableTree()
obj.add_trait(name, VarTree(val, iotype='in'))
obj = val
name = names[-1]
obj.add_trait(name, List(iotype='in'))
def _record_case(self, scope, case):
"""
Record case data from `scope` in ``case_outputs``.
Also sends case data to recorders.
"""
case_outputs, exc = case.fetch_outputs(scope)
if exc is None and case.exc is None:
index = case.index
for path, value in case_outputs:
path = make_legal_path(path)
if self.sequential and isinstance(value, VariableTree):
value = value.copy()
self.set('case_outputs.'+path, value,
index=(index,), force=True)
# Record workflow data in recorders.
workflow = self.workflow
if workflow._rec_required:
inputs = []
recording = workflow._rec_parameters
for name, param in self.get_parameters().items():
if param in recording:
if isinstance(name, tuple): # Use first target.
name = name[0]
inputs.append(case._inputs[name])
outputs = []
recording = set(workflow._rec_responses)
for path, value in case_outputs:
if path in recording:
if self.sequential and isinstance(value, VariableTree):
value = value.copy()
outputs.append(value)
itername = '%s.workflow.itername' % self.name
extra, extra_exc = case.fetch_outputs(scope, extra=True,
itername=itername)
for path, value in extra:
if self.sequential and isinstance(value, VariableTree):
value = value.copy()
outputs.append(value)
if itername in workflow._rec_outputs:
if self.itername:
outputs.append('%s.%s' % (self.itername, case.index+1))
else:
outputs.append('%s' % (case.index+1))
top = scope
while top.parent:
top = top.parent
for recorder in top.recorders:
recorder.record(self, inputs, outputs,
case.exc or exc or extra_exc,
case.uuid, self._case_uuid)
def _record_case(self, scope, case):
"""
Record case data from `scope` in ``case_outputs``.
Also sends case data to recorders.
"""
case_outputs, exc = case.fetch_outputs(scope)
if exc is None and case.exc is None:
index = case.index
for path, value in case_outputs:
path = make_legal_path(path)
if self.sequential and isinstance(value, VariableTree):
value = value.copy()
self.set('case_outputs.%s[%d]' % (path, index), value)
def add_response(self, expr, name=None, scope=None):
"""We need to create our special variable trees."""
super(AdaptiveSampleDriver, self).add_response(expr, name, scope)
path = _remove_spaces(expr) if name is None else name
path = make_legal_path(path)
desc = 'Holds all outputs processed by this driver.'
self._add_vartrees('all_case_outputs', path, desc)
desc = 'Holds just the DOE outputs.'
self._add_vartrees('DOE_outputs', path, desc)
def add_response(self, expr, name=None, scope=None):
"""We need to create our special variable trees."""
super(AdaptiveSampleDriver, self).add_response(expr, name, scope)
path = _remove_spaces(expr) if name is None else name
path = make_legal_path(path)
desc = 'Holds all outputs processed by this driver.'
self._add_vartrees('all_case_outputs', path, desc)
desc = 'Holds just the DOE outputs.'
self._add_vartrees('DOE_outputs', path, desc)
def remove_response(self, expr):
"""Removes the specified response expression. Spaces within
the expression are ignored.
"""
super(HasVarTreeResponses, self).remove_response(expr)
path = make_legal_path(_remove_spaces(expr))
obj = self.parent
names = ["case_outputs"] + path.split(".")
for name in names[:-1]:
obj = obj.get(name)
name = names[-1]
obj.remove_trait(name)
def remove_response(self, expr):
"""Removes the specified response expression. Spaces within
the expression are ignored.
"""
super(HasVarTreeResponses, self).remove_response(expr)
path = make_legal_path(_remove_spaces(expr))
obj = self.parent
names = ['case_outputs'] + path.split('.')
for name in names[:-1]:
obj = obj.get(name)
name = names[-1]
obj.remove_trait(name)
def remove_parameter(self, name):
"""Removes the parameter with the given name."""
super(HasVarTreeParameters, self).remove_parameter(name)
if isinstance(name, basestring):
path = name
else:
path = name[0]
path = make_legal_path(name)
obj = self.parent
names = ['case_inputs'] + path.split('.')
for name in names[:-1]:
obj = obj.get(name)
name = names[-1]
obj.remove_trait(name)
def set_inputs(self, generator):
""" Set case inputs from generator values. """
inputs = array([vals for vals in generator])
start = 0
for path, param in self.get_parameters().items():
size = param.size
if size == 1:
values = inputs[:, start]
else:
end = start + size
values = inputs[:, start:end].reshape((-1,)+param.shape)
start += size
if isinstance(path, tuple):
path = path[0] # Use first target of ParameterGroup.
path = make_legal_path(path)
self.set('case_inputs.'+path, list(values))
def set_inputs(self, generator):
""" Set case inputs from generator values. """
inputs = array([vals for vals in generator])
start = 0
for path, param in self.get_parameters().items():
size = param.size
if size == 1:
values = inputs[:, start]
else:
end = start + size
values = inputs[:, start:end].reshape((-1, ) + param.shape)
start += size
if isinstance(path, tuple):
path = path[0] # Use first target of ParameterGroup.
path = make_legal_path(path)
self.set('case_inputs.' + path, list(values))
def remove_parameter(self, name):
"""Removes the parameter with the given name."""
super(HasVarTreeParameters, self).remove_parameter(name)
if isinstance(name, basestring):
path = name
else:
path = name[0]
path = make_legal_path(path)
obj = self.parent
names = ['case_inputs'] + path.split('.')
for name in names[:-1]:
obj = obj.get(name)
name = names[-1]
obj.remove_trait(name)
def add_response(self, expr, name=None, scope=None):
"""Adds a response to the driver."""
super(HasVarTreeResponses, self).add_response(expr, name, scope)
path = _remove_spaces(expr) if name is None else name
path = make_legal_path(path)
obj = self.parent
names = ['case_outputs'] + path.split('.')
for name in names[:-1]:
if obj.get_trait(name):
val = obj.get(name)
else:
val = VariableTree()
obj.add_trait(name, VarTree(val, iotype='out'))
obj = val
name = names[-1]
obj.add_trait(name, List(iotype='out'))
def add_response(self, expr, name=None, scope=None):
"""Adds a response to the driver."""
super(HasVarTreeResponses, self).add_response(expr, name, scope)
path = _remove_spaces(expr) if name is None else name
path = make_legal_path(path)
obj = self.parent
names = ["case_outputs"] + path.split(".")
for name in names[:-1]:
if obj.get_trait(name):
val = obj.get(name)
else:
val = VariableTree()
obj.add_trait(name, VarTree(val, iotype="out"))
obj = val
name = names[-1]
obj.add_trait(name, List(iotype="out"))
def add_parameter(self, target, low=None, high=None,
scaler=None, adder=None, start=None,
fd_step=None, name=None, scope=None):
"""We need to create our special variable trees."""
super(AdaptiveSampleDriver, self).add_parameter(
target, low, high, scaler, adder, start, fd_step, name, scope)
# Parameter gets [0] as an initial case, just for connectability.
if name is not None:
target = name
elif isinstance(target, tuple):
target = target[0] # Use first target of ParameterGroup.
path = make_legal_path(target)
desc = 'Input for adding new points during parent iteration.'
self._add_vartrees('adaptive_inputs', path, desc)
desc = 'Holds all inputs processed by this driver.'
self._add_vartrees('all_case_inputs', path, desc)
desc = 'Holds just the DOE inputs.'
self._add_vartrees('DOE_inputs', path, desc)
def init_responses(self, length):
"""Initializes response storage in the driver."""
nans = [float('NaN')] * length
for path in self._responses:
path = make_legal_path(path)
self.parent.set('case_outputs.' + path, list(nans), force=True)
def execute(self):
""" Run each parameter set. """
color = self._color[self.mpi.rank]
if color == MPI.UNDEFINED or self.mpi.comm == MPI.COMM_NULL:
return
# Prepare parameters and responses.
case_paths = {}
inputs = []
values = []
for path in self.get_parameters():
if isinstance(path, tuple):
for target in path:
inputs.append(target)
path = path[0]
else:
inputs.append(path)
val = self.case_inputs.get(make_legal_path(path))
values.append(val)
if not inputs:
return
length = len(values[0])
for path in self.get_responses():
case_paths[path] = make_legal_path(path)
sizes, offsets = evenly_distrib_idxs(self._num_parallel_subs, length)
start = offsets[color]
end = start + sizes[color]
self.init_responses(length)
# Run each parameter set.
for i in range(start, end):
# Set inputs.
for j, path in enumerate(inputs):
self.set_parameter_by_name(path, values[j][i])
# Run workflow.
with MPIContext():
self.run_iteration()
# Get outputs.
for path in self.get_responses():
cpath = case_paths[path]
self.case_outputs.get(cpath)[i] = self.parent.get(path)
if self._num_parallel_subs > 1:
# Now, collect the results back from all parallel processes
for path in self.get_responses():
path = case_paths[path]
vals = self.case_outputs.get(path)
if self._resp_comm != MPI.COMM_NULL:
allvals = self._resp_comm.gather(vals, root=0)
if self._resp_comm.rank == 0:
for i in range(self._num_parallel_subs):
vals[offsets[i]:offsets[i] +
sizes[i]] = allvals[i][offsets[i]:offsets[i] +
sizes[i]]
junk = self.mpi.comm.bcast(vals, root=0)
else:
vals = self.mpi.comm.bcast(None, root=0)
else:
vals = self.mpi.comm.bcast(vals, root=0)
self.case_outputs.set(path, vals)
def init_responses(self, length):
"""Initializes response storage in the driver."""
nan = float('NaN')
for path in self._responses:
self.parent.case_outputs.set(make_legal_path(path), [nan] * length)
def init_responses(self, length):
"""Initializes response storage in the driver."""
nans = [float('NaN')] * length
for path in self._responses:
path = make_legal_path(path)
self.parent.set('case_outputs.'+path, list(nans), force=True)
def _setup(self):
""" Setup to begin new run. """
# if params have changed we need to setup systems again
if self.workflow._system is None:
obj = self
while obj.parent is not None:
obj = obj.parent
obj._setup()
if not self.sequential:
# Save model to egg.
# Must do this before creating any locks or queues.
self._replicants += 1
version = 'replicant.%d' % (self._replicants)
# If only local host will be used, we can skip determining
# distributions required by the egg.
allocators = RAM.list_allocators()
need_reqs = False
if not self.ignore_egg_requirements:
for allocator in allocators:
if not isinstance(allocator, LocalAllocator):
need_reqs = True
break
# Replicate and mutate model to run our workflow once.
# Originally this was done in-place, but that 'invalidated'
# various workflow quantities.
replicant = self.parent.copy()
workflow = replicant.get(self.name + '.workflow')
driver = replicant.add('driver', Driver())
workflow.parent = driver
workflow.scope = None
replicant.driver.workflow = workflow
egg_info = replicant.save_to_egg(self.name,
version,
need_requirements=need_reqs)
replicant = workflow = driver = None # Release objects.
gc.collect() # Collect/compact before possible fork.
self._egg_file = egg_info[0]
self._egg_required_distributions = egg_info[1]
self._egg_orphan_modules = [name for name, path in egg_info[2]]
inp_paths = []
inp_values = []
for path, param in self.get_parameters().items():
if isinstance(path, tuple):
path = path[0] # Use first target of ParameterGroup.
path = make_legal_path(path)
value = self.get('case_inputs.' + path)
for target in param.targets:
inp_paths.append(target)
inp_values.append(value)
outputs = self.get_responses().keys()
extra_outputs = self.workflow._rec_outputs
length = len(inp_values[0]) if inp_values else 0
cases = []
for i in range(length):
inputs = []
for j in range(len(inp_paths)):
inputs.append((inp_paths[j], inp_values[j][i]))
cases.append(
_Case(i,
inputs,
outputs,
extra_outputs,
parent_uuid=self._case_uuid))
self.init_responses(length)
self._iter = iter(cases)
self._abort_exc = None
def _setup(self):
""" Setup to begin new run. """
if not self.sequential:
# Save model to egg.
# Must do this before creating any locks or queues.
self._replicants += 1
version = 'replicant.%d' % (self._replicants)
# If only local host will be used, we can skip determining
# distributions required by the egg.
allocators = RAM.list_allocators()
need_reqs = False
if not self.ignore_egg_requirements:
for allocator in allocators:
if not isinstance(allocator, LocalAllocator):
need_reqs = True
break
# Replicate and mutate model to run our workflow once.
# Originally this was done in-place, but that 'invalidated'
# various workflow quantities.
replicant = self.parent.copy()
workflow = replicant.get(self.name+'.workflow')
driver = replicant.add('driver', Driver())
workflow.parent = driver
workflow.scope = None
replicant.driver.workflow = workflow
egg_info = replicant.save_to_egg(self.name, version,
need_requirements=need_reqs)
replicant = workflow = driver = None # Release objects.
gc.collect() # Collect/compact before possible fork.
self._egg_file = egg_info[0]
self._egg_required_distributions = egg_info[1]
self._egg_orphan_modules = [name for name, path in egg_info[2]]
inp_paths = []
inp_values = []
for path, param in self.get_parameters().items():
if isinstance(path, tuple):
path = path[0] # Use first target of ParameterGroup.
path = make_legal_path(path)
value = self.get('case_inputs.'+path)
for target in param.targets:
inp_paths.append(target)
inp_values.append(value)
outputs = self.get_responses().keys()
extra_outputs = self.workflow._rec_outputs
length = len(inp_values[0]) if inp_values else 0
cases = []
for i in range(length):
inputs = []
for j in range(len(inp_paths)):
inputs.append((inp_paths[j], inp_values[j][i]))
cases.append(_Case(i, inputs, outputs, extra_outputs,
parent_uuid=self._case_uuid))
self.init_responses(length)
self._iter = iter(cases)
self._abort_exc = None
def execute(self):
""" Run each parameter set. """
color = self._color[self.mpi.rank]
if color == MPI.UNDEFINED or self.mpi.comm == MPI.COMM_NULL:
return
# Prepare parameters and responses.
case_paths = {}
inputs = []
values = []
for path in self.get_parameters():
if isinstance(path, tuple):
for target in path:
inputs.append(target)
path = path[0]
else:
inputs.append(path)
val = self.case_inputs.get(make_legal_path(path))
values.append(val)
if not inputs:
return
length = len(values[0])
for path in self.get_responses():
case_paths[path] = make_legal_path(path)
sizes, offsets = evenly_distrib_idxs(self._num_parallel_subs,
length)
start = offsets[color]
end = start + sizes[color]
self.init_responses(length)
# Run each parameter set.
for i in range(start, end):
# Set inputs.
for j, path in enumerate(inputs):
self.set_parameter_by_name(path, values[j][i])
# Run workflow.
with MPIContext():
self.run_iteration()
# Get outputs.
for path in self.get_responses():
cpath = case_paths[path]
self.case_outputs.get(cpath)[i] = self.parent.get(path)
if self._num_parallel_subs > 1:
# Now, collect the results back from all parallel processes
for path in self.get_responses():
path = case_paths[path]
vals = self.case_outputs.get(path)
if self._resp_comm != MPI.COMM_NULL:
allvals = self._resp_comm.gather(vals, root=0)
if self._resp_comm.rank == 0:
for i in range(self._num_parallel_subs):
vals[offsets[i]:offsets[i]+sizes[i]] = allvals[i][offsets[i]:offsets[i]+sizes[i]]
junk = self.mpi.comm.bcast(vals, root=0)
else:
vals = self.mpi.comm.bcast(None, root=0)
else:
vals = self.mpi.comm.bcast(vals, root=0)
self.case_outputs.set(path, vals)
def init_responses(self, length):
"""Initializes response storage in the driver."""
nan = float("NaN")
for path in self._responses:
self.parent.case_outputs.set(make_legal_path(path), [nan] * length)
