def generate_design(self, uncertainties, size):
'''
This method provides an alternative implementation to the default
implementation provided by :class:`~sampler.Sampler`. This
version returns a full factorial design across the uncertainties.
:param uncertainties: a collection of
:class:`~uncertainties.ParameterUncertainty`
and :class:`~uncertainties.CategoricalUncertainty`
instances.
:param size: the resolution to use for :class:`~uncertainties.ParameterUncertainty`
instances. For instances of :class:`~uncertainties.CategoricalUncertainty`,
the categories are used.
:rtype: tuple with the designs as the first entry and the names
of the uncertainties as the second entry.
.. note:: The current implementation has a hard coded limit to the
number of designs possible. This is set to 50.000 designs.
If one want to go beyond this, set `self.max_designs` to
a higher value.
'''
def get_combos(branches):
return itertools.product(*branches)
categories = []
totalDesigns = 1
for uncertainty in uncertainties:
if type(uncertainty) == CategoricalUncertainty:
category = uncertainty.categories
else:
category = np.linspace(uncertainty.values[0],
uncertainty.values[1],
size)
if uncertainty.dist == 'integer':
category = np.round(category, 0)
category = set(category)
category = [int(entry) for entry in category]
category = sorted(category)
totalDesigns *= len(category)
categories.append(category)
if totalDesigns > self.max_designs:
warning("full factorial design results in %s designs"\
% totalDesigns)
raise Exception('too many designs')
else:
info("full factorial design results in %s designs"\
% totalDesigns)
designs = itertools.product(*categories)
return (designs, [uncertainty.name for uncertainty in uncertainties])
def callback(self, case, policy, name, result ):
'''
Method responsible for storing results. The implementation in this
class only keeps track of how many runs have been completed and
logging this.
:param case: the case to be stored
:param policy: the name of the policy being used
:param name: the name of the model being used
:param result: the result dict
'''
self.i+=1
debug(str(self.i)+" cases completed")
if self.i % 100 == 0:
info(str(self.i)+" cases completed")
def merge_results(results1, results2, downsample=None):
'''
convenience function for merging the return from
:meth:`~model.SimpleModelEnsemble.perform_experiments`.
The function merges results2 with results1. For the experiments,
it generates an empty array equal to the size of the sum of the
experiments. As dtype is uses the dtype from the experiments in results1.
The function assumes that the ordering of dtypes and names is identical in
both results.
A typical use case for this function is in combination with
:func:`~util.experiments_to_cases`. Using :func:`~util.experiments_to_cases`
one extracts the cases from a first set of experiments. One then
performs these cases on a different model or policy, and then one wants to
merge these new results with the old result for further analysis.
:param results1: first results to be merged
:param results2: second results to be merged
:param downsample: should be an integer, will be used in slicing the results
in order to avoid memory problems.
:return: the merged results
'''
#start of merging
old_exp, old_res = results1
new_exp, new_res = results2
#merge experiments
dtypes = old_exp.dtype
merged_exp = np.empty((old_exp.shape[0]+new_exp.shape[0],),dtype= dtypes)
merged_exp[0:old_exp.shape[0]] = old_exp
merged_exp[old_exp.shape[0]::] = new_exp
#only merge the results that are in both
keys = old_res.keys()
[keys.append(key) for key in new_res.keys()]
keys = set(keys)
info("intersection of keys: %s" % keys)
#merging results
merged_res = {}
for key in keys:
info("merge "+key)
old_value = old_res.get(key)
new_value = new_res.get(key)
i = old_value.shape[0]+new_value.shape[0]
j = old_value.shape[1]
slice = 1
if downsample:
j = int(math.ceil(j/downsample))
slice = downsample
merged_value = np.empty((i,j))
debug("merged shape: %s" % merged_value.shape)
merged_value[0:old_value.shape[0], :] = old_value[:, ::slice]
merged_value[old_value.shape[0]::, :] = new_value[:, ::slice]
merged_res[key] = merged_value
mr = (merged_exp, merged_res)
return mr
def __init__(self,
modelStructureInterfaces,
processes=None,
callback = None,
kwargs=None):
'''
:param modelStructureInterface: modelInterface class
:param processes: nr. of processes to spawn, if none, it is
set to equal the nr. of cores
:param callback: callback function for handling the output
:param kwargs: kwargs to be pased to :meth:`model_init`
'''
self._setup_queues()
self._taskqueue = Queue.Queue()
self._cache = {}
self._state = RUN
self._callback = callback
if processes is None:
try:
processes = cpu_count()
except NotImplementedError:
processes = 1
info("nr of processes is "+str(processes))
self.Process = LoggingProcess
self.logQueue = multiprocessing.Queue()
h = EMAlogging.NullHandler()
logging.getLogger(EMAlogging.LOGGER_NAME).addHandler(h)
# This thread will read from the subprocesses and write to the
# main log's handlers.
log_queue_reader = LogQueueReader(self.logQueue)
log_queue_reader.start()
self._pool = []
workingDirectories = []
debug('generating workers')
workerRoot = None
for i in range(processes):
debug('generating worker '+str(i))
workerName = 'PoolWorker'+str(i)
def ignore_function(path, names):
if path.find('.svn') != -1:
return names
else:
return []
#setup working directories for parallelEMA
for msi in modelStructureInterfaces:
if msi.workingDirectory != None:
if workerRoot == None:
workerRoot = os.path.dirname(os.path.abspath(modelStructureInterfaces[0].workingDirectory))
workingDirectory = os.path.join(workerRoot, workerName, msi.name)
workingDirectories.append(workingDirectory)
shutil.copytree(msi.workingDirectory,
workingDirectory,
ignore = ignore_function)
msi.set_working_directory(workingDirectory)
w = self.Process(
self.logQueue,
level = logging.getLogger(EMAlogging.LOGGER_NAME)\
.getEffectiveLevel(),
target=worker,
args=(self._inqueue,
self._outqueue,
modelStructureInterfaces,
kwargs)
)
self._pool.append(w)
w.name = w.name.replace('Process', workerName)
w.daemon = True
w.start()
debug(' worker '+str(i) + ' generated')
self._task_handler = threading.Thread(
target=CalculatorPool._handle_tasks,
args=(self._taskqueue,
self._quick_put,
self._outqueue,
self._pool,
self.logQueue)
)
self._task_handler.daemon = True
self._task_handler._state = RUN
self._task_handler.start()
self._result_handler = threading.Thread(
target=CalculatorPool._handle_results,
args=(self._outqueue, self._quick_get, self._cache)
)
self._result_handler.daemon = True
self._result_handler._state = RUN
self._result_handler.start()
self._terminate = Finalize(self,
self._terminate_pool,
args=(self._taskqueue,
self._inqueue,
self._outqueue,
self._pool,
self._task_handler,
self._result_handler,
self._cache,
workingDirectories,
),
exitpriority=15
)
EMAlogging.info("pool has been set up")