def updateIndexSet(self, newHierarchy):
"""
Remove entries present in self.indices that are no longer present
in newHierarchy. Add entries present in newHierarchy that are not
there in self.indices
"""
newIndices, newSamples = splitOneListIntoTwo(newHierarchy)
# Collect the positions in self.indices that are not
# present in newHierarchy
# TODO there must be a simpler way
list_of_positions_to_remove = []
for i in range(len(self.indices)):
is_index_found = False
for j in range(len(newIndices)):
if newIndices[j] == self.indices[i].indexValue:
is_index_found = True
break
if is_index_found is False:
list_of_positions_to_remove.append(i)
# Delete the positions in self.indices that are not
# present in newHierarchy
self._removeMonteCarloIndex(list_of_positions_to_remove)
# Collect the entries in newHierarchy that are not
# present in self.indices
# TODO there must be a simpler way
list_of_indices_to_add = []
for i in range(len(newIndices)):
is_index_found = False
for j in range(len(self.indices)):
if newIndices[i] == self.indices[j].indexValue:
is_index_found = True
break
if is_index_found is False:
list_of_indices_to_add.append(newIndices[i])
# Pass the list of missing entries to the method that creates
# these entries in self.indices
self._addMonteCarloIndex(list_of_indices_to_add)
# Rearrange self.indices to match the ordering of newHierarchy
for i in range(len(newIndices)):
for j in range(len(self.indices)):
if newIndices[i] == self.indices[j].indexValue:
break
if i == j:
break
else:
self.indices[i], self.indices[j] = self.indices[
j], self.indices[i]
def asynchronousPrepareBatches(self, newHierarchy):
"""
Method setting-up batches. If needed, the serialize method is called, to serialize Kratos Model and Kratos Parameters. Otherwise, serialized objects are passed to new batches, to avoid serializing multiple times.
For each batch, an index constructor dictionary is built. This way, local estimators may be computed in parallel and then update global estimators.
"""
newIndices, newSamples = splitOneListIntoTwo(newHierarchy)
if self.batchIndices is None: # iterationCounter = 0
# serialze Kratos objects into monteCarloIndex indeces instances
# requirements: KratosSolverWrapper as SolverWrapper
for lev in range(0, len(self.indices)):
# serialize level l
self.indices[lev].sampler.solvers[0].serialize()
if lev > 0:
# serialize level l-1, if needed
self.indices[lev].sampler.solvers[1].serialize()
# prepare batch indices and booleans
self.batchIndices = [[] for _ in range(self.numberBatches)]
self.batchesLaunched = [False for _ in range(self.numberBatches)]
self.batchesExecutionFinished = [
False for _ in range(self.numberBatches)
]
self.batchesAnalysisFinished = [
False for _ in range(self.numberBatches)
]
self.batchesConvergenceFinished = [
False for _ in range(self.numberBatches)
]
else: # iterationCounter > 0
self.asynchronousUpdateBatches()
# create monteCarloIndex instances
for batch in range(self.numberBatches):
if self.batchesLaunched[batch] is False:
index = 0
for i in newIndices:
self.indexConstructorDictionary["indexValue"] = i
self.batchIndices[batch].append(
self.indexConstructor(
**self.indexConstructorDictionary))
# save needed serializations stored in indices
# build finer level
self.asynchronousSerializeBatchIndices(batch,
index,
solver=0)
# build coarser level
if index > 0:
self.asynchronousSerializeBatchIndices(batch,
index,
solver=1)
index = index + 1
def asynchronousPrepareBatches(self, newHierarchy):
newIndices, newSamples = splitOneListIntoTwo(newHierarchy)
if self.batchIndices is None: # iterationCounter = 0
# serialze Kratos object sinto monteCarloIndex indeces instances
# requirements: KratosSolverWrapper as SolverWrapper and concurrent_adaptive_refinement as refinement_strategy
self.indices[0].sampler.solvers[0].serialize()
# prepare batch indices and booleans
self.batchIndices = [[] for _ in range(self.numberBatches)]
self.batchesLaunched = [False for _ in range(self.numberBatches)]
self.batchesExecutionFinished = [
False for _ in range(self.numberBatches)
]
self.batchesAnalysisFinished = [
False for _ in range(self.numberBatches)
]
self.batchesConvergenceFinished = [
False for _ in range(self.numberBatches)
]
else: # iterationCounter > 0
self.asynchronousUpdateBatches()
# create monteCarloIndex instances
for batch in range(self.numberBatches):
if self.batchesLaunched[batch] is False:
index = 0
for i in newIndices:
self.indexConstructorDictionary['indexValue'] = i
self.batchIndices[batch].append(
self.indexConstructor(
**self.indexConstructorDictionary))
# save needed serializations stored in indices
# build finer level
self.asynchronousSerializeBatchIndices(batch,
index,
solver=0)
# build coarser level
if (index > 0):
self.asynchronousSerializeBatchIndices(batch,
index,
solver=1)
index = index + 1
def updatePredictors(self, predictorCoordinates=None):
"""
For an entry of qoiPredictor, retrieve all index estimations for the corresponding
entry of estimatorsForPredictor and pass to the update method of qoiPredictor.
If self.isCostUpdated is True, then do the same for costPredictor as well
"""
# TODO - The implicit assumption here is that there is a one-to-one map
# between levelwise estimations and model built upon that estimation. We
# have not yet allowed for a mechanism similar to the assembler mechanism
# but at the indexwise quantities. Should we construct such a mechanism?
if not self.qoiPredictor:
return
# If nothing is specified, update all qoiPredictors
if predictorCoordinates is None:
predictorCoordinates = range(len(self.qoiPredictor))
# Extract current hierarchy from list of indices
hierarchy = self.hierarchy()
[indices, number_samples] = splitOneListIntoTwo(hierarchy)
# Retrieve all index estimations corresponding to each entry of predictorCooridnates
for i in predictorCoordinates:
# Extract qoiEstimator coordinates and value arguements from estimatorsForPredictor
qoi_estimator_coordinate = self.estimatorsForPredictor[i][0]
qoi_estimator_value_arguements = self.estimatorsForPredictor[i][1]
value_transform_function = self.estimatorsForPredictor[i][2]
# Assemble this quantity across all indices and apply transformation
estimations = self.indexEstimation(
qoi_estimator_coordinate, qoi_estimator_value_arguements
)
# TODO - qoiEstimator.value returns Var(Y_l)/N_l, whereas the model is fit
# on Var(Y_l). The following if-else is a hack to multiply by the number of
# samples . It is here temporarily until a better mechanism can be found.
# Refer the to-do above.
if qoi_estimator_value_arguements[-1] is False:
estimationsForPredictors = [value_transform_function(i) for i in estimations]
else:
estimationsForPredictors = [
value_transform_function(index, number_samples[i])
for i, index in enumerate(estimations)
]
# Extract only the indices for which all of the multi-index components are non-zero
data = []
for j in range(len(indices)):
index = indices[j]
if all([index[i] > 0 for i in range(len(index))]):
data.append([index, estimationsForPredictors[j]])
else:
pass
# Run update method of qoiPredictor on these index estimations
self.qoiPredictor[i].update(data)
# Retrieve all index cost estimations and pass them to costPredictor
if self.isCostUpdated:
estimationsForPredictors = self.indexCostEstimation(self.costEstimatorForPredictor)
# Extract only the indices for which all of the multi-index components are non-zero
data = []
for j in range(len(indices)):
index = indices[j]
if all([index[i] > 0 for i in range(len(index))]):
data.append([index, estimationsForPredictors[j]])
else:
pass
data = mergeTwoListsIntoOne(indices, estimationsForPredictors)
self.costPredictor.update(data)
def runXMC(self):
"""
Run an algorithm with generic structure.
"""
self.checkInitialisation(self)
# Iteration Loop will start here
flag = self.stoppingCriterion.flagStructure()
self.iterationCounter = 0
# print("Beginning Iterations for tolerance ", self.tolerances(self.errorsForStoppingCriterion)) #TODO not very robust
while not flag["stop"]:
# TODO Mostly outdated. Must be thoroughly checked.
newHierarchy, splittingParameter = self.optimalHierarchy()
self.splitTolerance(splittingParameter)
self.updateHierarchy(newHierarchy)
# synchronization point needed to launch new tasks if convergence is false
# put the synchronization point as in the end as possible
# TODO: remove from here the synchronization point to more hidden places
flag = self.stoppingCriterionFlag(self.iterationCounter)
flag = get_value_from_remote(flag)
# TODO Display selection is mostly guesswork here (very fragile)
errors = get_value_from_remote(
self.errorEstimation(self.errorsForStoppingCriterion)
)
dErrors = " ".join(["{err:.3e}".format(err=float(error)) for error in errors])
dHierarchy = " ".join([str(i[1]) for i in self.hierarchy()])
dTol = "None"
tols = self.tolerances(splittingParameter)
if tols:
dTol = " ".join(["{t:.3e}".format(t=tol) for tol in tols])
print(
f"Iteration — {self.iterationCounter}",
f"Tolerances — {dTol}",
f"Errors — {dErrors}",
f"Hierarchy — {dHierarchy}",
sep="\t",
)
if flag["updateTolerance"]:
self.updateTolerance()
if flag["updateIndexSpace"]:
self.updateHierarchySpace()
self.iterationCounter += 1
#### DATA DUMP ##########
if self.isDataDumped is True:
pathObject = pl.Path(self.outputFolderPath)
pathObject.mkdir(parents=True, exist_ok=True)
filename = (
self.outputFolderPath
+ "/iteration_"
+ str(self.iterationCounter)
+ ".pickle"
)
output_file = open(filename, "wb")
output_dict = {}
hier = self.hierarchy()
output_dict["predictedHierarchy"] = newHierarchy
output_dict["hierarchy"] = hier
if len(self.predictorsForHierarchy) != 0:
qoip = self.predictor()
costp = self.costPredictor()
output_dict["biasParameters"] = qoip[0].parameters
output_dict["varParameters"] = qoip[1].parameters
output_dict["costParameters"] = costp.parameters
output_dict["indexwiseBias"] = self.indexEstimation(0, [1, True, False])
errs = self.indexEstimation(0, [1, True, True])
levels, samples = splitOneListIntoTwo(hier)
output_dict["indexwiseVar"] = [errs[i] * samples[i] for i in range(len(errs))]
output_dict["indexwiseCost"] = self.indexCostEstimation([1, True, False])
hier = newHierarchy
levels, samples = splitOneListIntoTwo(hier)
costs = self.indexCostEstimation([1, True, False])
total_times = [sample * cost for sample, cost in zip(samples, costs)]
output_dict["totalTime"] = sum(total_times)
pickle.dump(output_dict, output_file)
# TODO - Debug statement. Remove for PYCOMPSS tests
displayEstimation = get_value_from_remote(self.estimation())
displayEstimation = " ".join(["{e:.3e}".format(e=est) for est in displayEstimation])
displayError = get_value_from_remote(
self.errorEstimation(self.errorsForStoppingCriterion)
)
displayError = " ".join(["{e:.3e}".format(e=error) for error in displayError])
displayCost = get_value_from_remote(self.indexCostEstimation([1, True, False]))
displayCost = " ".join(["{c:.3e}".format(c=cost) for cost in displayCost])
print(
f"Estimations — {displayEstimation}",
f"Final errors — {displayError}",
f"Levelwise mean costs — {displayCost}",
sep="\n",
)
def runXMC(self):
"""
Run an algorithm with generic structure, as described in the documentation.
Other methods may be implemented in the future, for specific needs that do
not fit even this generic algorithm.
"""
swDict = self.monteCarloSampler.indexConstructorDictionary[
"samplerInputDictionary"]["solverWrapperInputDictionary"]
self.checkInitialisation(solverWrapperDictionary=swDict)
# Iteration Loop will start here
flag = self.stoppingCriterion.flagStructure()
self.iterationCounter = 0
#print("Beginning Iterations for tolerance ", self.tolerances(self.errorsForStoppingCriterion)) #TODO not very robust
while(not flag['stop']):
# TODO Mostly outdated. Must be thoroughly checked.
newHierarchy,splittingParameter = self.optimalHierarchy()
self.splitTolerance(splittingParameter)
self.updateHierarchy(newHierarchy)
# synchronization point needed to launch new tasks if convergence is false
# put the synchronization point as in the end as possible
# TODO: remove from here the synchronization point to more hidden places
flag = self.stoppingCriterionFlag(self.iterationCounter)
flag = get_value_from_remote(flag)
# TODO Display selection is mostly guesswork here (very fragile)
errors = get_value_from_remote(
self.errorEstimation(self.errorsForStoppingCriterion))
dErrors = ' '.join(['{err:.3e}'.format(err=float(error)) for error in errors])
dHierarchy = ' '.join([str(i[1]) for i in self.hierarchy()])
dTol = ' '.join(['{t:.3e}'.format(t=tol)
for tol in self.tolerances(self.tolerancesForHierarchy)])
print(f'Iteration — {self.iterationCounter}', f'Tolerances — {dTol}',
f'Errors — {dErrors}', f'Hierarchy — {dHierarchy}', sep='\t')
if flag['updateTolerance']:
self.updateTolerance()
if flag['updateIndexSpace']:
self.updateHierarchySpace()
self.iterationCounter += 1
#### DATA DUMP ##########
if self.isDataDumped is True:
pathObject = pl.Path(self.outputFolderPath)
pathObject.mkdir(parents=True,exist_ok=True)
filename = self.outputFolderPath+'/iteration_'+str(self.iterationCounter)+'.pickle'
output_file = open(filename,'wb')
output_dict = {}
hier = self.hierarchy()
output_dict['predictedHierarchy']=newHierarchy
output_dict['hierarchy']=hier
if(len(self.predictorsForHierarchy)!=0):
qoip = self.predictor()
costp = self.costPredictor()
output_dict['biasParameters']=qoip[0].parameters
output_dict['varParameters']=qoip[1].parameters
output_dict['costParameters']=costp.parameters
output_dict['indexwiseBias']=self.indexEstimation(0,[1,True,False])
errs = self.indexEstimation(0,[1,True,True])
levels,samples = splitOneListIntoTwo(hier)
output_dict['indexwiseVar']=[errs[i]*samples[i] for i in range(len(errs))]
output_dict['indexwiseCost']=self.indexCostEstimation([1,True,False])
hier = newHierarchy
levels,samples = splitOneListIntoTwo(hier)
costs = self.indexCostEstimation([1,True,False])
total_times = [sample*cost for sample,cost in zip(samples,costs)]
output_dict['totalTime']=sum(total_times)
pickle.dump(output_dict,output_file)
# TODO - Debug statement. Remove for PYCOMPSS tests
displayEstimation = get_value_from_remote(self.estimation())
displayEstimation = ' '.join(['{e:.3e}'.format(e=est) for est in displayEstimation])
displayError = get_value_from_remote(
self.errorEstimation(self.errorsForStoppingCriterion))
displayError = ' '.join(['{e:.3e}'.format(e=error) for error in displayError])
displayCost = get_value_from_remote(self.indexCostEstimation([1,True,False]))
displayCost = ' '.join(['{c:.3e}'.format(c=cost) for cost in displayCost])
print(f'Estimations — {displayEstimation}', f'Final errors — {displayError}',
f'Levelwise mean costs — {displayCost}', sep='\n')
def asynchronousPrepareBatches(self, newHierarchy):
"""
Method setting-up batches.
If needed, the serialize method is called, to serialize Kratos Model and Kratos Parameters. Otherwise, serialized objects are passed to new batches, to avoid serializing multiple times.
For each batch, an index constructor dictionary is built. This way, local estimators may be computed in parallel and then update global estimators.
If random generator is using fixed samples, i.e. EventDatabase as RandomGeneratorWrapper, _eventCounter attribute is modified accordingly to the batch we are preparing.
"""
newIndices, newSamples = splitOneListIntoTwo(newHierarchy)
if self.batchIndices is None: # iterationCounter = 0
# serialze Kratos objects into monteCarloIndex indeces instances
# requirements: KratosSolverWrapper as SolverWrapper
for lev in range(0, len(self.indices)):
# serialize level l
self.indices[lev].sampler.solvers[0].serialize()
if lev > 0:
# serialize level l-1, if needed
self.indices[lev].sampler.solvers[1].serialize()
# prepare batch indices and booleans
self.batchIndices = [[] for _ in range(self.numberBatches)]
self.batchesLaunched = [False for _ in range(self.numberBatches)]
self.batchesExecutionFinished = [
False for _ in range(self.numberBatches)
]
self.batchesAnalysisFinished = [
False for _ in range(self.numberBatches)
]
self.batchesConvergenceFinished = [
False for _ in range(self.numberBatches)
]
else: # iterationCounter > 0
self.asynchronousUpdateBatches()
# create new clean monteCarloIndex instances
for batch in range(self.numberBatches):
if self.batchesLaunched[batch] is False:
index = 0
for i in newIndices:
# set index value
self.indexConstructorDictionary["indexValue"] = i
# initialize monteCarloIndex class of asynchronous batch "batch"
self.batchIndices[batch].append(
self.indexConstructor(
**self.indexConstructorDictionary))
# save needed serializations stored in indices
# build finer level
self.asynchronousSerializeBatchIndices(batch,
index,
solver=0)
# build coarser level
if index > 0:
self.asynchronousSerializeBatchIndices(batch,
index,
solver=1)
# update eventCounter of EventDatabase random generator, if using EventDatabase as RandomGeneratorWrapper
if self.indexConstructorDictionary["samplerInputDictionary"][
"randomGenerator"] == "xmc.randomGeneratorWrapper.EventDatabase":
# we may have multiple monteCarloIndex objects, that is multiple levels
for j in range(len(self.batchIndices[batch])):
# set _eventCounter for having independent events for different levels
self.batchIndices[batch][
j].sampler.randomGenerator._eventCounter = self.samplesCounter
# update overall samples counter
self.samplesCounter += newSamples[index]
else:
# update overall samples counter
self.samplesCounter += newSamples[index]
# update index counter
index = index + 1
def runXMC(self):
"""
Run an algorithm with generic structure, as described in the documentation.
Other methods may be implemented in the future, for specific needs that do
not fit even this generic algorithm.
"""
self.checkInitialisation()
# Iteration Loop will start here
flag = self.stoppingCriterion.flagStructure()
self.iterationCounter = 0
#print("Beginning Iterations for tolerance ", self.tolerances(self.errorsForStoppingCriterion)) #TODO not very robust
while (not flag['stop']):
# TODO Mostly outdated. Must be thoroughly checked.
newHierarchy, splittingParameter = self.optimalHierarchy()
self.splitTolerance(splittingParameter)
self.updateHierarchy(newHierarchy)
# synchronization point needed to launch new tasks if convergence is false
# put the synchronization point as in the end as possible
# TODO: remove from here the synchronization point to more hidden places
flag = self.stoppingCriterionFlag(self.iterationCounter)
# TODO Print selection is mostly guesswork here (very fragile)
print("Iteration ", self.iterationCounter, "\tTolerance - ", [
'%.3e' % tol
for tol in self.tolerances(self.tolerancesForHierarchy)
], "\tError - ", [
'%.3e' % err for err in get_value_from_remote(
self.errorEstimation(self.errorsForStoppingCriterion))
], "\tHierarchy - ", self.hierarchy())
if flag['updateTolerance']:
self.updateTolerance()
if flag['updateIndexSpace']:
self.updateHierarchySpace()
self.iterationCounter += 1
#### DATA DUMP ##########
if self.isDataDumped is True:
pathObject = pl.Path(self.outputFolderPath)
pathObject.mkdir(parents=True, exist_ok=True)
filename = self.outputFolderPath + "/iteration_" + str(
self.iterationCounter) + ".pickle"
output_file = open(filename, "wb")
output_dict = {}
hier = self.hierarchy()
output_dict['predictedHierarchy'] = newHierarchy
output_dict['hierarchy'] = hier
if (len(self.predictorsForHierarchy) != 0):
qoip = self.qoiPrediction()
costp = self.costPrediction()
output_dict['biasParameters'] = get_value_from_remote(
qoip[0].parameters)
output_dict['varParameters'] = get_value_from_remote(
qoip[1].parameters)
output_dict['costParameters'] = get_value_from_remote(
costp.parameters)
output_dict['indexwiseBias'] = get_value_from_remote(
self.indexEstimation(0, [1, True, False]))
errs = get_value_from_remote(
self.indexEstimation(0, [1, True, True]))
levels, samples = splitOneListIntoTwo(hier)
output_dict['indexwiseVar'] = [
errs[i] * samples[i] for i in range(len(errs))
]
output_dict['indexwiseCost'] = get_value_from_remote(
self.indexCostEstimation([1, True, False]))
hier = newHierarchy
levels, samples = splitOneListIntoTwo(hier)
costs = get_value_from_remote(
self.indexCostEstimation([1, True, False]))
total_times = [
sample * cost for sample, cost in zip(samples, costs)
]
output_dict['totalTime'] = sum(total_times)
pickle.dump(output_dict, output_file)
# TODO - Debug statement. Remove for PYCOMPSS tests
print(
"Estimation - ", get_value_from_remote(self.estimation()),
"\nFinal Error - ",
get_value_from_remote(
self.errorEstimation(self.errorsForStoppingCriterion)),
"\nLevelwise Mean Times - ",
get_value_from_remote(self.indexCostEstimation([1, True, False])))
