def updateIndexSet(self, newHierarchy): """ Remove entries present in self.indices that are no longer present in newHierarchy. Add entries present in newHierarchy that are not there in self.indices """ newIndices, newSamples = splitOneListIntoTwo(newHierarchy) # Collect the positions in self.indices that are not # present in newHierarchy # TODO there must be a simpler way list_of_positions_to_remove = [] for i in range(len(self.indices)): is_index_found = False for j in range(len(newIndices)): if newIndices[j] == self.indices[i].indexValue: is_index_found = True break if is_index_found is False: list_of_positions_to_remove.append(i) # Delete the positions in self.indices that are not # present in newHierarchy self._removeMonteCarloIndex(list_of_positions_to_remove) # Collect the entries in newHierarchy that are not # present in self.indices # TODO there must be a simpler way list_of_indices_to_add = [] for i in range(len(newIndices)): is_index_found = False for j in range(len(self.indices)): if newIndices[i] == self.indices[j].indexValue: is_index_found = True break if is_index_found is False: list_of_indices_to_add.append(newIndices[i]) # Pass the list of missing entries to the method that creates # these entries in self.indices self._addMonteCarloIndex(list_of_indices_to_add) # Rearrange self.indices to match the ordering of newHierarchy for i in range(len(newIndices)): for j in range(len(self.indices)): if newIndices[i] == self.indices[j].indexValue: break if i == j: break else: self.indices[i], self.indices[j] = self.indices[ j], self.indices[i]
def asynchronousPrepareBatches(self, newHierarchy): """ Method setting-up batches. If needed, the serialize method is called, to serialize Kratos Model and Kratos Parameters. Otherwise, serialized objects are passed to new batches, to avoid serializing multiple times. For each batch, an index constructor dictionary is built. This way, local estimators may be computed in parallel and then update global estimators. """ newIndices, newSamples = splitOneListIntoTwo(newHierarchy) if self.batchIndices is None: # iterationCounter = 0 # serialze Kratos objects into monteCarloIndex indeces instances # requirements: KratosSolverWrapper as SolverWrapper for lev in range(0, len(self.indices)): # serialize level l self.indices[lev].sampler.solvers[0].serialize() if lev > 0: # serialize level l-1, if needed self.indices[lev].sampler.solvers[1].serialize() # prepare batch indices and booleans self.batchIndices = [[] for _ in range(self.numberBatches)] self.batchesLaunched = [False for _ in range(self.numberBatches)] self.batchesExecutionFinished = [ False for _ in range(self.numberBatches) ] self.batchesAnalysisFinished = [ False for _ in range(self.numberBatches) ] self.batchesConvergenceFinished = [ False for _ in range(self.numberBatches) ] else: # iterationCounter > 0 self.asynchronousUpdateBatches() # create monteCarloIndex instances for batch in range(self.numberBatches): if self.batchesLaunched[batch] is False: index = 0 for i in newIndices: self.indexConstructorDictionary["indexValue"] = i self.batchIndices[batch].append( self.indexConstructor( **self.indexConstructorDictionary)) # save needed serializations stored in indices # build finer level self.asynchronousSerializeBatchIndices(batch, index, solver=0) # build coarser level if index > 0: self.asynchronousSerializeBatchIndices(batch, index, solver=1) index = index + 1
def asynchronousPrepareBatches(self, newHierarchy): newIndices, newSamples = splitOneListIntoTwo(newHierarchy) if self.batchIndices is None: # iterationCounter = 0 # serialze Kratos object sinto monteCarloIndex indeces instances # requirements: KratosSolverWrapper as SolverWrapper and concurrent_adaptive_refinement as refinement_strategy self.indices[0].sampler.solvers[0].serialize() # prepare batch indices and booleans self.batchIndices = [[] for _ in range(self.numberBatches)] self.batchesLaunched = [False for _ in range(self.numberBatches)] self.batchesExecutionFinished = [ False for _ in range(self.numberBatches) ] self.batchesAnalysisFinished = [ False for _ in range(self.numberBatches) ] self.batchesConvergenceFinished = [ False for _ in range(self.numberBatches) ] else: # iterationCounter > 0 self.asynchronousUpdateBatches() # create monteCarloIndex instances for batch in range(self.numberBatches): if self.batchesLaunched[batch] is False: index = 0 for i in newIndices: self.indexConstructorDictionary['indexValue'] = i self.batchIndices[batch].append( self.indexConstructor( **self.indexConstructorDictionary)) # save needed serializations stored in indices # build finer level self.asynchronousSerializeBatchIndices(batch, index, solver=0) # build coarser level if (index > 0): self.asynchronousSerializeBatchIndices(batch, index, solver=1) index = index + 1
def updatePredictors(self, predictorCoordinates=None): """ For an entry of qoiPredictor, retrieve all index estimations for the corresponding entry of estimatorsForPredictor and pass to the update method of qoiPredictor. If self.isCostUpdated is True, then do the same for costPredictor as well """ # TODO - The implicit assumption here is that there is a one-to-one map # between levelwise estimations and model built upon that estimation. We # have not yet allowed for a mechanism similar to the assembler mechanism # but at the indexwise quantities. Should we construct such a mechanism? if not self.qoiPredictor: return # If nothing is specified, update all qoiPredictors if predictorCoordinates is None: predictorCoordinates = range(len(self.qoiPredictor)) # Extract current hierarchy from list of indices hierarchy = self.hierarchy() [indices, number_samples] = splitOneListIntoTwo(hierarchy) # Retrieve all index estimations corresponding to each entry of predictorCooridnates for i in predictorCoordinates: # Extract qoiEstimator coordinates and value arguements from estimatorsForPredictor qoi_estimator_coordinate = self.estimatorsForPredictor[i][0] qoi_estimator_value_arguements = self.estimatorsForPredictor[i][1] value_transform_function = self.estimatorsForPredictor[i][2] # Assemble this quantity across all indices and apply transformation estimations = self.indexEstimation( qoi_estimator_coordinate, qoi_estimator_value_arguements ) # TODO - qoiEstimator.value returns Var(Y_l)/N_l, whereas the model is fit # on Var(Y_l). The following if-else is a hack to multiply by the number of # samples . It is here temporarily until a better mechanism can be found. # Refer the to-do above. if qoi_estimator_value_arguements[-1] is False: estimationsForPredictors = [value_transform_function(i) for i in estimations] else: estimationsForPredictors = [ value_transform_function(index, number_samples[i]) for i, index in enumerate(estimations) ] # Extract only the indices for which all of the multi-index components are non-zero data = [] for j in range(len(indices)): index = indices[j] if all([index[i] > 0 for i in range(len(index))]): data.append([index, estimationsForPredictors[j]]) else: pass # Run update method of qoiPredictor on these index estimations self.qoiPredictor[i].update(data) # Retrieve all index cost estimations and pass them to costPredictor if self.isCostUpdated: estimationsForPredictors = self.indexCostEstimation(self.costEstimatorForPredictor) # Extract only the indices for which all of the multi-index components are non-zero data = [] for j in range(len(indices)): index = indices[j] if all([index[i] > 0 for i in range(len(index))]): data.append([index, estimationsForPredictors[j]]) else: pass data = mergeTwoListsIntoOne(indices, estimationsForPredictors) self.costPredictor.update(data)
def runXMC(self): """ Run an algorithm with generic structure. """ self.checkInitialisation(self) # Iteration Loop will start here flag = self.stoppingCriterion.flagStructure() self.iterationCounter = 0 # print("Beginning Iterations for tolerance ", self.tolerances(self.errorsForStoppingCriterion)) #TODO not very robust while not flag["stop"]: # TODO Mostly outdated. Must be thoroughly checked. newHierarchy, splittingParameter = self.optimalHierarchy() self.splitTolerance(splittingParameter) self.updateHierarchy(newHierarchy) # synchronization point needed to launch new tasks if convergence is false # put the synchronization point as in the end as possible # TODO: remove from here the synchronization point to more hidden places flag = self.stoppingCriterionFlag(self.iterationCounter) flag = get_value_from_remote(flag) # TODO Display selection is mostly guesswork here (very fragile) errors = get_value_from_remote( self.errorEstimation(self.errorsForStoppingCriterion) ) dErrors = " ".join(["{err:.3e}".format(err=float(error)) for error in errors]) dHierarchy = " ".join([str(i[1]) for i in self.hierarchy()]) dTol = "None" tols = self.tolerances(splittingParameter) if tols: dTol = " ".join(["{t:.3e}".format(t=tol) for tol in tols]) print( f"Iteration — {self.iterationCounter}", f"Tolerances — {dTol}", f"Errors — {dErrors}", f"Hierarchy — {dHierarchy}", sep="\t", ) if flag["updateTolerance"]: self.updateTolerance() if flag["updateIndexSpace"]: self.updateHierarchySpace() self.iterationCounter += 1 #### DATA DUMP ########## if self.isDataDumped is True: pathObject = pl.Path(self.outputFolderPath) pathObject.mkdir(parents=True, exist_ok=True) filename = ( self.outputFolderPath + "/iteration_" + str(self.iterationCounter) + ".pickle" ) output_file = open(filename, "wb") output_dict = {} hier = self.hierarchy() output_dict["predictedHierarchy"] = newHierarchy output_dict["hierarchy"] = hier if len(self.predictorsForHierarchy) != 0: qoip = self.predictor() costp = self.costPredictor() output_dict["biasParameters"] = qoip[0].parameters output_dict["varParameters"] = qoip[1].parameters output_dict["costParameters"] = costp.parameters output_dict["indexwiseBias"] = self.indexEstimation(0, [1, True, False]) errs = self.indexEstimation(0, [1, True, True]) levels, samples = splitOneListIntoTwo(hier) output_dict["indexwiseVar"] = [errs[i] * samples[i] for i in range(len(errs))] output_dict["indexwiseCost"] = self.indexCostEstimation([1, True, False]) hier = newHierarchy levels, samples = splitOneListIntoTwo(hier) costs = self.indexCostEstimation([1, True, False]) total_times = [sample * cost for sample, cost in zip(samples, costs)] output_dict["totalTime"] = sum(total_times) pickle.dump(output_dict, output_file) # TODO - Debug statement. Remove for PYCOMPSS tests displayEstimation = get_value_from_remote(self.estimation()) displayEstimation = " ".join(["{e:.3e}".format(e=est) for est in displayEstimation]) displayError = get_value_from_remote( self.errorEstimation(self.errorsForStoppingCriterion) ) displayError = " ".join(["{e:.3e}".format(e=error) for error in displayError]) displayCost = get_value_from_remote(self.indexCostEstimation([1, True, False])) displayCost = " ".join(["{c:.3e}".format(c=cost) for cost in displayCost]) print( f"Estimations — {displayEstimation}", f"Final errors — {displayError}", f"Levelwise mean costs — {displayCost}", sep="\n", )
def runXMC(self): """ Run an algorithm with generic structure, as described in the documentation. Other methods may be implemented in the future, for specific needs that do not fit even this generic algorithm. """ swDict = self.monteCarloSampler.indexConstructorDictionary[ "samplerInputDictionary"]["solverWrapperInputDictionary"] self.checkInitialisation(solverWrapperDictionary=swDict) # Iteration Loop will start here flag = self.stoppingCriterion.flagStructure() self.iterationCounter = 0 #print("Beginning Iterations for tolerance ", self.tolerances(self.errorsForStoppingCriterion)) #TODO not very robust while(not flag['stop']): # TODO Mostly outdated. Must be thoroughly checked. newHierarchy,splittingParameter = self.optimalHierarchy() self.splitTolerance(splittingParameter) self.updateHierarchy(newHierarchy) # synchronization point needed to launch new tasks if convergence is false # put the synchronization point as in the end as possible # TODO: remove from here the synchronization point to more hidden places flag = self.stoppingCriterionFlag(self.iterationCounter) flag = get_value_from_remote(flag) # TODO Display selection is mostly guesswork here (very fragile) errors = get_value_from_remote( self.errorEstimation(self.errorsForStoppingCriterion)) dErrors = ' '.join(['{err:.3e}'.format(err=float(error)) for error in errors]) dHierarchy = ' '.join([str(i[1]) for i in self.hierarchy()]) dTol = ' '.join(['{t:.3e}'.format(t=tol) for tol in self.tolerances(self.tolerancesForHierarchy)]) print(f'Iteration — {self.iterationCounter}', f'Tolerances — {dTol}', f'Errors — {dErrors}', f'Hierarchy — {dHierarchy}', sep='\t') if flag['updateTolerance']: self.updateTolerance() if flag['updateIndexSpace']: self.updateHierarchySpace() self.iterationCounter += 1 #### DATA DUMP ########## if self.isDataDumped is True: pathObject = pl.Path(self.outputFolderPath) pathObject.mkdir(parents=True,exist_ok=True) filename = self.outputFolderPath+'/iteration_'+str(self.iterationCounter)+'.pickle' output_file = open(filename,'wb') output_dict = {} hier = self.hierarchy() output_dict['predictedHierarchy']=newHierarchy output_dict['hierarchy']=hier if(len(self.predictorsForHierarchy)!=0): qoip = self.predictor() costp = self.costPredictor() output_dict['biasParameters']=qoip[0].parameters output_dict['varParameters']=qoip[1].parameters output_dict['costParameters']=costp.parameters output_dict['indexwiseBias']=self.indexEstimation(0,[1,True,False]) errs = self.indexEstimation(0,[1,True,True]) levels,samples = splitOneListIntoTwo(hier) output_dict['indexwiseVar']=[errs[i]*samples[i] for i in range(len(errs))] output_dict['indexwiseCost']=self.indexCostEstimation([1,True,False]) hier = newHierarchy levels,samples = splitOneListIntoTwo(hier) costs = self.indexCostEstimation([1,True,False]) total_times = [sample*cost for sample,cost in zip(samples,costs)] output_dict['totalTime']=sum(total_times) pickle.dump(output_dict,output_file) # TODO - Debug statement. Remove for PYCOMPSS tests displayEstimation = get_value_from_remote(self.estimation()) displayEstimation = ' '.join(['{e:.3e}'.format(e=est) for est in displayEstimation]) displayError = get_value_from_remote( self.errorEstimation(self.errorsForStoppingCriterion)) displayError = ' '.join(['{e:.3e}'.format(e=error) for error in displayError]) displayCost = get_value_from_remote(self.indexCostEstimation([1,True,False])) displayCost = ' '.join(['{c:.3e}'.format(c=cost) for cost in displayCost]) print(f'Estimations — {displayEstimation}', f'Final errors — {displayError}', f'Levelwise mean costs — {displayCost}', sep='\n')
def asynchronousPrepareBatches(self, newHierarchy): """ Method setting-up batches. If needed, the serialize method is called, to serialize Kratos Model and Kratos Parameters. Otherwise, serialized objects are passed to new batches, to avoid serializing multiple times. For each batch, an index constructor dictionary is built. This way, local estimators may be computed in parallel and then update global estimators. If random generator is using fixed samples, i.e. EventDatabase as RandomGeneratorWrapper, _eventCounter attribute is modified accordingly to the batch we are preparing. """ newIndices, newSamples = splitOneListIntoTwo(newHierarchy) if self.batchIndices is None: # iterationCounter = 0 # serialze Kratos objects into monteCarloIndex indeces instances # requirements: KratosSolverWrapper as SolverWrapper for lev in range(0, len(self.indices)): # serialize level l self.indices[lev].sampler.solvers[0].serialize() if lev > 0: # serialize level l-1, if needed self.indices[lev].sampler.solvers[1].serialize() # prepare batch indices and booleans self.batchIndices = [[] for _ in range(self.numberBatches)] self.batchesLaunched = [False for _ in range(self.numberBatches)] self.batchesExecutionFinished = [ False for _ in range(self.numberBatches) ] self.batchesAnalysisFinished = [ False for _ in range(self.numberBatches) ] self.batchesConvergenceFinished = [ False for _ in range(self.numberBatches) ] else: # iterationCounter > 0 self.asynchronousUpdateBatches() # create new clean monteCarloIndex instances for batch in range(self.numberBatches): if self.batchesLaunched[batch] is False: index = 0 for i in newIndices: # set index value self.indexConstructorDictionary["indexValue"] = i # initialize monteCarloIndex class of asynchronous batch "batch" self.batchIndices[batch].append( self.indexConstructor( **self.indexConstructorDictionary)) # save needed serializations stored in indices # build finer level self.asynchronousSerializeBatchIndices(batch, index, solver=0) # build coarser level if index > 0: self.asynchronousSerializeBatchIndices(batch, index, solver=1) # update eventCounter of EventDatabase random generator, if using EventDatabase as RandomGeneratorWrapper if self.indexConstructorDictionary["samplerInputDictionary"][ "randomGenerator"] == "xmc.randomGeneratorWrapper.EventDatabase": # we may have multiple monteCarloIndex objects, that is multiple levels for j in range(len(self.batchIndices[batch])): # set _eventCounter for having independent events for different levels self.batchIndices[batch][ j].sampler.randomGenerator._eventCounter = self.samplesCounter # update overall samples counter self.samplesCounter += newSamples[index] else: # update overall samples counter self.samplesCounter += newSamples[index] # update index counter index = index + 1
def runXMC(self): """ Run an algorithm with generic structure, as described in the documentation. Other methods may be implemented in the future, for specific needs that do not fit even this generic algorithm. """ self.checkInitialisation() # Iteration Loop will start here flag = self.stoppingCriterion.flagStructure() self.iterationCounter = 0 #print("Beginning Iterations for tolerance ", self.tolerances(self.errorsForStoppingCriterion)) #TODO not very robust while (not flag['stop']): # TODO Mostly outdated. Must be thoroughly checked. newHierarchy, splittingParameter = self.optimalHierarchy() self.splitTolerance(splittingParameter) self.updateHierarchy(newHierarchy) # synchronization point needed to launch new tasks if convergence is false # put the synchronization point as in the end as possible # TODO: remove from here the synchronization point to more hidden places flag = self.stoppingCriterionFlag(self.iterationCounter) # TODO Print selection is mostly guesswork here (very fragile) print("Iteration ", self.iterationCounter, "\tTolerance - ", [ '%.3e' % tol for tol in self.tolerances(self.tolerancesForHierarchy) ], "\tError - ", [ '%.3e' % err for err in get_value_from_remote( self.errorEstimation(self.errorsForStoppingCriterion)) ], "\tHierarchy - ", self.hierarchy()) if flag['updateTolerance']: self.updateTolerance() if flag['updateIndexSpace']: self.updateHierarchySpace() self.iterationCounter += 1 #### DATA DUMP ########## if self.isDataDumped is True: pathObject = pl.Path(self.outputFolderPath) pathObject.mkdir(parents=True, exist_ok=True) filename = self.outputFolderPath + "/iteration_" + str( self.iterationCounter) + ".pickle" output_file = open(filename, "wb") output_dict = {} hier = self.hierarchy() output_dict['predictedHierarchy'] = newHierarchy output_dict['hierarchy'] = hier if (len(self.predictorsForHierarchy) != 0): qoip = self.qoiPrediction() costp = self.costPrediction() output_dict['biasParameters'] = get_value_from_remote( qoip[0].parameters) output_dict['varParameters'] = get_value_from_remote( qoip[1].parameters) output_dict['costParameters'] = get_value_from_remote( costp.parameters) output_dict['indexwiseBias'] = get_value_from_remote( self.indexEstimation(0, [1, True, False])) errs = get_value_from_remote( self.indexEstimation(0, [1, True, True])) levels, samples = splitOneListIntoTwo(hier) output_dict['indexwiseVar'] = [ errs[i] * samples[i] for i in range(len(errs)) ] output_dict['indexwiseCost'] = get_value_from_remote( self.indexCostEstimation([1, True, False])) hier = newHierarchy levels, samples = splitOneListIntoTwo(hier) costs = get_value_from_remote( self.indexCostEstimation([1, True, False])) total_times = [ sample * cost for sample, cost in zip(samples, costs) ] output_dict['totalTime'] = sum(total_times) pickle.dump(output_dict, output_file) # TODO - Debug statement. Remove for PYCOMPSS tests print( "Estimation - ", get_value_from_remote(self.estimation()), "\nFinal Error - ", get_value_from_remote( self.errorEstimation(self.errorsForStoppingCriterion)), "\nLevelwise Mean Times - ", get_value_from_remote(self.indexCostEstimation([1, True, False])))