def __init__(self):
"""
Default Constructor that will initialize member variables with reasonable
defaults or empty lists/dictionaries where applicable.
@ In, None
@ Out, None
"""
AdaptiveSampler.__init__(self)
self._initialValues = {
} # dict stores the user provided initial values, i.e. {var: val}
self._updateValues = {
} # dict stores input variables values for the current MCMC iteration, i.e. {var:val}
self._proposal = {
} # dict stores the proposal distributions for input variables, i.e. {var:dist}
self._priorFuns = {
} # dict stores the prior functions for input variables, i.e. {var:fun}
self._burnIn = 0 # integers indicate how many samples will be discarded
self._likelihood = None # stores the output from the likelihood
self._logLikelihood = False # True if the user provided likelihood is in log format
self._availProposal = {
'normal': Distributions.Normal(0.0, 1.0),
'uniform': Distributions.Uniform(-1.0, 1.0)
} # available proposal distributions
self._acceptDist = Distributions.Uniform(
0.0, 1.0) # uniform distribution for accept/rejection purpose
self.toBeCalibrated = {} # parameters that will be calibrated
# assembler objects
self.addAssemblerObject('proposal',
InputData.Quantity.zero_to_infinity)
self.addAssemblerObject('probabilityFunction',
InputData.Quantity.zero_to_infinity)
def initialize(self,externalSeeding=None,solutionExport=None):
"""
This function should be called every time a clean optimizer is needed. Called before takeAstep in <Step>
@ In, externalSeeding, int, optional, external seed
@ In, solutionExport, DataObject, optional, a PointSet to hold the solution
@ Out, None
"""
self.counter['mdlEval'] = 0
self.counter['varsUpdate'] = [0]*len(self.optTraj)
self.nVar = len(self.optVars)
self.mdlEvalHist = self.assemblerDict['TargetEvaluation'][0][3]
self.objSearchingROM = SupervisedLearning.returnInstance('SciKitLearn', self, **{'SKLtype':'neighbors|KNeighborsRegressor', 'Features':','.join(list(self.optVars)), 'Target':self.objVar, 'n_neighbors':1,'weights':'distance'})
self.solutionExport = solutionExport
if solutionExport != None and type(solutionExport).__name__ != "HistorySet":
self.raiseAnError(IOError,'solutionExport type is not a HistorySet. Got '+ type(solutionExport).__name__+ '!')
if 'Function' in self.assemblerDict.keys():
self.constraintFunction = self.assemblerDict['Function'][0][3]
if 'constrain' not in self.constraintFunction.availableMethods():
self.raiseAnError(IOError,'the function provided to define the constraints must have an implemented method called "constrain"')
if self.initSeed != None:
Distributions.randomSeed(self.initSeed)
# specializing the self.localInitialize()
if solutionExport != None:
self.localInitialize(solutionExport=solutionExport)
else:
self.localInitialize()
def _generateDistributions(self, availableDist, availableFunc):
"""
Generates the distributions and functions.
@ In, availableDist, dict, dict of distributions
@ In, availableFunc, dict, dict of functions
@ Out, None
"""
if self.initSeed != None:
Distributions.randomSeed(self.initSeed)
for key in self.toBeSampled.keys():
if self.toBeSampled[key] not in availableDist.keys():
self.raiseAnError(
IOError, 'Distribution ' + self.toBeSampled[key] +
' not found among available distributions (check input)!')
self.distDict[key] = availableDist[self.toBeSampled[key]]
self.inputInfo['crowDist'][key] = json.dumps(
self.distDict[key].getCrowDistDict())
for key, val in self.dependentSample.items():
if val not in availableFunc.keys():
self.raiseAnError(
'Function', val,
'was not found among the available functions:',
availableFunc.keys())
self.funcDict[key] = availableFunc[val]
# check if the correct method is present
if "evaluate" not in self.funcDict[key].availableMethods():
self.raiseAnError(
IOError, 'Function ' + self.funcDict[key].name +
' does not contain a method named "evaluate". It must be present if this needs to be used in a Sampler!'
)
def init():
distributions = [
Dist.NormalDistribution(),
Dist.CauchyDistribution(),
Dist.LaplaceDistribution(),
Dist.PoissonDistribution(),
Dist.UniformDistribution()
]
sizes = [20, 100]
return distributions, sizes,
def __init__(self,
sensors,
target,
cmdargs,
using_safe_mode=False,
with_predictor=False):
self._sensors = sensors
self._target = target
self._cmdargs = cmdargs
self._with_predictor = with_predictor
self._radar = sensors['radar']
self._gps = sensors['gps']
self.debug_info = {
"last_mbv": np.array([0, 0]),
'drawing_pdf': np.zeros(360)
}
self.normal_speed = cmdargs.robot_speed
self.max_speed = 10
if (self._cmdargs.speedmode == 5):
self.normal_speed = 10
self._speed_adjust_pdf = Distributions.Gaussian()
self._speed_adjust_pdf.degree_resolution = 2
self._PDF = Distributions.Gaussian()
if (self._cmdargs.target_distribution_type == 'rectangular'):
self._PDF = Distributions.Rectangular()
# Set up obstacle predictor
if with_predictor:
self._obstacle_predictor = HMMObstaclePredictor(
360, self._radar.radius)
# Number of steps taken in the navigation
self.stepNum = 0
# Function that selects an angle based on a distribution
self._pdf_angle_selector = self._center_of_gravity_pdfselector
# Function that combines pdfs
self._combine_pdfs = np.minimum
# Memory parameters
self.visited_points = []
self.memory_sigma = cmdargs.robot_memory_sigma
self.memory_decay = cmdargs.robot_memory_decay
self.memory_size = cmdargs.robot_memory_size
# Used to allow real-time plotting
if cmdargs.show_real_time_plot:
plt.ion()
plt.show()
def localInputAndChecks(self, xmlNode):
"""
Local method for additional reading.
@ In, xmlNode, xml.etree.ElementTree.Element, Xml element node
@ Out, None
"""
GradientBasedOptimizer.localInputAndChecks(self, xmlNode)
self.paramDict['alpha'] = float(self.paramDict.get('alpha', 0.602))
self.paramDict['gamma'] = float(self.paramDict.get('gamma', 0.101))
self.paramDict['A'] = float(
self.paramDict.get('A', self.limit['mdlEval'] / 10))
self.paramDict['a'] = float(self.paramDict.get('a', 0.16))
self.paramDict['c'] = float(self.paramDict.get('c', 0.005))
# Normalize the parameters...
if self.gradDict['normalize']:
tempMax = -1
for var in self.optVars:
if self.optVarsInit['upperBound'][var] - self.optVarsInit[
'lowerBound'][var] > tempMax:
tempMax = self.optVarsInit['upperBound'][
var] - self.optVarsInit['lowerBound'][var]
self.paramDict['c'] = copy.deepcopy(self.paramDict['c'] / tempMax)
self.paramDict['a'] = copy.deepcopy(self.paramDict['a'] /
(tempMax**2))
self.constraintHandlingPara['innerBisectionThreshold'] = float(
self.paramDict.get('innerBisectionThreshold', 1e-2))
self.constraintHandlingPara['innerLoopLimit'] = float(
self.paramDict.get('innerLoopLimit', 1000))
self.gradDict['pertNeeded'] = self.gradDict['numIterForAve'] * 2
if self.paramDict.get('stochasticDistribution',
'Bernoulli') == 'Bernoulli':
self.stochasticDistribution = Distributions.returnInstance(
'Bernoulli', self)
self.stochasticDistribution.p = 0.5
self.stochasticDistribution.initializeDistribution()
# Initialize bernoulli distribution for random perturbation. Add artificial noise to avoid that specular loss functions get false positive convergence
self.stochasticEngine = lambda: [
1.0 + (Distributions.random() / 1000.0
) * Distributions.randomIntegers(-1, 1, self)
if self.stochasticDistribution.rvs() == 1 else -1.0 +
(Distributions.random() / 1000.0
) * Distributions.randomIntegers(-1, 1, self)
for _ in range(self.nVar)
]
else:
self.raiseAnError(
IOError, self.paramDict['stochasticEngine'] +
'is currently not support for SPSA')
def transformDistDict(self):
"""
Performs distribution transformation
If the method 'pca' is used in the variables transformation (i.e. latentVariables to manifestVariables), the corrrelated variables
will be tranformed into uncorrelated variables with standard normal distributions. Thus, the dictionary of distributions will
be also transformed.
@ In, None
@ Out, distDicts, dict, distribution dictionary {varName:DistributionObject}
"""
# Generate a standard normal distribution, this is used to generate the sparse grid points and weights for multivariate normal
# distribution if PCA is used.
standardNormal = Distributions.Normal()
standardNormal.messageHandler = self.messageHandler
standardNormal.mean = 0.0
standardNormal.sigma = 1.0
standardNormal.initializeDistribution()
distDicts = {}
for varName in self.variables2distributionsMapping.keys():
distDicts[varName] = self.distDict[varName]
if self.variablesTransformationDict:
for key, varsDict in self.variablesTransformationDict.items():
if self.transformationMethod[key] == 'pca':
listVars = varsDict['latentVariables']
for var in listVars:
distDicts[var] = standardNormal
return distDicts
def localInputAndChecks(self, xmlNode):
"""
Local method for additional reading.
@ In, xmlNode, xml.etree.ElementTree.Element, Xml element node
@ Out, None
"""
GradientBasedOptimizer.localInputAndChecks(self, xmlNode)
self.paramDict['alpha'] = float(self.paramDict.get('alpha', 0.602))
self.paramDict['gamma'] = float(self.paramDict.get('gamma', 0.101))
self.paramDict['A'] = float(
self.paramDict.get('A', self.limit['mdlEval'] / 10.))
self.paramDict['a'] = self.paramDict.get('a', None)
self.paramDict['c'] = float(self.paramDict.get('c', 0.005))
#FIXME the optimization parameters should probably all operate ONLY on normalized data!
# -> perhaps the whole optimizer should only work on optimized data.
#FIXME normalizing doesn't seem to have the desired effect, currently; it makes the step size very small (for large scales)
#if "a" was defaulted, use the average scale of the input space.
#This is the suggested value from the paper, missing a 1/gradient term since we don't know it yet.
if self.paramDict['a'] is None:
self.paramDict['a'] = mathUtils.hyperdiagonal(
np.ones(len(
self.getOptVars()))) # the features are always normalized
self.raiseAMessage('Defaulting "a" gradient parameter to',
self.paramDict['a'])
else:
self.paramDict['a'] = float(self.paramDict['a'])
self.constraintHandlingPara['innerBisectionThreshold'] = float(
self.paramDict.get('innerBisectionThreshold', 1e-2))
self.constraintHandlingPara['innerLoopLimit'] = float(
self.paramDict.get('innerLoopLimit', 1000))
self.gradDict['pertNeeded'] = self.gradDict['numIterForAve'] * 2
stochDist = self.paramDict.get('stochasticDistribution', 'Hypersphere')
if stochDist == 'Bernoulli':
self.stochasticDistribution = Distributions.returnInstance(
'Bernoulli', self)
self.stochasticDistribution.p = 0.5
self.stochasticDistribution.initializeDistribution()
# Initialize bernoulli distribution for random perturbation. Add artificial noise to avoid that specular loss functions get false positive convergence
# FIXME there has to be a better way to get two random numbers
self.stochasticEngine = lambda: [
(0.5 + randomUtils.random() * (1. + randomUtils.random(
) / 1000. * randomUtils.randomIntegers(-1, 1, self)))
if self.stochasticDistribution.rvs() == 1 else -1. *
(0.5 + randomUtils.random() * (1. + randomUtils.random(
) / 1000. * randomUtils.randomIntegers(-1, 1, self)))
for _ in range(len(self.getOptVars()))
]
elif stochDist == 'Hypersphere':
self.stochasticEngine = lambda: randomUtils.randPointsOnHypersphere(
len(self.getOptVars()))
else:
self.raiseAnError(
IOError, self.paramDict['stochasticEngine'] +
'is currently not supported for SPSA')
def makeDistribution(self):
"""
Used to make standardized distribution for this poly type.
@ In, None
@ Out, None
"""
uniform = Distributions.Uniform(-1.0, 1.0)
uniform.initializeDistribution()
return uniform
def makeDistribution(self):
"""
Used to make standardized distribution for this poly type.
@ In, None
@ Out, gamma, Distribution, the gamma distribution
"""
gamma = Distributions.Gamma(0.0, self.params[0] + 1)
gamma.initializeDistribution()
return gamma
def makeDistribution(self):
"""
Used to make standardized distribution for this poly type.
@ In, None
@ Out, normal, Distribution, the normal distribution
"""
normal = Distributions.Normal(0.0, 1.0)
normal.initializeDistribution()
return normal
def ml_estimate(self,data): # Returns list of parameters for each distribution type; parameter list of length same as distribution # probability distribution of x given theta(parameters) # guassian n_features = data.shape[1] - 1 if not self.isNaive: # Fit a multivariate Gaussian in this case self.parameters.append(Distributions.gaussian_mle(data)) return for i in range(len(self.distribution)): X = np.vstack((data[:,i],data[:,-1])) X = X.transpose() di = self.distribution[i] if di == 0: self.parameters.append(Distributions.gaussian_mle(X)) elif di ==1: self.parameters.append(Distributions.multinomial_mle(X)) elif di == -1: self.parameters.append(-1)
def getDistribution(xmlElement):
"""
Parses the xmlElement and returns the distribution
"""
distributionInstance = Distributions.returnInstance(xmlElement.tag, mh)
distributionInstance.setMessageHandler(mh)
paramInput = distributionInstance.getInputSpecification()()
paramInput.parseNode(xmlElement)
distributionInstance._handleInput(paramInput)
distributionInstance.initializeDistribution()
return distributionInstance
def readSamplerInit(self, xmlNode):
"""
This method is responsible to read only the samplerInit block in the .xml file.
This method has been moved from the base sampler class since the samplerInit block is needed only for the MC and stratified (LHS) samplers
@ In, xmlNode, xml.etree.ElementTree.Element, Xml element node
@ Out, None
"""
for child in xmlNode:
if child.tag == "samplerInit":
self.initSeed = Distributions.randomIntegers(0, 2**31, self)
for childChild in child:
if childChild.tag == "limit":
try:
self.limit = int(childChild.text)
except ValueError:
self.raiseAnError(
IOError,
'reading the attribute for the sampler ' +
self.name +
' it was not possible to perform the conversion to integer for the attribute limit with value '
+ str(childChild.text))
if childChild.tag == "initialSeed":
try:
self.initSeed = int(childChild.text)
except ValueError:
self.raiseAnError(
IOError,
'reading the attribute for the sampler ' +
self.name +
' it was not possible to perform the conversion to integer for the attribute initialSeed with value '
+ str(childChild.text))
elif childChild.tag == "reseedEachIteration":
if childChild.text.lower(
) in utils.stringsThatMeanTrue():
self.reseedAtEachIteration = True
elif childChild.tag == "distInit":
for childChildChild in childChild:
NDdistData = {}
for childChildChildChild in childChildChild:
if childChildChildChild.tag == 'initialGridDisc':
NDdistData[childChildChildChild.tag] = int(
childChildChildChild.text)
elif childChildChildChild.tag == 'tolerance':
NDdistData[
childChildChildChild.tag] = float(
childChildChildChild.text)
else:
self.raiseAnError(
IOError, 'Unknown tag ' +
childChildChildChild.tag +
' .Available are: initialGridDisc and tolerance!'
)
self.NDSamplingParams[
childChildChild.attrib['name']] = NDdistData
def __init__(self):
"""
Default Constructor
"""
GradientBasedOptimizer.__init__(self)
self.stochasticDistribution = None # Distribution used to generate perturbations
self.stochasticEngine = None # Random number generator used to generate perturbations
self.stochasticEngineForConstraintHandling = Distributions.returnInstance(
'Normal', self)
self.stochasticEngineForConstraintHandling.mean, self.stochasticEngineForConstraintHandling.sigma = 0, 1
self.stochasticEngineForConstraintHandling.upperBoundUsed, self.stochasticEngineForConstraintHandling.lowerBoundUsed = False, False
self.stochasticEngineForConstraintHandling.initializeDistribution()
def fit(self, test_X):
# multiply likelihood to priors
# print(len(self.parameters))
# print(self.parameters[0])
# print(self.parameters[1])
# print(self.parameters[2])
# print(self.parameters[3])
# print(self.parameters[4])
# print(self.parameters[5])
predicted_class = []
probas = []
for x in test_X:
# x = x[:-1]
posteriors = {}
for c in self.priors:
# c = int(c)
if not self.isNaive:
likelihood = Distributions.gaussian_multivar(x[:-1], self.parameters[0][0][c], self.parameters[0][1][c])
else:
likelihood = 1
for i in range(len(self.distribution)):
di = self.distribution[i]
if di == 0:
likelihood = likelihood*Distributions.gaussian(x[i],self.parameters[i][0][c][0],self.parameters[i][1][c][0][0])
elif di == 1:
likelihood = likelihood*self.parameters[i][c][x[i]]
elif di == -1:
continue
posteriors[c] = likelihood*self.priors[c]
best_class = max(posteriors.items(), key=operator.itemgetter(1))[0]
predicted_class.append(best_class)
probas.append(posteriors)
self.posterior_probas = probas
return predicted_class
def makeDistribution(self):
"""
Used to make standardized distribution for this poly type.
@ In, None
@ Out, gamma, Distribution, the gamma distribution
"""
gammaElement = ET.Element("Gamma")
element = ET.Element("low", {})
element.text = "0"
gammaElement.append(element)
element = ET.Element("alpha", {})
element.text = "%s" % (self.params[0] + 1)
gammaElement.append(element)
gamma = Distributions.Gamma()
gamma._readMoreXML(gammaElement)
gamma.initializeDistribution()
return gamma
def makeDistribution(self):
"""
Used to make standardized distribution for this poly type.
@ In, None
@ Out, normal, Distribution, the normal distribution
"""
normalElement = ET.Element("Normal")
element = ET.Element("mean", {})
element.text = "0"
normalElement.append(element)
element = ET.Element("sigma", {})
element.text = "1"
normalElement.append(element)
normal = Distributions.Normal()
normal._readMoreXML(normalElement)
normal.initializeDistribution()
return normal
def makeDistribution(self):
"""
Used to make standardized distribution for this poly type.
@ In, None
@ Out, None
"""
uniformElement = ET.Element("Uniform")
element = ET.Element("lowerBound", {})
element.text = "-1"
uniformElement.append(element)
element = ET.Element("upperBound", {})
element.text = "1"
uniformElement.append(element)
uniform = Distributions.Uniform()
uniform._readMoreXML(uniformElement)
uniform.initializeDistribution()
return uniform
def _trainMultivariateNormal(self, dim, means, cov):
"""
Trains multivariate normal distribution for future sampling
@ In, dim, int, number of dimensions
@ In, means, np.array, distribution mean
@ In, cov, np.ndarray, dim x dim matrix of covariance terms
@ Out, dist, Distributions.MultivariateNormal, distribution
"""
dist = Distributions.MultivariateNormal()
dist.method = 'pca'
dist.dimension = dim
dist.rank = dim
dist.mu = means
dist.covariance = np.ravel(cov)
dist.messageHandler = self.messageHandler
dist.initializeDistribution()
return dist
def localInputAndChecks(self, xmlNode, paramInput):
"""
Local method for additional reading.
@ In, xmlNode, xml.etree.ElementTree.Element, Xml element node
@ In, paramInput, InputData.ParameterInput, the parsed parameters
@ Out, None
"""
GradientBasedOptimizer.localInputAndChecks(self, xmlNode, paramInput)
self.currentDirection = None
numValues = self._numberOfSamples()
# set the initial step size
## use the hyperdiagonal of a unit hypercube with a side length equal to the user's provided initialStepSize * 1.0
stepPercent = float(self.paramDict.get('initialStepSize', 0.05))
self.paramDict['initialStepSize'] = mathUtils.hyperdiagonal(np.ones(numValues)*stepPercent)
self.raiseADebug('Based on initial step size factor of "{:1.5e}", initial step size is "{:1.5e}"'
.format(stepPercent, self.paramDict['initialStepSize']))
# set the perturbation distance
## if not given, default to 10% of the step size
self.paramDict['pertDist'] = float(self.paramDict.get('perturbationDistance',0.01))
self.raiseADebug('Perturbation distance is "{:1.5e}" percent of the step size'
.format(self.paramDict['pertDist']))
self.constraintHandlingPara['innerBisectionThreshold'] = float(self.paramDict.get('innerBisectionThreshold', 1e-2))
if not 0 < self.constraintHandlingPara['innerBisectionThreshold'] < 1:
self.raiseAnError(IOError,'innerBisectionThreshold must be between 0 and 1; got',self.constraintHandlingPara['innerBisectionThreshold'])
self.constraintHandlingPara['innerLoopLimit'] = float(self.paramDict.get('innerLoopLimit', 1000))
self.gradDict['pertNeeded'] = self.gradDict['numIterForAve'] * (self.paramDict['pertSingleGrad']+1)
# determine the number of indpendent variables (scalar and vectors included)
stochDist = self.paramDict.get('stochasticDistribution', 'Hypersphere')
if stochDist == 'Bernoulli':
self.stochasticDistribution = Distributions.returnInstance('Bernoulli',self)
self.stochasticDistribution.p = 0.5
self.stochasticDistribution.initializeDistribution()
# Initialize bernoulli distribution for random perturbation. Add artificial noise to avoid that specular loss functions get false positive convergence
# FIXME there has to be a better way to get two random numbers
self.stochasticEngine = lambda: [(0.5+randomUtils.random()*(1.+randomUtils.random()/1000.*randomUtils.randomIntegers(-1, 1, self))) if self.stochasticDistribution.rvs() == 1 else
-1.*(0.5+randomUtils.random()*(1.+randomUtils.random()/1000.*randomUtils.randomIntegers(-1, 1, self))) for _ in range(numValues)]
elif stochDist == 'Hypersphere':
# TODO assure you can't get a "0" along any dimension! Need to be > 1e-15. Right now it's just highly unlikely.
self.stochasticEngine = lambda: randomUtils.randPointsOnHypersphere(numValues) if numValues > 1 else [randomUtils.randPointsOnHypersphere(numValues)]
else:
self.raiseAnError(IOError, self.paramDict['stochasticEngine']+'is currently not supported for SPSA')
def __init__(self, sensors, target, cmdargs):
self._sensors = sensors
self._cmdargs = cmdargs
self._target = target
self._radar = self._sensors['radar']
self._radar_data = None
self._dynamic_radar_data = None
self._gps = self._sensors['gps']
self._normal_speed = cmdargs.robot_speed
self._max_sampling_iters = 200
self._current_mem_bias_pdf = None
self._safety_threshold = 0.1
self._stepNum = 0
self._cur_traj = []
self._cur_traj_index = 0
self.debug_info = {
'node_list': [],
'drawing_pdf': np.zeros(360),
"cur_dist": np.zeros(360),
}
# Memory parameters
self.visited_points = []
self.memory_sigma = cmdargs.robot_memory_sigma
self.memory_decay = cmdargs.robot_memory_decay
self.memory_size = cmdargs.robot_memory_size
self._mem_bias_vec = np.array([0.7, 0.7])
self.using_safe_mode = True
gaussian_sigma = 100
self._gaussian = Distributions.Gaussian(
sigma=gaussian_sigma,
amplitude=(1 / (np.sqrt(2 * np.pi) * gaussian_sigma)))
# Obstecle Predictor
self._obstacle_predictor = CollisionConeObstaclePredictor(
360, sensors['radar'].radius, 5)
def __init__(self,
num_hist_int,
bias_gen,
data_gen,
DE,
repeat=1,
trafo=Transformations.trafo_keep_axes,
model=svm.SVC(kernel='linear')):
self.num_hist_int = num_hist_int
self.bias_gen = bias_gen
self.data_gen = data_gen
self.repeat = repeat
self.model = svm.SVC(kernel='linear')
self.reset(new_data=True)
self.density_func = Distributions.scaled_norm(ends_zero=True)
self.DE = DE
self.trafo = trafo
self.colors = [plt.cm.viridis(0), 'teal', 'goldenrod', 'deepskyblue']
self.colormap = lambda x: [
self.colors[self.D.labels.index(xi)] for xi in x
]
def __init__(self):
"""
Default Constructor that will initialize member variables with reasonable
defaults or empty lists/dictionaries where applicable.
@ In, None
@ Out, None
"""
AdaptiveSampler.__init__(self)
self.onlySampleAfterCollecting = True
self._initialValues = {} # dict stores the user provided initial values, i.e. {var: val}
self._updateValues = {} # dict stores input variables values for the current MCMC iteration, i.e. {var:val}
self._proposal = {} # dict stores the proposal distributions for input variables, i.e. {var:dist}
self._proposalDist = {} # dist stores the input variables for each proposal distribution, i.e. {distName:[(var,dim)]}
self._priorFuns = {} # dict stores the prior functions for input variables, i.e. {var:fun}
self._burnIn = 0 # integers indicate how many samples will be discarded
self._likelihood = None # stores the output from the likelihood
self._logLikelihood = False # True if the user provided likelihood is in log format
self._availProposal = {'normal': Distributions.Normal,
'multivariateNormal': Distributions.MultivariateNormal} # available proposal distributions
self._acceptDist = Distributions.Uniform(0.0, 1.0) # uniform distribution for accept/rejection purpose
self.toBeCalibrated = {} # parameters that will be calibrated
self._correlated = False # True if input variables are correlated else False
self.netLogPosterior = 0.0 # log-posterior vs iteration
self._localReady = True # True if the submitted job finished
self._currentRlz = None # dict stores the current realizations, i.e. {var: val}
self._acceptRate = 1. # The accept rate for MCMC
self._acceptCount = 1 # The total number of accepted samples
self._tune = True # Tune the scaling parameter if True
self._tuneInterval = 100 # the number of sample steps for each tuning of scaling parameter
self._scaling = 1.0 # The initial scaling parameter
self._countsUntilTune = self._tuneInterval # The remain number of sample steps until the next tuning
self._acceptInTune = 0 # The accepted number of samples for given tune interval
self._accepted = False # The indication of current samples, True if accepted otherwise False
self._stdProposalDefault = 0.2 # the initial scaling of the std of proposal distribution (only apply to default)
# assembler objects
self.addAssemblerObject('proposal', InputData.Quantity.zero_to_infinity)
self.addAssemblerObject('probabilityFunction', InputData.Quantity.zero_to_infinity)
def __init__(self, sensors, target, cmdargs, params=None):
self._sensors = sensors
self._cmdargs = cmdargs
self._target = target
self._params = self._get_default_params()
if params is not None:
for param_name in params:
self._params[param_name] = params[param_name]
self._radar = self._sensors['radar']
self._radar_data = None
self._dynamic_radar_data = None
self._gps = self._sensors['gps']
self._normal_speed = cmdargs.robot_speed
self._max_sampling_iters = self._params['max_sampling_iters']
self._safety_threshold = self._params['safety_threshold']
self._stepNum = 0
self._cur_traj = []
self._cur_traj_index = 0
self.debug_info = {
"cur_dist": np.zeros(360),
}
gaussian_sigma = self._params['gaussian_sigma']
self._gaussian = Distributions.Gaussian(
sigma=gaussian_sigma,
amplitude=(1 / (np.sqrt(2 * np.pi) * gaussian_sigma)))
# Obstecle Predictor
self._obstacle_predictor = CollisionConeObstaclePredictor(
360, sensors['radar'].radius, 5)
def generateInput(self, model, oldInput):
"""
This method has to be overwritten to provide the specialization for the specific sampler
The model instance in might be needed since, especially for external codes,
only the code interface possesses the dictionary for reading the variable definition syntax
@ In, model, model instance, it is the instance of a RAVEN model
@ In, oldInput, list, a list of the original needed inputs for the model (e.g. list of files, etc. etc)
@ Out, generateInput, tuple(0,list), list contains the new inputs -in reality it is the model that returns this; the Sampler generates the value to be placed in the input of the model.
The Out parameter depends on the results of generateInput
If a new point is found, the default Out above is correct.
If a restart point is found:
@ Out, generateInput, tuple(int,dict), (1,realization dictionary)
"""
self.counter += 1 #since we are creating the input for the next run we increase the counter and global counter
self.auxcnt += 1
#exit if over the limit
if self.counter > self.limit:
self.raiseADebug(
'Exceeded number of points requested in sampling! Moving on...'
)
#FIXME, the following condition check is make sure that the require info is only printed once when dump metadata to xml, this should be removed in the future when we have a better way to dump the metadata
if self.counter > 1:
for key in self.entitiesToRemove:
self.inputInfo.pop(key, None)
if self.reseedAtEachIteration:
Distributions.randomSeed(self.auxcnt - 1)
self.inputInfo['prefix'] = str(self.counter)
model.getAdditionalInputEdits(self.inputInfo)
self.localGenerateInput(model, oldInput)
##### TRANSFORMATION #####
# add latent variables and original variables to self.inputInfo
if self.variablesTransformationDict:
for dist, var in self.variablesTransformationDict.items():
if self.transformationMethod[dist] == 'pca':
self.pcaTransform(var, dist)
else:
self.raiseAnError(
NotImplementedError,
'transformation method is not yet implemented for ' +
self.transformationMethod[dist] + ' method')
##### REDUNDANT FUNCTIONALS #####
# generate the function variable values
for var in self.dependentSample.keys():
test = self.funcDict[var].evaluate("evaluate", self.values)
for corrVar in var.split(","):
self.values[corrVar.strip()] = test
##### CONSTANT VALUES ######
self._constantVariables()
##### RESTART #####
#check if point already exists
if self.restartData is not None:
inExisting = self.restartData.getMatchingRealization(
self.values, tol=self.restartTolerance)
else:
inExisting = None
#if not found or not restarting, we have a new point!
if inExisting is None:
self.raiseADebug('Found new point to sample:', self.values)
return 0, model.createNewInput(oldInput, self.type,
**self.inputInfo)
#otherwise, return the restart point
else:
self.raiseADebug('Point found in restart:', inExisting['inputs'])
realization = {}
realization['metadata'] = copy.deepcopy(self.inputInfo)
realization['inputs'] = inExisting['inputs']
realization['outputs'] = inExisting['outputs']
realization['prefix'] = self.inputInfo['prefix']
return 1, realization
def initialize(self, externalSeeding=None, solutionExport=None):
"""
This function should be called every time a clean sampler is needed. Called before takeAstep in <Step>
@ In, externalSeeding, int, optional, external seed
@ In, solutionExport, DataObject, optional, in goal oriented sampling (a.k.a. adaptive sampling this is where the space/point satisfying the constrains)
@ Out, None
"""
if self.initSeed == None:
self.initSeed = Distributions.randomIntegers(0, 2**31, self)
self.counter = 0
if not externalSeeding:
Distributions.randomSeed(
self.initSeed) #use the sampler initialization seed
self.auxcnt = self.initSeed
elif externalSeeding == 'continue':
pass #in this case the random sequence needs to be preserved
else:
Distributions.randomSeed(
externalSeeding) #the external seeding is used
self.auxcnt = externalSeeding
#grab restart dataobject if it's available, then in localInitialize the sampler can deal with it.
if 'Restart' in self.assemblerDict.keys():
self.raiseADebug('Restart object: ' +
str(self.assemblerDict['Restart']))
self.restartData = self.assemblerDict['Restart'][0][3]
self.raiseAMessage('Restarting from ' + self.restartData.name)
#check consistency of data
try:
rdata = self.restartData.getAllMetadata()['crowDist']
sdata = self.inputInfo['crowDist']
self.raiseAMessage('sampler inputs:')
for sk, sv in sdata.items():
self.raiseAMessage('| ' + str(sk) + ': ' + str(sv))
for i, r in enumerate(rdata):
if type(r) != dict: continue
if not r == sdata:
self.raiseAMessage('restart inputs %i:' % i)
for rk, rv in r.items():
self.raiseAMessage('| ' + str(rk) + ': ' +
str(rv))
self.raiseAnError(
IOError,
'Restart "%s" data[%i] does not have same inputs as sampler!'
% (self.restartData.name, i))
except KeyError as e:
self.raiseAWarning(
"No CROW distribution available in restart -", e)
else:
self.raiseAMessage('No restart for ' + self.printTag)
#load restart data into existing points
if self.restartData is not None:
if not self.restartData.isItEmpty():
inps = self.restartData.getInpParametersValues()
outs = self.restartData.getOutParametersValues()
#FIXME there is no guarantee ordering is accurate between restart data and sampler
inputs = list(v for v in inps.values())
existingInps = zip(*inputs)
outVals = zip(*list(v for v in outs.values()))
self.existing = dict(zip(existingInps, outVals))
#specializing the self.localInitialize() to account for adaptive sampling
if solutionExport != None:
self.localInitialize(solutionExport=solutionExport)
else:
self.localInitialize()
for distrib in self.NDSamplingParams:
if distrib in self.distributions2variablesMapping:
params = self.NDSamplingParams[distrib]
temp = utils.first(
self.distributions2variablesMapping[distrib][0].keys())
self.distDict[temp].updateRNGParam(params)
else:
self.raiseAnError(
IOError,
'Distribution "%s" specified in distInit block of sampler "%s" does not exist!'
% (distrib, self.name))
# Store the transformation matrix in the metadata
if self.variablesTransformationDict:
self.entitiesToRemove = []
for variable in self.variables2distributionsMapping.keys():
distName = self.variables2distributionsMapping[variable][
'name']
dim = self.variables2distributionsMapping[variable]['dim']
totDim = self.variables2distributionsMapping[variable][
'totDim']
if totDim > 1 and dim == 1:
transformDict = {}
transformDict['type'] = self.distDict[
variable.strip()].type
transformDict['transformationMatrix'] = self.distDict[
variable.strip()].transformationMatrix()
self.inputInfo['transformation-' +
distName] = transformDict
self.entitiesToRemove.append('transformation-' + distName)
def _readMoreXMLbase(self, xmlNode):
"""
Function to read the portion of the xml input that belongs to the base sampler only
and initialize some stuff based on the inputs got
The text is supposed to contain the info where and which variable to change.
In case of a code the syntax is specified by the code interface itself
@ In, xmlNode, xml.etree.ElementTree.Element, Xml element node1
@ Out, None
"""
for child in xmlNode:
prefix = ""
if child.tag == 'Distribution':
for childChild in child:
if childChild.tag == 'distribution':
prefix = "<distribution>"
tobesampled = childChild.text
self.toBeSampled[prefix + child.attrib['name']] = tobesampled
#if child.attrib['name'] != tobesampled:self.raiseAnError(IOError,"name of the <Distribution> node and <distribution> mismatches for node named "+ child.attrib['name'])
elif child.tag == 'variable':
foundDistOrFunc = False
for childChild in child:
if childChild.tag == 'distribution':
if not foundDistOrFunc: foundDistOrFunc = True
else:
self.raiseAnError(
IOError,
'A sampled variable cannot have both a distribution and a function!'
)
tobesampled = childChild.text
varData = {}
varData['name'] = childChild.text
if childChild.get('dim') == None:
dim = 1
else:
dim = childChild.attrib['dim']
varData['dim'] = int(dim)
self.variables2distributionsMapping[
child.attrib['name']] = varData
self.toBeSampled[prefix +
child.attrib['name']] = tobesampled
elif childChild.tag == 'function':
if not foundDistOrFunc: foundDistOrFunc = True
else:
self.raiseAnError(
IOError,
'A sampled variable cannot have both a distribution and a function!'
)
tobesampled = childChild.text
self.dependentSample[
prefix + child.attrib['name']] = tobesampled
if not foundDistOrFunc:
self.raiseAnError(
IOError, 'Sampled variable', child.attrib['name'],
'has neither a <distribution> nor <function> node specified!'
)
elif child.tag == "variablesTransformation":
transformationDict = {}
listIndex = None
for childChild in child:
if childChild.tag == "latentVariables":
transformationDict[childChild.tag] = list(
inp.strip()
for inp in childChild.text.strip().split(','))
elif childChild.tag == "manifestVariables":
transformationDict[childChild.tag] = list(
inp.strip()
for inp in childChild.text.strip().split(','))
elif childChild.tag == "manifestVariablesIndex":
# the index provided by the input file starts from 1, but the index used by the code starts from 0.
listIndex = list(
int(inp.strip()) - 1
for inp in childChild.text.strip().split(','))
elif childChild.tag == "method":
self.transformationMethod[
child.attrib['distribution']] = childChild.text
if listIndex == None:
self.raiseAWarning(
'Index is not provided for manifestVariables, default index will be used instead!'
)
listIndex = range(
len(transformationDict["manifestVariables"]))
transformationDict["manifestVariablesIndex"] = listIndex
self.variablesTransformationDict[
child.attrib['distribution']] = transformationDict
elif child.tag == "constant":
value = utils.partialEval(child.text)
if value is None:
self.raiseAnError(
IOError,
'The body of "constant" XML block should be a number. Got: '
+ child.text)
try:
self.constants[child.attrib['name']] = value
except KeyError:
self.raiseAnError(
KeyError,
child.tag + ' must have the attribute "name"!!!')
elif child.tag == "restartTolerance":
self.restartTolerance = float(child.text)
if len(self.constants) > 0:
# check if constant variables are also part of the sampled space. In case, error out
if not set(self.toBeSampled.keys()).isdisjoint(
self.constants.keys()):
self.raiseAnError(
IOError,
"Some constant variables are also in the sampling space:" +
' '.join([
i if i in self.toBeSampled.keys() else ""
for i in self.constants.keys()
]))
if self.initSeed == None:
self.initSeed = Distributions.randomIntegers(0, 2**31, self)
# Creation of the self.distributions2variablesMapping dictionary: {'distName': ({'variable_name1': dim1}, {'variable_name2': dim2})}
for variable in self.variables2distributionsMapping.keys():
distName = self.variables2distributionsMapping[variable]['name']
dim = self.variables2distributionsMapping[variable]['dim']
listElement = {}
listElement[variable] = dim
if (distName in self.distributions2variablesMapping.keys()):
self.distributions2variablesMapping[distName].append(
listElement)
else:
self.distributions2variablesMapping[distName] = [listElement]
# creation of the self.distributions2variablesIndexList dictionary:{'distName':[dim1,dim2,...,dimN]}
self.distributions2variablesIndexList = {}
for distName in self.distributions2variablesMapping.keys():
positionList = []
for var in self.distributions2variablesMapping[distName]:
position = utils.first(var.values())
positionList.append(position)
positionList = list(set(positionList))
positionList.sort()
self.distributions2variablesIndexList[distName] = positionList
for key in self.variables2distributionsMapping.keys():
distName = self.variables2distributionsMapping[key]['name']
dim = self.variables2distributionsMapping[key]['dim']
reducedDim = self.distributions2variablesIndexList[distName].index(
dim) + 1
self.variables2distributionsMapping[key][
'reducedDim'] = reducedDim # the dimension of variable in the transformed space
self.variables2distributionsMapping[key]['totDim'] = max(
self.distributions2variablesIndexList[distName]
) # We will reset the value if the node <variablesTransformation> exist in the raven input file
if not self.variablesTransformationDict and self.variables2distributionsMapping[
key]['totDim'] > 1:
if self.variables2distributionsMapping[key]['totDim'] != len(
self.distributions2variablesIndexList[distName]):
self.raiseAnError(
IOError,
'The "dim" assigned to the variables insider Sampler are not correct! the "dim" should start from 1, and end with the full dimension of given distribution'
)
#Checking the variables transformation
if self.variablesTransformationDict:
for dist, varsDict in self.variablesTransformationDict.items():
maxDim = len(varsDict['manifestVariables'])
listLatentElement = varsDict['latentVariables']
if len(set(listLatentElement)) != len(listLatentElement):
dups = set(var for var in listLatentElement
if listLatentElement.count(var) > 1)
self.raiseAnError(
IOError,
'The following are duplicated variables listed in the latentVariables: '
+ str(dups))
if len(set(varsDict['manifestVariables'])) != len(
varsDict['manifestVariables']):
dups = set(var for var in varsDict['manifestVariables']
if varsDict['manifestVariables'].count(var) > 1)
self.raiseAnError(
IOError,
'The following are duplicated variables listed in the manifestVariables: '
+ str(dups))
if len(set(varsDict['manifestVariablesIndex'])) != len(
varsDict['manifestVariablesIndex']):
dups = set(
var + 1 for var in varsDict['manifestVariablesIndex']
if varsDict['manifestVariablesIndex'].count(var) > 1)
self.raiseAnError(
IOError,
'The following are duplicated variables indices listed in the manifestVariablesIndex: '
+ str(dups))
listElement = self.distributions2variablesMapping[dist]
for var in listElement:
self.variables2distributionsMapping[var.keys()[0]][
'totDim'] = maxDim #reset the totDim to reflect the totDim of original input space
tempListElement = {
k.strip(): v
for x in listElement for ks, v in x.items()
for k in list(ks.strip().split(','))
}
listIndex = []
for var in listLatentElement:
if var not in set(tempListElement.keys()):
self.raiseAnError(
IOError,
'The variable listed in latentVariables ' + var +
' is not listed in the given distribution: ' +
dist)
listIndex.append(tempListElement[var] - 1)
if max(listIndex) > maxDim:
self.raiseAnError(
IOError, 'The maximum dim = ' + str(max(listIndex)) +
' defined for latent variables is exceeded the dimension of the problem '
+ str(maxDim))
if len(set(listIndex)) != len(listIndex):
dups = set(var + 1 for var in listIndex
if listIndex.count(var) > 1)
self.raiseAnError(
IOError,
'Each of the following dimensions are assigned to multiple latent variables in Samplers: '
+ str(dups))
# update the index for latentVariables according to the 'dim' assigned for given var defined in Sampler
self.variablesTransformationDict[dist][
'latentVariablesIndex'] = listIndex
def ems_segment(hyperDataFiles,
hypoDataFiles,
maskFiles,
atlasFiles,
tumor_lower_co=None,
tumor_upper_co=None,
time_index=None,
tumor_center=None,
show=True):
'''
Segments a brain image in tumor tissue and in brain tissues (white
matter, grey matter and cerebrospinal fluid).
Parameters
----------
- hyperDataFiles: list of strings
path to Images in which tumor appears hyperintensive
- hypoDataFiles: list of strings
paths to images in which tumor appears hypointensive
- maskFiles: list of strings
paths to binary mask(s) of one or more of the images in hyper- and
hypoDataFiles. Defaults to intersection of registered
atlas in atlasFiles and image data in hyper- and hypoDataFiles
- atlasFiles: list of 3 strings
the 1st, 2nd and 3rd string reference atlas files of grey matter,
white matter and cerebrospinal fluid *respectively*.
- time_index: integer [optional]
integer denoting a time index, used for paths to which results are written
(only integrated in filename so that multiple image series can be stored with
different filenames)
- tumor_center: tuple of three integers [optional]
index of tumor_center, for visualisation purposes used only
- show: boolean [optional]
Generate and visualise images during runtime or not
.. NOTE:: All files should contain preprocessed images, in the sense that,
all images should have the same resolution, the same isotropic pixel
dimensions and they should be in the same reference space.
Accepted file extensions are: *TIFF, JPEG, PNG, BMP, DICOM, GIPL,
Bio-Rad, LSM, Nifti, Analyze, SDT/SPR (Stimulate), Nrrd or VTK images*.
Returns
-------
All of the returned itk images are also written to files in a
subdirectory of the current directory: ::./results/ .
- tumorImages: list of 3D itk images
Soft tumor segmentations for each supplied file following the order of
the concatenation of hyperDataFiles and hypoDataFiles.
Written to ::./results/<filename>_tumor_t<time_index>.nii ,
where filename is the original filename extracted from the supplied
filenames (hypo- and hyperDataFiles)
- gmImage: 3D itk image
Soft grey matter segmentation. Written to ::./results/gm.nii .
- wmImage: 3D itk image
Soft white matter segmentation. Written to ::./results/wm.nii .
- csfImage: 3D itk image
Soft cerebrospinal fluid segmentation. Written to ::./results/csf.nii .
'''
print('')
print('Simple EMS has started')
print('no biasfield correction and no MRF spatial regularisator')
print('Input data is supposed to be preprocessed')
print('Images, masks and atlas should be in the same reference space')
print('No tumor will be allowed in CSF voxels')
print('')
start_time = time.time()
# Concatenate datafiles and store which ones have hyperintense tumors.
if hyperDataFiles is None:
hyperDataFiles = []
if hypoDataFiles is None:
hypoDataFiles = []
if isinstance(hyperDataFiles, str):
hyperDataFiles = [hyperDataFiles]
if isinstance(hypoDataFiles, str):
hypoDataFiles = [hypoDataFiles]
dataFiles = np.concatenate((hyperDataFiles, hypoDataFiles), axis=0)
modalities = getModalityStrings(dataFiles)
nrOfChannels = len(dataFiles)
hyper = np.zeros(shape=(nrOfChannels, ))
hyper[0:len(hyperDataFiles)] = 1
# Check data dimensionalities.
maxDim = 0
for channel in range(nrOfChannels):
noNeed, tmpDIM, tmpSpaces = oitk.getItkImageData(dataFiles[channel])
dataType = noNeed.dtype
if maxDim < np.prod(tmpDIM):
if channel is not 0:
ask_continue_datasize('data', channel + 1)
DIM = tmpDIM
maxDim = np.prod(DIM)
dataImagePrototype = oitk.getItkImage(dataFiles[channel])
dataSpaces = tmpSpaces
print('Data size of channel ' + str(channel + 1) + ' is: ' +
str(tmpDIM))
# Read brain mask(s).
if maskFiles != None:
playing = np.ones(shape=(np.prod(DIM)), dtype='?')
if isinstance(maskFiles, str):
maskFiles = [maskFiles]
else:
print('You supplied ' + str(len(maskFiles)) + ' masks')
print('The intersection will be taken as the true mask')
for fileNb in range(len(maskFiles)):
mask, DIMmask = oitk.getItkImageData(maskFiles[fileNb])[0:-1]
if DIMmask != DIM:
ask_continue_datasize('mask', fileNb + 1)
playing = np.logical_and(playing, np.ravel(mask))
# Read atlas.
classGM = 0
classWM = 1
classCSF = 2
nrOfClasses = len(atlasFiles)
atlasTmp = np.zeros(shape=(nrOfClasses, np.prod(DIM)))
for classNr in range(nrOfClasses):
atlasRaw, DIMatlas, noNeed = oitk.getItkImageData(atlasFiles[classNr])
if DIMatlas != DIM:
ask_continue_datasize('atlas', fileNb + 1)
atlasTmp[classNr, :] = np.ravel(atlasRaw)
atlasTmp = norm.window(atlasTmp, 0, 1)
# Define brain voxels in case mask is not given.
sumAtlas = np.sum(atlasTmp, axis=0)
if maskFiles == None:
playing = np.ones(shape=(np.prod(DIM), ), dtype='?')
playing[sumAtlas > 0.5] = True
playing[sumAtlas <= 0.5] = False
else: #throw out brain voxels with nonzero atlas probabilities
playing = np.logical_and(sumAtlas > 0, playing)
nrOfBrainVoxels = np.count_nonzero(playing)
# Normalise and mask the atlas.
atlas = np.zeros(shape=(nrOfClasses, nrOfBrainVoxels))
for classNr in range(nrOfClasses):
atlasTmp[classNr,
playing] = atlasTmp[classNr, playing] / sumAtlas[playing]
atlas[classNr, :] = atlasTmp[classNr, playing]
# Read, normalise and mask the data.
data = np.zeros(shape=(nrOfChannels, nrOfBrainVoxels), dtype=dataType)
dataNotNorm = np.zeros(shape=(nrOfChannels, np.prod(DIM)), dtype=dataType)
for channel in range(nrOfChannels):
image, dataDIM, dontNeed = oitk.getItkImageData(dataFiles[channel])
flatImage = np.ravel(image)
dataNotNorm[channel, :] = flatImage
data[channel, :] = flatImage[playing]
inclusion = getInclusion(dataFiles)
# Specify how many gaussians are used to model each tissue class
nc = np.array([1, 1, 2]) # Number of Gaussians per tissue class
lkp = [] # Class to which a certain gaussian belongs
for i in range(len(nc)):
lkp.extend(np.ones(nc[i], dtype='int') * (i))
lkp = np.array(lkp)
nrOfGaussians = len(lkp)
# Read out tumor mask
print 'original image dimensions: ' + str(DIM)
if tumor_lower_co is None:
tlc = np.zeros(3, dtype='int')
else:
tlc = tumor_lower_co
if tumor_upper_co is None:
tuc = DIM - np.ones(3, dtype='int')
else:
tuc = tumor_upper_co
DIMbox = (tuc - tlc) + np.ones(3, dtype='int')
print 'tumor box dimensions: ' + str(DIMbox)
tumorMask = np.zeros(DIM)
tumorMask[tlc[0]:tuc[0] + 1, tlc[1]:tuc[1] + 1, tlc[2]:tuc[2] + 1] = 1
tumorMask = np.ravel(tumorMask)[playing]
# Initialise brain tumor atlas
tumorAtlas = np.zeros(shape=(2, nrOfBrainVoxels))
tumorBit = 1
noTumorBit = 0
tumorAtlas[noTumorBit, :] = tumorMask * flat_prior
tumorAtlas[tumorBit, :] = 1 - tumorAtlas[noTumorBit, :]
# Initialise tumor classification.
tumorClassification = np.zeros(shape=(nrOfChannels, nrOfBrainVoxels))
for channel in range(nrOfChannels):
tumorClassification[channel, :] = tumorAtlas[tumorBit, :]
# Specifiy tumor classes: tumor is allowed only in WM and GM, not in CSF.
tumorClasses = np.zeros((nrOfClasses, ), dtype='?')
tumorClasses[classWM] = 1
tumorClasses[classGM] = 1
noTumorClasses = 1 - tumorClasses
# Specify tumor presence configurations along the channels.
nrTumorConf = 2**nrOfChannels
tumorConf = np.zeros(shape=(nrTumorConf, nrOfChannels), dtype='?')
for conf in range(nrTumorConf):
tumorConf[conf, :] = [
bool(int(i))
for i in list(format(conf, '0' + str(nrOfChannels) + 'b'))
]
tumorTransitionMatrix = tumorConf
# throw out unrealistic tumor transitions
if inclusion:
tumorTransitionMatrix = inclusion_restrictions(tumorTransitionMatrix,
modalities)
nrTumorTrans = tumorTransitionMatrix.shape[0]
# Calculate matrix with all tumor configurations over all tissue classes.
nrTissueConf = np.dot(
nc, noTumorClasses) + np.dot(nc, tumorClasses) * nrTumorTrans
classes = np.zeros(shape=(nrTissueConf))
gaussians = np.zeros(shape=(nrTissueConf))
tumorTransitions = np.zeros(shape=(nrTissueConf, nrOfChannels), dtype='?')
ind = 0
for gauss in range(nrOfGaussians):
classNr = lkp[gauss]
if tumorClasses[classNr] == noTumorBit:
classes[ind] = classNr
gaussians[ind] = gauss
tumorTransitions[ind, :] = 0
ind = ind + 1
if tumorClasses[classNr] == tumorBit:
classes[ind:ind + nrTumorTrans] = classNr
gaussians[ind:ind + nrTumorTrans] = gauss
tumorTransitions[ind:ind + nrTumorTrans, :] = tumorTransitionMatrix
ind = ind + nrTumorTrans
nrTissueConf = len(classes)
mask = playing.astype('int')
# Initialisations for intensity guassians.
normalMeans = np.zeros(shape=(nrOfGaussians, nrOfChannels))
normalVars = np.zeros(shape=(nrOfGaussians, nrOfChannels, nrOfChannels))
tumorMeans = np.zeros(shape=(nrOfChannels, ))
tumorVars = np.zeros(shape=(nrOfChannels, nrOfChannels))
tissueConfVars = np.zeros(shape=(nrTissueConf, nrOfChannels, nrOfChannels))
tissueConfMeans = np.zeros(shape=(nrTissueConf, nrOfChannels))
# Initialise healthy tissue classification.
normalClassification = np.zeros(shape=(nrOfClasses, nrOfBrainVoxels))
gaussianClassification = np.zeros(shape=(nrOfGaussians, nrOfBrainVoxels))
atlasPrior = np.zeros(shape=(nrOfGaussians, nrOfBrainVoxels))
for gauss in range(nrOfGaussians):
classNr = lkp[gauss]
normalClassification[classNr, :] = atlas[classNr, :]
atlasPrior[gauss, :] = atlas[classNr, :] / nc[classNr]
gaussianClassification[gauss, :] = atlasPrior[gauss, :]
# Initialisations for iterations.
converged = 0
iteration = 0
# Numerical declarations.
global EPS, TINY
EPS = np.finfo(np.double).eps
TINY = np.finfo(np.double).tiny
# Log-likelihood initialisations.
logLLHArray = np.zeros((maxIteration, ))
logLLH = 1 / EPS
relativeLogLLH = 1
# Create directory to store results.
dir_results = os.path.dirname(dataFiles[0]) + '/ems_results/'
make_dir(dir_results)
if inclusion:
dir_results = os.path.join(dir_results, 'with_inclusion/')
make_dir(dir_results)
nii_dir = dir_results + 'images/'
make_dir(nii_dir)
fig_dir = dir_results + 'figures/'
make_dir(fig_dir)
# Visualisation.
if show:
opInst = opc.OwnPlot(visual_center=tumor_center,
visual_axis=0,
spacing=dataSpaces[::-1],
file_extension='.png',
fig_dir=fig_dir)
opInst.show3Dplot(to_matrix(mask, DIM), None, 'Mask')
opInst.show5Dplot(to_matrix(atlas, DIM, playing), None, 'Input Atlas')
opInst.show5Dplot(to_matrix(dataNotNorm, DIM), None, 'Input Data')
for channel in range(nrOfChannels):
dataSlice = dataNotNorm[channel, :].reshape(DIM)
opInst.show2Dplot(dataSlice, None, '')
# Start iterating
while converged == False:
iteration = iteration + 1
previousLogLLH = logLLH
print('---------------')
print('Iteration ' + str(iteration))
# if verbose:
# memory = float( awk(ps('u','-p',os.getpid()),
# '{sum=sum+$6}; END {print sum/1024}') )
# print('Memory used: ' + str(memory) + ' MB')
# print('')
#
# Estimate Gaussian mixture parameters mean and covariance
print('Estimating mixture parameters')
print('')
# 1. Calculate normal means
new_normalWeights = np.zeros(
[nrOfGaussians, nrOfChannels, nrOfBrainVoxels])
for gauss in range(nrOfGaussians):
new_normalWeights[gauss, :, :] = np.tile(
gaussianClassification[gauss, :],
(nrOfChannels, 1)) * (1 - tumorClassification)
for channel in range(nrOfChannels):
new_normalWeights[gauss, channel, :] = norm.rescale(
new_normalWeights[gauss, channel, :])
# If weights all become zero, reset to weights of previous iteration
if iteration > 1:
for channel in range(nrOfChannels):
if np.sum(new_normalWeights[gauss, channel, :]) == 0:
new_normalWeights[gauss,channel,:] = \
np.copy(normalWeights[gauss,channel,:])
normalMeans[gauss, :] = distr.getMean(
data, new_normalWeights[gauss, :, :])
normalVars[gauss, :, :], nonsymm = distr.getVariance(
data,
new_normalWeights[gauss, :, :],
normalMeans[gauss, :],
verbose=verbose)
if nonsymm:
pdb.set_trace()
normalWeights = new_normalWeights
# Split gaussians within the same tissue class
# gaussians would remain identical after initialisation if we do not push
# them apart artificially
if iteration <= 3:
print('Splitting clusters of normal tissue classes')
if verbose:
print('Old means: ' + str(normalMeans))
normalMeans = splitClassMeans(normalMeans, lkp)
if verbose:
print('Normal means of iteration ' + str(iteration) + ': ')
print(str(normalMeans))
print('')
# 2. Calculate tumor means
tumorWeights = np.zeros([nrOfChannels, nrOfBrainVoxels])
for channel in range(nrOfChannels):
# Initialise weights with previous tumor classification
weights = tumorClassification[channel, :]
# Do a binary threshold of the previous tumor classification
weights[weights < np.mean(weights[weights > 0])] = 0.1
weights[weights >= np.mean(weights[weights > 0])] = 0.9
# do not weight with non-presence of normal tissue,
# normalCLassification is overlapping with tumorClassification
# Throw out pixels which are far from hyper- or hypo-intensive
channelNormalMeans = normalMeans[:, channel]
meanGM = np.average(channelNormalMeans[lkp == classGM])
meanWM = np.average(channelNormalMeans[lkp == classWM])
# Weights should be inversily related to the WM probabilities
WMprob = np.zeros((nrOfBrainVoxels, ))
for gauss in np.nditer(np.where(lkp == classWM)):
gaussWMprob = distr.gaussianEval_multiVar(
data[channel, :], normalMeans[gauss, channel],
normalVars[gauss, channel, channel],
False) * gaussianClassification[gauss, :]
WMprob = np.amax([WMprob, gaussWMprob], axis=0)
notWMprob = 1 - WMprob
notWMprob = norm.window(notWMprob, 0, 1)
weights = weights * notWMprob
df = threshold * np.absolute(
meanGM - meanWM) # distance between WM & GM means
if (hyper[channel] == 1):
# print('WM Mean for '+modalities[channel]+': '+str(meanWM))
# print('GM Mean for '+modalities[channel]+': '+str(meanGM))
weights[data[channel, :] <= (meanWM + df)] = 0
else:
weights[data[channel, :] >= (meanGM)] = 0
try:
if (np.sum(weights) == 0):
raise ZeroDivisionError('Tumor weights are all zero')
except ZeroDivisionError:
print('No tumor pixels present to update tumor gaussian')
print('Problem in channel ' + modalities[channel])
print('Maybe no tumor is present in the input images?')
print('If tumor is present, try loosening the tumor threshold')
print('Check #df in code')
raise
tumorWeights[channel, :] = norm.rescale(weights)
if np.count_nonzero(tumorWeights < 0) is not 0:
print('Warning: Negative weights present for gaussian update')
pdb.set_trace()
tumorMeans = distr.getMean(data, tumorWeights)
tumorVars, nonsymm = distr.getVariance(data, tumorWeights, tumorMeans)
if nonsymm:
pdb.set_trace()
if verbose:
print('Tumor means of iteration ' + str(iteration) + ': ')
print(str(tumorMeans))
print('')
# 3. Calculate covariance matrices for each tissue configuration map
for tissueConf in range(nrTissueConf):
gauss = gaussians[tissueConf]
tumorArray = tumorTransitions[tissueConf, :]
tumorInd = tumorArray == tumorBit
normInd = tumorArray == noTumorBit
currentMeans = np.zeros((nrOfChannels, ))
currentMeans[tumorInd] = tumorMeans[tumorInd]
if np.all(tumorArray == tumorBit):
tissueConfMeans[tissueConf, :] = currentMeans
tissueConfVars[tissueConf, :, :] = tumorVars
else:
currentMeans[normInd] = normalMeans[gauss, normInd]
tissueConfMeans[tissueConf, :] = currentMeans
if np.all(tumorArray == noTumorBit):
tissueConfVars[tissueConf, :, :] = normalVars[gauss, :, :]
else:
currentWeights = np.zeros((nrOfChannels, nrOfBrainVoxels))
currentWeights[normInd, :] = normalWeights[gauss,
normInd, :]
currentWeights[tumorInd, :] = tumorWeights[tumorInd, :]
tissueConfVars[tissueConf,:,:],nonsymm = \
distr.getVariance(data,currentWeights,currentMeans)
if nonsymm:
pdb.set_trace()
print('Estimating tissue classification')
print('')
# Calculate classification
tissueClassification = np.zeros(shape=(nrTissueConf, nrOfBrainVoxels))
LikeliProduct = np.zeros(shape=(nrOfBrainVoxels, ))
for tissueConf in range(nrTissueConf):
classNr = classes[tissueConf]
gauss = gaussians[tissueConf]
tumorArray = tumorTransitions[tissueConf, :]
LikeliProduct[:] = atlasPrior[gauss, :]
tumorAdaptedData = np.copy(data)
for channel in range(nrOfChannels):
# if no tumor in channel
if (tumorArray[channel] == noTumorBit):
LikeliProduct = LikeliProduct * tumorAtlas[noTumorBit, :]
# if tumor in channel
if (tumorArray[channel] == tumorBit):
LikeliProduct = LikeliProduct * tumorAtlas[tumorBit, :]
# Hyperintense channels are modeled by a sigmoid function
# Therefore, data values are temporally adjusted so that very high intensities
# are at least as likely to belong to the tumor as the gaussian mean intensity
if hyper[channel]:
data_tmp = tumorAdaptedData[channel, :]
data_tmp = data_tmp - np.sqrt(tumorVars[channel,
channel])
data_tmp[data_tmp < 0] = 0
data_tmp[data_tmp >
tumorMeans[channel]] = tumorMeans[channel]
tumorAdaptedData[channel, :] = data_tmp
justEval = distr.gaussianEval_multiVar(
tumorAdaptedData, tissueConfMeans[tissueConf, :],
tissueConfVars[tissueConf, :, :], True)
LikeliProduct = LikeliProduct * justEval
tissueClassification[tissueConf, :] = np.amax(
[LikeliProduct, np.zeros_like(LikeliProduct)], axis=0)
# Calculate likelihood and normalise classification
likelihood = np.fmax(np.sum(tissueClassification, axis=0), TINY)
tissueClassification = tissueClassification / \
np.tile(likelihood,(nrTissueConf,1))
logLLH = np.sum(np.log(likelihood))
logLLHArray[iteration - 1] = logLLH
relativeLogLLH = np.absolute((previousLogLLH - logLLH) / logLLH)
#converged = (relativeLogLLH<0.0001) | (iteration==maxIteration)
converged = iteration == maxIteration
print('=> logLikelihood = ' + str(logLLH))
print('=> relative change in loglikelihood = ' + str(relativeLogLLH))
print('')
# Calculating normal tissue classification and tumor classification.
print('Calculating normal tissue and tumor classification')
print('')
normalClassification = np.zeros(shape=(nrOfClasses, nrOfBrainVoxels))
gaussianClassification = np.zeros(shape=(nrOfGaussians,
nrOfBrainVoxels))
tumorClassification = np.zeros(shape=(nrOfChannels, nrOfBrainVoxels))
for tissueConf in range(nrTissueConf):
configurationProb = tissueClassification[tissueConf, :]
tumorArray = tumorTransitions[tissueConf, :]
tumorChannels = np.where(tumorArray == tumorBit)
tumorClassification[
tumorChannels, :] += configurationProb * tumorMask
classNr = classes[tissueConf]
gauss = gaussians[tissueConf]
normalClassification[classNr, :] += configurationProb
gaussianClassification[gauss, :] += configurationProb
tumorAtlas[tumorBit,:] = np.sum(tumorClassification,axis=0) / \
nrOfChannels
tumorAtlas[noTumorBit, :] = 1 - tumorAtlas[tumorBit, :]
tumorClassification = norm.window(tumorClassification, 0, 1)
normalClassification = norm.window(normalClassification, 0, 1)
gaussianClassification = norm.window(gaussianClassification, 0, 1)
oitk.writeItkImage(oitk.makeItkImage(to_matrix(tumorClassification[1,:],
DIM, playing),dataImagePrototype),
'/home/alberts/Downloads/tumor_'+\
modalities[1]+'_'+str(iteration)+'.nii')
# Visualisation
if converged:
opInst.setVisualCenter(tumor_center)
opInst.show4Dplot(
to_matrix(tumorClassification, DIM, playing), None,
'Tumor Segmentation Per Channel Iter ' + str(iteration))
opInst.show5Dplot(to_matrix(normalClassification, DIM,
playing), None,
'Tissue segmentations Iter ' + str(iteration))
for channel in range(nrOfChannels):
dataSlice = to_matrix(dataNotNorm[channel, :], DIM)
tumorSlice = to_matrix(tumorClassification[channel, :], DIM,
playing)
opInst.show2Dplot(dataSlice, tumorSlice,
'Tumor Segmentation channel ' + str(channel))
# Write tissue segmentations
if time_index is not None:
time_string = '_t' + str(time_index)
else:
time_string = ''
for tissueClass in range(nrOfClasses):
segmentTissueClass = to_matrix(normalClassification[tissueClass, :],
DIM, playing)
image = oitk.makeItkImage(segmentTissueClass, dataImagePrototype)
if tissueClass == classGM:
gmImage = image
oitk.writeItkImage(gmImage, nii_dir + 'gm' + time_string + '.nii')
if tissueClass == classWM:
wmImage = image
oitk.writeItkImage(wmImage, nii_dir + 'wm' + time_string + '.nii')
if tissueClass == classCSF:
csfImage = image
oitk.writeItkImage(image, nii_dir + 'csf' + time_string + '.nii')
# Write channel-specific tumor segmentations.
tumorChannels = np.zeros(shape=(nrOfChannels, DIM[0], DIM[1], DIM[2]))
result_files = na.getEMsegmFiles(dataFiles, time_index, inclusion)
tumorImages = []
for channel in range(nrOfChannels):
tumorChannelTmp = to_matrix(tumorClassification[channel, :], DIM,
playing)
tumorImage = oitk.makeItkImage(tumorChannelTmp, dataImagePrototype)
oitk.writeItkImage(tumorImage, result_files[channel])
print 'Tumor nii image written to ' + result_files[channel]
print 'Tumor volume ' + str(np.sum(tumorClassification[channel, :]))
tumorImages.append(tumorImage)
# Plot log likelihood curve along iterations.
logLLHArray = logLLHArray[0:iteration]
if show:
opInst.plotGraph(
np.arange(iteration) + 1, logLLHArray, 'Log likelihood')
# Print running time.
elapsed_time = time.time() - start_time
print('Convergence reached in ' + str(elapsed_time) + ' seconds')
print('')
plt.close('all')
return tumorImages, wmImage, gmImage, csfImage
@ In, name, string, the name printed out if it fails
@ In, dist, instance, the distrubtion to inquire
@ In, low, float, the lower bound of the dist
@ In, high, float, the uppper bound of the dist
@ In, numpts, int, optional, the number of integration points
@ In, tol, float, optional, the tolerance
@ Out, None
"""
xs = np.linspace(low, high, numpts)
dx = (high - low) / float(numpts)
tot = sum(dist.pdf(x) * dx for x in xs)
checkAnswer(name + ' unity integration', tot, 1, tol)
#Test module methods
print(Distributions.knownTypes())
#Test error
try:
Distributions.returnInstance("unknown", 'dud')
except:
print("error worked")
#Test Uniform
uniformElement = ET.Element("Uniform")
uniformElement.append(createElement("lowerBound", text="1.0"))
uniformElement.append(createElement("upperBound", text="3.0"))
#ET.dump(uniformElement)
uniform = Distributions.Uniform()