def setUp(self):
self.r1 = Raster('../../examples/multifact.tif')
# r1 -> r1 transition
self.r1r1 = [[
5,
5,
15,
], [
15,
10,
5,
], [
0,
15,
5,
]]
self.r2 = Raster('../../examples/multifact.tif')
self.r2.resetMask([0])
self.r2r2 = [[
0,
0,
8,
], [
8,
4,
0,
], [
100,
8,
0,
]]
self.r3 = Raster('../../examples/multifact.tif')
self.r3.resetMask([2])
def sim(self):
"""
Make 1 iteracion of simulation.
"""
# TODO: eleminate AreaAnalyst.getChangeMap() from the process
transition = self.crosstable.getCrosstable()
prediction = self.getPrediction()
state = self.getState()
new_state = state.getBand(1).copy() # New states (the result of simulation) will be stored there.
analyst = AreaAnalyst(state, prediction)
classes = analyst.classes
changes = analyst.getChangeMap().getBand(1)
# Make transition between classes according to
# number of moved pixel in crosstable
self.rangeChanged.emit(self.tr("Simulation process %p%"), len(classes) ** 2 - len(classes))
for initClass in classes:
for finalClass in classes:
if initClass == finalClass:
continue
# TODO: Calculate number of pixels to be moved via TransitoionMatrix and state raster
n = transition.getTransition(
initClass, finalClass
) # Number of pixels to be moved (constant count now).
# Find n appropriate places for transition initClass -> finalClass
class_code = analyst.encode(initClass, finalClass)
places = changes == class_code # Array of places where transitions initClass -> finalClass are occured
placesCount = np.sum(places)
if placesCount < n:
self.logMessage.emit(
self.tr("There are more transitions in the transition matrix, then the model have found")
)
n = placesCount
confidence = self.getConfidence().getBand(1)
confidence = (
confidence * places
) # The higher is number in cell, the higer is probability of transition in the cell
indices = []
for i in range(n):
index = np.unravel_index(
confidence.argmax(), confidence.shape
) # Select the cell with biggest probability
indices.append(index)
confidence[index] = -1 # Mark the cell to prevent second selection
# Now "indices" contains indices of the appropriate places,
# make transition initClass -> finalClass
for index in indices:
new_state[index] = finalClass
self.updateProgress.emit()
result = Raster()
result.create([new_state], state.getGeodata())
self.state = result
self.updatePrediction(result)
self.processFinished.emit()
def test_get_state(self):
smp = Sampler(self.state, self.factors, self.output, ns=0)
assert_array_equal(smp.get_state(self.state, 1, 1), [0, 0])
assert_array_equal(smp.get_state(self.state, 0, 0), [0, 1])
smp = Sampler(self.state, self.factors, self.output, ns=1)
res = [
0,
1,
0,
0,
0,
1,
0,
1,
0,
0,
0,
1,
1,
0,
0,
1,
0,
0,
]
assert_array_equal(smp.get_state(self.state, 1, 1), res)
inputRast = Raster('../../examples/sites.tif')
inputRast.resetMask([0])
smp = Sampler(inputRast, self.factors, self.output, ns=0)
assert_array_equal(smp.get_state(self.state, 1, 1), [0])
assert_array_equal(smp.get_state(self.state, 0, 0), [1])
class TestMCE(unittest.TestCase):
def setUp(self):
self.factor = Raster('../../examples/multifact.tif')
#~ [1,1,3]
#~ [3,2,1]
#~ [0,3,1]
self.state = Raster('../../examples/sites.tif')
self.state.resetMask(maskVals= [0])
#~ [1,2,1],
#~ [1,2,1],
#~ [0,1,2]
self.areaAnalyst = AreaAnalyst(self.state, second=None)
def test_MCE(self):
data = [
[1.0, 4.0, 6.0, 7.0],
[1.0/4, 1.0, 3.0, 4.0],
[1.0/6, 1.0/3, 1.0, 2.0],
[1.0/7, 1.0/4, 1.0/2, 1]
]
# Multiband
factor = Raster('../../examples/two_band.tif')
mce = MCE([self.factor, factor, self.factor], data, 1, 2, self.areaAnalyst)
w = mce.getWeights()
answer = [0.61682294, 0.22382863, 0.09723423, 0.06211421]
assert_almost_equal(w, answer)
# One-band
mce = MCE([self.factor, self.factor, self.factor, self.factor], data, 1, 2, self.areaAnalyst)
w = mce.getWeights()
answer = [0.61682294, 0.22382863, 0.09723423, 0.06211421]
assert_almost_equal(w, answer)
mask = [
[False, False, False],
[False, False, False],
[False, False, True]
]
p = mce.getPrediction(self.state).getBand(1)
self.assertEquals(p.dtype, np.uint8)
answer = [ # The locations where the big numbers are stored must be masked (see mask and self.state)
[1, 3, 1],
[1, 3, 1],
[100, 1, 100]
]
answer = np.ma.array(data = answer, mask = mask)
assert_almost_equal(p, answer)
c = mce.getConfidence().getBand(1)
self.assertEquals(c.dtype, np.uint8)
answer = [ # The locations where the big numbers are stored must be masked (see mask and self.state)
[sum((w*100).astype(int))/3, 0, sum((w*100).astype(int))],
[sum((w*100).astype(int)), 0, sum((w*100).astype(int))/3],
[10000, sum((w*100).astype(int)), 10000]
]
answer = np.ma.array(data = answer, mask = mask)
assert_almost_equal(c, answer)
def setUp(self):
self.r1 = Raster('examples/multifact.tif')
self.r2 = Raster('examples/sites.tif')
self.r3 = Raster('examples/two_band.tif')
# r1
data1 = np.array(
[
[1,1,3],
[3,2,1],
[0,3,1]
])
# r2
data2 = np.array(
[
[1,2,1],
[1,2,1],
[0,1,2]
])
mask = [
[False, False, False],
[False, False, False],
[False, False, False]
]
self.data1 = ma.array(data=data1, mask=mask)
self.data2 = ma.array(data=data2, mask=mask)
def _predict(self, state, calcTransitions=False):
'''
Predict the changes.
'''
try:
geodata = state.getGeodata()
rows, cols = geodata['ySize'], geodata['xSize']
self.transitionPotentials = None # Reset tr.potentials if they exist
# Get locations where self.initStateNum is occurs
band = state.getBand(1)
initStateMask = binaryzation(band, [self.initStateNum])
mask = band.mask
# Calculate summary map of factors weights
# Transition potentials:
# current implementation: potential and confidence are equal (two-class implementation)
# Confidence:
# confidence is summary map of factors scaled to 0-100, if current state = self.initState
# confidence is 0, if current state != self.initState
# Prediction:
# predicted value is a constant = areaAnalyst.encode(initStateNum, finalStateNum), if current state = self.initState
# predicted value is the transition code current_state -> current_state, if current state != self.initState
confidence = np.zeros((rows,cols), dtype=np.uint8)
weights = self.getWeights()
weightNum = 0 # Number of processed weights
for f in self.factors:
if not f.geoDataMatch(state):
raise MCEError('Geometries of the state and factor rasters are different!')
f.normalize(mode = 'maxmin')
for i in xrange(f.getBandsCount()):
band = f.getBand(i+1)
confidence = confidence + (band*weights[weightNum]*100).astype(np.uint8)
mask = np.ma.mask_or(mask, band.mask)
weightNum = weightNum + 1
confidence = confidence*initStateMask
prediction = np.copy(state.getBand(1))
for code in self.areaAnalyst.categories:
if code != self.initStateNum:
prediction[prediction==code] = self.areaAnalyst.encode(code, code)
else:
prediction[prediction==code] = self.areaAnalyst.encode(self.initStateNum, self.finalStateNum)
predicted_band = np.ma.array(data=prediction, mask=mask, dtype=np.uint8)
self.prediction = Raster()
self.prediction.create([predicted_band], geodata)
confidence_band = np.ma.array(data=confidence, mask=mask, dtype=np.uint8)
self.confidence = Raster()
self.confidence.create([confidence_band], geodata)
code = self.areaAnalyst.encode(self.initStateNum, self.finalStateNum)
self.transitionPotentials = {code: self.confidence}
except MemoryError:
self.errorReport.emit(self.tr("The system out of memory during MCE prediction"))
raise
except:
self.errorReport.emit(self.tr("An unknown error occurs during MCE prediction"))
raise
class TestAreaAnalysisManager(unittest.TestCase):
def setUp(self):
self.r1 = Raster('../../examples/multifact.tif')
# r1 -> r1 transition
self.r1r1 = [[
5,
5,
15,
], [
15,
10,
5,
], [
0,
15,
5,
]]
self.r2 = Raster('../../examples/multifact.tif')
self.r2.resetMask([0])
self.r2r2 = [[
0,
0,
8,
], [
8,
4,
0,
], [
100,
8,
0,
]]
self.r3 = Raster('../../examples/multifact.tif')
self.r3.resetMask([2])
def test_AreaAnalyst(self):
aa = AreaAnalyst(self.r1, self.r1)
raster = aa.getChangeMap()
band = raster.getBand(1)
assert_array_equal(band, self.r1r1)
# Masked raster
aa = AreaAnalyst(self.r2, self.r2)
raster = aa.getChangeMap()
band = raster.getBand(1)
assert_array_equal(band, self.r2r2)
def test_encode(self):
aa = AreaAnalyst(self.r1, self.r1)
self.assertEqual(aa.categories, [0, 1, 2, 3])
self.assertEqual(aa.encode(1, 2), 6)
for initClass in range(4):
for finalClass in range(4):
k = aa.encode(initClass, finalClass)
self.assertEqual(aa.decode(k), (initClass, finalClass))
self.assertEqual(aa.finalCodes(0), [0, 1, 2, 3])
self.assertEqual(aa.finalCodes(1), [4, 5, 6, 7])
def setUp(self):
self.reference = Raster('../../examples/data.tif')
#~ [1,1,1,1],
#~ [1,1,2,2],
#~ [2,2,2,2],
#~ [3,3,3,3]
self.simulated = Raster('../../examples/data1.tif')
class TestSimulator(unittest.TestCase):
def setUp(self):
# Raster1:
# ~ [1, 1, 3,],
# ~ [3, 2, 1,],
# ~ [0, 3, 1,]
self.raster1 = Raster("../../examples/multifact.tif")
self.raster1.resetMask([0])
self.X = np.array([[1, 2, 3], [3, 2, 1], [0, 1, 1]])
self.X = np.ma.array(self.X, mask=(self.X == 0))
self.raster2 = Raster()
self.raster2.create([self.X], self.raster1.getGeodata())
self.aa = AreaAnalyst(self.raster1, self.raster2)
self.crosstab = CrossTableManager(self.raster1, self.raster2)
# Simple model
self.model = Model(state=self.raster1)
def test_compute_table(self):
# print self.crosstab.getCrosstable().getCrosstable()
# CrossTab:
# [[ 3. 1. 0.]
# [ 0. 1. 0.]
# [ 1. 0. 2.]]
prediction = self.model.getPrediction(self.raster1)
# print prediction.getBand(1)
# prediction = [[1.0 1.0 6.0]
# [6.0 5.0 1.0]
# [-- 6.0 1.0]]
# confidence = self.model.getConfidence()
# print confidence.getBand(1)
# confidence = [[1.0 0.5 0.33]
# [0.5 0.33 0.25]
# [-- 0.25 0.2]]
result = np.array([[2.0, 1.0, 3.0], [1.0, 2.0, 1.0], [0, 3.0, 1.0]])
result = np.ma.array(result, mask=(result == 0))
simulator = Simulator(
state=self.raster1, factors=None, model=self.model, crosstable=self.crosstab
) # The model does't use factors
simulator.setIterationCount(1)
simulator.simN()
state = simulator.getState().getBand(1)
assert_array_equal(result, state)
result = np.array([[2.0, 1.0, 1.0], [2.0, 2.0, 1.0], [0, 3.0, 1.0]])
result = np.ma.array(result, mask=(result == 0))
simulator = Simulator(self.raster1, None, self.model, self.crosstab)
simulator.setIterationCount(2)
simulator.simN()
state = simulator.getState().getBand(1)
assert_array_equal(result, state)
def main(initRaster, finalRaster, factors):
print 'Start Reading Init Data...', clock()
initRaster = Raster(initRaster)
finalRaster = Raster(finalRaster)
factors = [Raster(rasterName) for rasterName in factors]
print 'Finish Reading Init Data', clock(), '\n'
print "Start Making CrossTable...", clock()
crosstab = CrossTableManager(initRaster, finalRaster)
print "Finish Making CrossTable", clock(), '\n'
# Create and Train LR Model
model = LR(ns=1)
print 'Start Setting LR Trainig Data...', clock()
model.setTrainingData(initRaster,
factors,
finalRaster,
mode='Stratified',
samples=1000)
print 'Finish Setting Trainig Data', clock(), '\n'
print 'Start LR Training...', clock()
model.train()
print 'Finish Trainig', clock(), '\n'
print 'Start LR Prediction...', clock()
predict = model.getPrediction(initRaster, factors)
filename = 'lr_predict.tiff'
try:
predict.save(filename)
finally:
os.remove(filename)
print 'Finish LR Prediction...', clock(), '\n'
# simulation
print 'Start Simulation...', clock()
simulator = Simulator(initRaster, factors, model, crosstab)
# Make 1 cycle of simulation:
simulator.simN(1)
monteCarloSim = simulator.getState() # Result of MonteCarlo simulation
errors = simulator.errorMap(finalRaster) # Risk class validation
riskFunct = simulator.getConfidence() # Risk function
# Make K cycles of simulation:
# simulator.simN(K)
try:
monteCarloSim.save('simulation_result.tiff')
errors.save('risk_validation.tiff')
riskFunct.save('risk_func.tiff')
finally:
pass
os.remove('simulation_result.tiff')
os.remove('risk_validation.tiff')
os.remove('risk_func.tiff')
print 'Finish Simulation', clock(), '\n'
print 'Done', clock()
def _predict(self, state, calcTransitions=False):
"""
Predict the changes.
"""
try:
self.rangeChanged.emit(self.tr("Initialize model %p%"), 1)
rows, cols = self.geodata["ySize"], self.geodata["xSize"]
if not self.changeMap.geoDataMatch(state):
raise WoeManagerError("Geometries of the state and changeMap rasters are different!")
prediction = np.zeros((rows, cols), dtype=np.uint8)
confidence = np.zeros((rows, cols), dtype=np.uint8)
mask = np.zeros((rows, cols), dtype=np.byte)
stateBand = state.getBand(1)
self.updateProgress.emit()
self.rangeChanged.emit(self.tr("Prediction %p%"), rows)
for r in xrange(rows):
for c in xrange(cols):
oldMax, currMax = -1000, -1000 # Small numbers
indexMax = -1 # Index of Max weight
initCat = stateBand[r, c] # Init category (state before transition)
try:
codes = self.analyst.codes(initCat) # Possible final states
for code in codes:
try: # If not all possible transitions are presented in the changeMap
map = self.woe[code] # Get WoE map of transition 'code'
except KeyError:
continue
w = map[r, c] # The weight in the (r,c)-pixel
if w > currMax:
indexMax, oldMax, currMax = code, currMax, w
prediction[r, c] = indexMax
confidence[r, c] = int(100 * (sigmoid(currMax) - sigmoid(oldMax)))
except ValueError:
mask[r, c] = 1
self.updateProgress.emit()
predicted_band = np.ma.array(data=prediction, mask=mask, dtype=np.uint8)
self.prediction = Raster()
self.prediction.create([predicted_band], self.geodata)
confidence_band = np.ma.array(data=confidence, mask=mask, dtype=np.uint8)
self.confidence = Raster()
self.confidence.create([confidence_band], self.geodata)
except MemoryError:
self.errorReport.emit(self.tr("The system out of memory during WOE prediction"))
raise
except:
self.errorReport.emit(self.tr("An unknown error occurs during WoE prediction"))
raise
finally:
self.processFinished.emit()
def _predict(self, state, calcTransitions=False):
'''
Predict the changes.
'''
try:
self.rangeChanged.emit(self.tr("Initialize model %p%"), 1)
rows, cols = self.geodata['ySize'], self.geodata['xSize']
if not self.changeMap.geoDataMatch(state):
raise WoeManagerError('Geometries of the state and changeMap rasters are different!')
prediction = np.zeros((rows,cols), dtype=np.uint8)
confidence = np.zeros((rows,cols), dtype=np.uint8)
mask = np.zeros((rows,cols), dtype=np.byte)
stateBand = state.getBand(1)
self.updateProgress.emit()
self.rangeChanged.emit(self.tr("Prediction %p%"), rows)
for r in xrange(rows):
for c in xrange(cols):
oldMax, currMax = -1000, -1000 # Small numbers
indexMax = -1 # Index of Max weight
initCat = stateBand[r,c] # Init category (state before transition)
try:
codes = self.analyst.codes(initCat) # Possible final states
for code in codes:
try: # If not all possible transitions are presented in the changeMap
map = self.woe[code] # Get WoE map of transition 'code'
except KeyError:
continue
w = map[r,c] # The weight in the (r,c)-pixel
if w > currMax:
indexMax, oldMax, currMax = code, currMax, w
prediction[r,c] = indexMax
confidence[r,c] = int(100*(sigmoid(currMax) - sigmoid(oldMax)))
except ValueError:
mask[r,c] = 1
self.updateProgress.emit()
predicted_band = np.ma.array(data=prediction, mask=mask, dtype=np.uint8)
self.prediction = Raster()
self.prediction.create([predicted_band], self.geodata)
confidence_band = np.ma.array(data=confidence, mask=mask, dtype=np.uint8)
self.confidence = Raster()
self.confidence.create([confidence_band], self.geodata)
except MemoryError:
self.errorReport.emit(self.tr("The system out of memory during WOE prediction"))
raise
except:
self.errorReport.emit(self.tr("An unknown error occurs during WoE prediction"))
raise
finally:
self.processFinished.emit()
class TestMCE(unittest.TestCase):
def setUp(self):
self.factor = Raster('../../examples/multifact.tif')
#~ [1,1,3]
#~ [3,2,1]
#~ [0,3,1]
self.state = Raster('../../examples/sites.tif')
self.state.resetMask(maskVals=[0])
#~ [1,2,1],
#~ [1,2,1],
#~ [0,1,2]
self.areaAnalyst = AreaAnalyst(self.state, second=None)
def test_MCE(self):
data = [[1.0, 4.0, 6.0, 7.0], [1.0 / 4, 1.0, 3.0, 4.0],
[1.0 / 6, 1.0 / 3, 1.0, 2.0], [1.0 / 7, 1.0 / 4, 1.0 / 2, 1]]
# Multiband
factor = Raster('../../examples/two_band.tif')
mce = MCE([self.factor, factor, self.factor], data, 1, 2,
self.areaAnalyst)
w = mce.getWeights()
answer = [0.61682294, 0.22382863, 0.09723423, 0.06211421]
assert_almost_equal(w, answer)
# One-band
mce = MCE([self.factor, self.factor, self.factor, self.factor], data,
1, 2, self.areaAnalyst)
w = mce.getWeights()
answer = [0.61682294, 0.22382863, 0.09723423, 0.06211421]
assert_almost_equal(w, answer)
mask = [[False, False, False], [False, False, False],
[False, False, True]]
p = mce.getPrediction(self.state).getBand(1)
self.assertEquals(p.dtype, np.uint8)
answer = [ # The locations where the big numbers are stored must be masked (see mask and self.state)
[1, 3, 1], [1, 3, 1], [100, 1, 100]
]
answer = np.ma.array(data=answer, mask=mask)
assert_almost_equal(p, answer)
c = mce.getConfidence().getBand(1)
self.assertEquals(c.dtype, np.uint8)
answer = [ # The locations where the big numbers are stored must be masked (see mask and self.state)
[sum((w * 100).astype(int)) / 3, 0,
sum((w * 100).astype(int))],
[sum((w * 100).astype(int)), 0,
sum((w * 100).astype(int)) / 3],
[10000, sum((w * 100).astype(int)), 10000]
]
answer = np.ma.array(data=answer, mask=mask)
assert_almost_equal(c, answer)
def test_cat2vect(self):
smp = Sampler(self.state, self.factors, self.output, ns=0)
assert_array_equal(smp.cat2vect(0), [1, 0])
assert_array_equal(smp.cat2vect(1), [0, 1])
assert_array_equal(smp.cat2vect(2), [0, 0])
inputRast = Raster('../../examples/sites.tif')
inputRast.resetMask([0])
smp = Sampler(inputRast, self.factors, self.output, ns=0)
assert_array_equal(smp.cat2vect(1), [1])
assert_array_equal(smp.cat2vect(2), [0])
def test_save(self):
try:
filename = 'temp.tiff'
self.r1.save(filename)
r2 = Raster(filename)
self.assertEqual(r2.get_dtype(), self.r1.get_dtype())
self.assertEqual(r2.getBandsCount(), self.r1.getBandsCount())
for i in range(r2.getBandsCount()):
assert_array_equal(r2.getBand(i+1), self.r1.getBand(i+1))
finally:
os.remove(filename)
def test_cat2vect(self):
smp = Sampler(self.state, self.factors, self.output, ns=0)
assert_array_equal(smp.cat2vect(0), [1, 0])
assert_array_equal(smp.cat2vect(1), [0, 1])
assert_array_equal(smp.cat2vect(2), [0, 0])
inputRast = Raster('../../examples/sites.tif')
inputRast.resetMask([0])
smp = Sampler(inputRast, self.factors, self.output, ns=0)
assert_array_equal(smp.cat2vect(1), [1])
assert_array_equal(smp.cat2vect(2), [0])
def train(self):
"""
Train the model
"""
self.transitionPotentials = {}
try:
iterCount = len(self.codes) * len(self.factors)
self.rangeChanged.emit(self.tr("Training WoE... %p%"), iterCount)
changeMap = self.changeMap.getBand(1)
for code in self.codes:
sites = binaryzation(changeMap, [code])
# Reclass factors (continuous factor -> ordinal factor)
wMap = np.ma.zeros(changeMap.shape) # The map of summary weight of the all factors
self.weights[code] = {} # Dictionary for storing wheights of every raster's band
for k in xrange(len(self.factors)):
fact = self.factors[k]
self.weights[code][k] = {} # Weights of the factor
factorW = self.weights[code][k]
if self.bins: # Get bins of the factor
bin = self.bins[k]
if (bin != None) and fact.getBandsCount() != len(bin):
raise WoeManagerError("Count of bins list for multiband factor is't equal to band count!")
else:
bin = None
for i in range(1, fact.getBandsCount() + 1):
band = fact.getBand(i)
if bin and bin[i - 1]: #
band = reclass(band, bin[i - 1])
band, sites = masks_identity(band, sites, dtype=np.uint8) # Combine masks of the rasters
woeRes = woe(
band, sites, self.unit_cell
) # WoE for the 'code' (initState->finalState) transition and current 'factor'.
weights = woeRes["map"]
wMap = wMap + weights
factorW[i] = woeRes["weights"]
self.updateProgress.emit()
# Reclassification finished => set WoE coefficients
self.woe[code] = wMap # WoE for all factors and the transition code.
# Potentials are WoE map rescaled to 0--100 percents
band = (sigmoid(wMap) * 100).astype(np.uint8)
p = Raster()
p.create([band], self.geodata)
self.transitionPotentials[code] = p
gc.collect()
except MemoryError:
self.errorReport.emit("The system out of memory during WoE trainig")
raise
except:
self.errorReport.emit(self.tr("An unknown error occurs during WoE trainig"))
raise
finally:
self.processFinished.emit()
def setUp(self):
self.factor = Raster('../../examples/multifact.tif')
#~ [1,1,3]
#~ [3,2,1]
#~ [0,3,1]
self.sites = Raster('../../examples/sites.tif')
#~ [1,2,1],
#~ [1,2,1],
#~ [0,1,2]
self.sites.resetMask(maskVals=[0])
self.mask = [[
False,
False,
False,
], [
False,
False,
False,
], [
True,
False,
False,
]]
fact = [[
1,
1,
3,
], [
3,
2,
1,
], [
0,
3,
1,
]]
site = [[
False,
True,
False,
], [
False,
True,
False,
], [
False,
False,
True,
]]
self.factraster = ma.array(data=fact, mask=self.mask, dtype=np.int)
self.sitesraster = ma.array(data=site, mask=self.mask, dtype=np.bool)
def setUp(self):
self.factor = Raster('../../examples/multifact.tif')
#~ [1,1,3]
#~ [3,2,1]
#~ [0,3,1]
self.state = Raster('../../examples/sites.tif')
self.state.resetMask(maskVals=[0])
#~ [1,2,1],
#~ [1,2,1],
#~ [0,1,2]
self.areaAnalyst = AreaAnalyst(self.state, second=None)
def errorMap(self, answer):
'''
Create map of correct and incorrect prediction.
This function compares the known answer and the result of predicting procedure,
correct pixel is marked as 0.
'''
state = self.getState()
b = state.getBand(1)
a = answer.getBand(1)
diff = (a - b).astype(np.int16)
result = Raster()
result.create([diff], state.getGeodata())
return result
def errorMap(self, answer):
'''
Create map of correct and incorrect prediction.
This function compares the known answer and the result of predicting procedure,
correct pixel is marked as 0.
'''
state = self.getState()
b = state.getBand(1)
a = answer.getBand(1)
diff = (a-b).astype(np.int16)
result = Raster()
result.create([diff], state.getGeodata())
return result
def train(self):
'''
Train the model
'''
self.transitionPotentials = {}
try:
iterCount = len(self.codes)*len(self.factors)
self.rangeChanged.emit(self.tr("Training WoE... %p%"), iterCount)
changeMap = self.changeMap.getBand(1)
for code in self.codes:
sites = binaryzation(changeMap, [code])
# Reclass factors (continuous factor -> ordinal factor)
wMap = np.ma.zeros(changeMap.shape) # The map of summary weight of the all factors
self.weights[code] = {} # Dictionary for storing wheights of every raster's band
for k in xrange(len(self.factors)):
fact = self.factors[k]
self.weights[code][k] = {} # Weights of the factor
factorW = self.weights[code][k]
if self.bins: # Get bins of the factor
bin = self.bins[k]
if (bin != None) and fact.getBandsCount() != len(bin):
raise WoeManagerError("Count of bins list for multiband factor is't equal to band count!")
else: bin = None
for i in range(1, fact.getBandsCount()+1):
band = fact.getBand(i)
if bin and bin[i-1]: #
band = reclass(band, bin[i-1])
band, sites = masks_identity(band, sites, dtype=np.uint8) # Combine masks of the rasters
woeRes = woe(band, sites, self.unit_cell) # WoE for the 'code' (initState->finalState) transition and current 'factor'.
weights = woeRes['map']
wMap = wMap + weights
factorW[i] = woeRes['weights']
self.updateProgress.emit()
# Reclassification finished => set WoE coefficients
self.woe[code]=wMap # WoE for all factors and the transition code.
# Potentials are WoE map rescaled to 0--100 percents
band = (sigmoid(wMap)*100).astype(np.uint8)
p = Raster()
p.create([band], self.geodata)
self.transitionPotentials[code] = p
gc.collect()
except MemoryError:
self.errorReport.emit('The system out of memory during WoE trainig')
raise
except:
self.errorReport.emit(self.tr("An unknown error occurs during WoE trainig"))
raise
finally:
self.processFinished.emit()
def test_getStat(self):
reference = Raster('../../examples/data.tif')
simulated = Raster('../../examples/data1.tif')
reference.resetMask([2])
simulated.resetMask([2])
eb = EBudget(reference, simulated)
stat = eb.getStat(nIter=3)
ans0 = {
'NoNo': 0.5,
'NoMed': (5.0 * 5 / 8 + 3.0 * 3 / 8) / 8,
'MedMed': 4.0 / 8,
'MedPer': 1.0,
'PerPer': 1.0
}
for k in stat[0].keys():
np.testing.assert_almost_equal(stat[0][k], ans0[k])
ans1 = {
'NoNo': 0.5,
'NoMed': (5.0 / 8 + 5.0 / 32 + 3.0 / 16 + 3.0 / 32) / 2,
'MedMed': 4.0 / 8,
'MedPer': 1.0,
'PerPer': 1.0
}
for k in stat[1].keys():
np.testing.assert_almost_equal(stat[0][k], ans1[k])
def test_Mask(self):
reference = Raster('../../examples/data.tif')
simulated = Raster('../../examples/data1.tif')
reference.resetMask([2])
simulated.resetMask([2])
eb = EBudget(reference, simulated)
W = eb.W
S1 = weightedSum(eb.Sj[1], W)
S3 = weightedSum(eb.Sj[3], W)
np.testing.assert_almost_equal(S1, 5.0 / 8)
np.testing.assert_almost_equal(S3, 3.0 / 8)
noNo = eb.NoNo()
np.testing.assert_almost_equal(noNo, 0.5)
noM = eb.NoMed()
np.testing.assert_almost_equal(noM, (5.0 * 5 / 8 + 3.0 * 3 / 8) / 8)
medM = eb.MedMed()
np.testing.assert_almost_equal(medM, 4.0 / 8)
medP = eb.MedPer()
np.testing.assert_almost_equal(medP, 1.0)
def setUp(self):
self.factors = [Raster('../../examples/multifact.tif')]
self.output = Raster('../../examples/sites.tif')
#~ sites.tif is 1-band 3x3 raster:
#~ [1,2,1],
#~ [1,2,1],
#~ [0,1,2]
self.factors2 = [Raster('../../examples/multifact.tif'), Raster('../../examples/multifact.tif')]
self.factors3 = [Raster('../../examples/two_band.tif')]
self.factors4 = [Raster('../../examples/two_band.tif'), Raster('../../examples/multifact.tif')]
self.output1 = Raster('../../examples/data.tif')
self.state1 = self.output1
self.factors1 = [Raster('../../examples/fact16.tif')]
def setUp(self):
self.factor = Raster('../../examples/multifact.tif')
#~ [1,1,3]
#~ [3,2,1]
#~ [0,3,1]
self.sites = Raster('../../examples/sites.tif')
#~ [1,2,1],
#~ [1,2,1],
#~ [0,1,2]
self.sites.resetMask(maskVals= [0])
self.mask = [
[False, False, False,],
[False, False, False,],
[True, False, False,]
]
fact = [
[1, 1, 3,],
[3, 2, 1,],
[0, 3, 1,]
]
site = [
[False, True, False,],
[False, True, False,],
[False, False, True,]
]
self.factraster = ma.array(data = fact, mask=self.mask, dtype=np.int)
self.sitesraster = ma.array(data = site, mask=self.mask, dtype=np.bool)
def test_roundBands(self):
rast = Raster('examples/multifact.tif')
rast.bands = rast.bands * 0.1
rast.roundBands()
answer = [[[
0,
0,
0,
], [0, 0, 0], [0, 0, 0]]]
assert_array_equal(answer, rast.bands)
rast = Raster('examples/multifact.tif')
rast.bands = rast.bands * 1.1
rast.roundBands(decimals=1)
answer = np.array([[[1.1, 1.1, 3.3], [3.3, 2.2, 1.1], [0.0, 3.3,
1.1]]])
assert_array_equal(answer, rast.bands)
def makeChangeMap(self):
f, s = self.first, self.second
rows, cols = self.geodata['ySize'], self.geodata['xSize']
band = np.zeros([rows, cols])
self.rangeChanged.emit(self.tr("Creating change map %p%"), rows)
for i in xrange(rows):
for j in xrange(cols):
if not f.mask[i,j]:
r = f[i,j]
c = s[i,j]
band[i, j] = self.encode(r, c)
self.updateProgress.emit()
bands = [np.ma.array(data = band, mask = f.mask)]
raster = Raster()
raster.create(bands, self.geodata)
self.processFinished.emit(raster)
self.changeMap = raster
def test_get_state(self):
smp = Sampler(self.state, self.factors, self.output, ns=0)
assert_array_equal(smp.get_state(self.state, 1,1), [0, 0])
assert_array_equal(smp.get_state(self.state, 0,0), [0, 1])
smp = Sampler(self.state, self.factors, self.output, ns=1)
res = [ 0, 1, 0, 0, 0, 1,
0, 1, 0, 0, 0, 1,
1, 0, 0, 1, 0, 0,
]
assert_array_equal(smp.get_state(self.state, 1,1), res)
inputRast = Raster('../../examples/sites.tif')
inputRast.resetMask([0])
smp = Sampler(inputRast, self.factors, self.output, ns=0)
assert_array_equal(smp.get_state(self.state, 1,1), [0])
assert_array_equal(smp.get_state(self.state, 0,0), [1])
class TestLRManager (unittest.TestCase):
def setUp(self):
self.output = Raster('../../examples/multifact.tif')
#~ [1,1,3]
#~ [3,2,1]
#~ [0,3,1]
self.output.resetMask([0])
self.state = self.output
self.factors = [Raster('../../examples/sites.tif'), Raster('../../examples/sites.tif')]
#~ [1,2,1],
#~ [1,2,1],
#~ [0,1,2]
self.output1 = Raster('../../examples/data.tif')
self.state1 = self.output1
self.factors1 = [Raster('../../examples/fact16.tif')]
def test_LR(self):
data = [
[3.0, 1.0, 3.0],
[3.0, 1.0, 3.0],
[0, 3.0, 1.0]
]
result = np.ma.array(data = data, mask = (data==0))
lr = LR(ns=0) # 3-class problem
lr.setTrainingData(self.state, self.factors, self.output)
lr.train()
predict = lr.getPrediction(self.state, self.factors)
predict = predict.getBand(1)
assert_array_equal(predict, result)
lr = LR(ns=1) # Two-class problem (it's because of boundary effect)
lr.setTrainingData(self.state1, self.factors1, self.output1)
lr.train()
predict = lr.getPrediction(self.state1, self.factors1)
predict = predict.getBand(1)
data = [
[0.0, 0.0, 0.0, 0.0],
[0.0, 1.0, 2.0, 0.0],
[0.0, 2.0, 2.0, 0.0],
[0.0, 0.0, 0.0, 0.0],
]
result = np.ma.array(data = data, mask = (data==0))
assert_array_equal(predict, result)
def setUp(self):
self.factor = Raster('../../examples/multifact.tif')
#~ [1,1,3]
#~ [3,2,1]
#~ [0,3,1]
self.state = Raster('../../examples/sites.tif')
self.state.resetMask(maskVals= [0])
def setUp(self):
self.r1 = Raster('../../examples/multifact.tif')
# r1 -> r1 transition
self.r1r1 = [
[5, 5, 15,],
[15, 10, 5, ],
[0, 15, 5, ]
]
self.r2 = Raster('../../examples/multifact.tif')
self.r2.resetMask([0])
self.r2r2 = [
[0, 0, 8,],
[8, 4, 0,],
[100, 8, 0,]
]
self.r3 = Raster('../../examples/multifact.tif')
self.r3.resetMask([2])
def _predict(self, state, factors):
'''
Calculate output and confidence rasters using MLP model and input rasters
@param state Raster of the current state (classes) values.
@param factors List of the factor rasters (predicting variables).
'''
geodata = state.getGeodata()
rows, cols = geodata['ySize'], geodata['xSize']
for r in factors:
if not state.geoDataMatch(r):
raise MlpManagerError('Geometries of the input rasters are different!')
# Normalize factors before prediction:
for f in factors:
f.normalize(mode = 'mean')
predicted_band = np.zeros([rows, cols])
confidence_band = np.zeros([rows, cols])
sampler = Sampler(state, factors, ns=self.ns)
mask = state.getBand(1).mask.copy()
for i in xrange(rows):
for j in xrange(cols):
if not mask[i,j]:
input = sampler.get_inputs(state, factors, i,j)
if input != None:
out = self.getOutput(input)
# Get index of the biggest output value as the result
biggest = max(out)
res = list(out).index(biggest)
predicted_band[i, j] = self.classlist[res]
confidence = self.outputConfidence(out)
confidence_band[i, j] = confidence
else: # Input sample is incomplete => mask this pixel
mask[i, j] = True
predicted_bands = [np.ma.array(data = predicted_band, mask = mask)]
confidence_bands = [np.ma.array(data = confidence_band, mask = mask)]
self.prediction = Raster()
self.prediction.create(predicted_bands, geodata)
self.confidence = Raster()
self.confidence.create(confidence_bands, geodata)
def _predict(self, state):
'''
Predict the changes.
'''
geodata = self.changeMap.getGeodata()
rows, cols = geodata['ySize'], geodata['xSize']
if not self.changeMap.geoDataMatch(state):
raise WoeManagerError('Geometries of the state and changeMap rasters are different!')
prediction = np.zeros((rows,cols))
confidence = np.zeros((rows,cols))
mask = np.zeros((rows,cols))
woe = self.getWoe()
stateBand = state.getBand(1)
for r in xrange(rows):
for c in xrange(cols):
oldMax, currMax = -1000, -1000 # Small numbers
indexMax = -1 # Index of Max weight
initClass = stateBand[r,c] # Init class (state before transition)
try:
codes = self.analyst.codes(initClass) # Possible final states
for code in codes:
try: # If not all possible transitions are presented in the changeMap
map = woe[code] # Get WoE map of transition 'code'
except KeyError:
continue
w = map[r,c] # The weight in the (r,c)-pixel
if w > currMax:
indexMax, oldMax, currMax = code, currMax, w
decode = self.analyst.decode(indexMax) # Get init & final classes (initState, finalState)
prediction[r,c] = decode[1] # final class
confidence[r,c] = sigmoid(currMax) - sigmoid(oldMax)
except ValueError:
mask[r,c] = 1
predicted_band = np.ma.array(data=prediction, mask=mask)
self.prediction = Raster()
self.prediction.create([predicted_band], geodata)
confidence_band = np.ma.array(data=confidence, mask=mask)
self.confidence = Raster()
self.confidence.create([confidence_band], geodata)
class TestAreaAnalysisManager (unittest.TestCase):
def setUp(self):
self.r1 = Raster('../../examples/multifact.tif')
# r1 -> r1 transition
self.r1r1 = [
[5, 5, 15,],
[15, 10, 5, ],
[0, 15, 5, ]
]
self.r2 = Raster('../../examples/multifact.tif')
self.r2.resetMask([0])
self.r2r2 = [
[0, 0, 8,],
[8, 4, 0,],
[100, 8, 0,]
]
self.r3 = Raster('../../examples/multifact.tif')
self.r3.resetMask([2])
def test_AreaAnalyst(self):
aa = AreaAnalyst(self.r1, self.r1)
raster = aa.getChangeMap()
band = raster.getBand(1)
assert_array_equal(band, self.r1r1)
# Masked raster
aa = AreaAnalyst(self.r2, self.r2)
raster = aa.getChangeMap()
band = raster.getBand(1)
assert_array_equal(band, self.r2r2)
def test_encode(self):
aa = AreaAnalyst(self.r1, self.r1)
self.assertEqual(aa.classes, [0,1,2,3])
self.assertEqual(aa.encode(1,2), 6)
for initClass in range(4):
for finalClass in range(4):
k = aa.encode(initClass, finalClass)
self.assertEqual(aa.decode(k), (initClass, finalClass))
self.assertEqual(aa.finalCodes(0), [0,1,2,3])
self.assertEqual(aa.finalCodes(1), [4,5,6,7])
def _predict(self, state):
'''
Predict the changes.
'''
geodata = state.getGeodata()
rows, cols = geodata['ySize'], geodata['xSize']
# Get locations where self.initStateNum is occurs
band = state.getBand(1)
initStateMask = binaryzation(band, [self.initStateNum])
mask = band.mask
# Calculate summary map of factors weights
# Confidence:
# confidence is summary map of factors, if current state = self.initState
# confidence is 0, if current state != self.initState
# Prediction:
# predicted value is a constant = self.finalStateNum, if current state = self.initState
# predicted value is current state, if current state != self.initState
confidence = np.zeros((rows,cols))
weights = self.getWeights()
weightNum = 0 # Number of processed weights
for f in self.factors:
if not f.geoDataMatch(state):
raise MCEError('Geometries of the state and factor rasters are different!')
f.normalize(mode = 'maxmin')
for i in xrange(f.getBandsCount()):
band = f.getBand(i+1)
confidence = confidence + band*weights[weightNum]
mask = np.ma.mask_or(mask, band.mask)
weightNum = weightNum + 1
confidence = confidence*initStateMask
prediction = np.copy(state.getBand(1))
prediction = np.logical_not(initStateMask) * prediction
prediction = prediction + initStateMask*self.finalStateNum
predicted_band = np.ma.array(data=prediction, mask=mask)
self.prediction = Raster()
self.prediction.create([predicted_band], geodata)
confidence_band = np.ma.array(data=confidence, mask=mask)
self.confidence = Raster()
self.confidence.create([confidence_band], geodata)
def setUp(self):
self.factor = Raster("../../examples/multifact.tif")
self.sites = Raster("../../examples/sites.tif")
self.sites.resetMask(maskVals=[0])
mask = [[False, False, False], [False, False, False], [True, False, False]]
fact = [[1, 1, 3], [3, 2, 1], [0, 3, 1]]
site = [[False, True, False], [False, True, False], [False, False, True]]
self.factraster = ma.array(data=fact, mask=mask, dtype=np.int)
self.sitesraster = ma.array(data=site, mask=mask, dtype=np.bool)
def setUp(self):
# Raster1:
# ~ [1, 1, 3,],
# ~ [3, 2, 1,],
# ~ [0, 3, 1,]
self.raster1 = Raster("../../examples/multifact.tif")
self.raster1.resetMask([0])
self.X = np.array([[1, 2, 3], [3, 2, 1], [0, 1, 1]])
self.X = np.ma.array(self.X, mask=(self.X == 0))
self.raster2 = Raster()
self.raster2.create([self.X], self.raster1.getGeodata())
self.aa = AreaAnalyst(self.raster1, self.raster2)
self.crosstab = CrossTableManager(self.raster1, self.raster2)
# Simple model
self.model = Model(state=self.raster1)
def setUp(self):
self.factor = Raster('../../examples/multifact.tif')
#~ [1,1,3]
#~ [3,2,1]
#~ [0,3,1]
self.state = Raster('../../examples/sites.tif')
self.state.resetMask(maskVals= [0])
#~ [1,2,1],
#~ [1,2,1],
#~ [0,1,2]
self.areaAnalyst = AreaAnalyst(self.state, second=None)
def test_Mask(self):
reference = Raster('../../examples/data.tif')
simulated = Raster('../../examples/data1.tif')
reference.resetMask([2])
simulated.resetMask([2])
eb = EBudget(reference, simulated)
W = eb.W
S1 = weightedSum(eb.Sj[1], W)
S3 = weightedSum(eb.Sj[3], W)
np.testing.assert_almost_equal(S1, 5.0/8)
np.testing.assert_almost_equal(S3, 3.0/8)
noNo = eb.NoNo()
np.testing.assert_almost_equal(noNo, 0.5)
noM = eb.NoMed()
np.testing.assert_almost_equal(noM, (5.0*5/8 + 3.0*3/8)/8)
medM= eb.MedMed()
np.testing.assert_almost_equal(medM, 4.0/8)
medP= eb.MedPer()
np.testing.assert_almost_equal(medP, 1.0)
class Model(object):
"""
Simple predicting model for Simulator tests
"""
def __init__(self, state):
self._predict(state)
def getConfidence(self):
return self.confidence
def getPrediction(self, state, factors=None, calcTransitions=False):
self._predict(state, factors)
return self.prediction
def _predict(self, state, factors=None, calcTransitions=False):
geodata = state.getGeodata()
band = state.getBand(1)
rows, cols = geodata["ySize"], geodata["xSize"]
# Let the new state is: 1 -> 2, 2- >3, 3 -> 1, then
# the prediction is 1->1, 2->5, 3->6
predicted_band = np.copy(band)
predicted_band[band == 1] = 1.0
predicted_band[band == 2] = 5.0
predicted_band[band == 3] = 6.0
# Let the confidence is 1/(1+row+col), where row is row number of the cell, col is column number of the cell.
confidence_band = np.zeros([rows, cols])
for i in xrange(cols):
for j in xrange(rows):
confidence_band[i, j] = 1.0 / (1 + i + j)
predicted_bands = [np.ma.array(data=predicted_band, mask=band.mask)]
confidence_bands = [np.ma.array(data=confidence_band, mask=band.mask)]
self.prediction = Raster()
self.prediction.create(predicted_bands, state.geodata)
self.confidence = Raster()
self.confidence.create(confidence_bands, state.geodata)
class Model(object):
'''
Simple predicting model for Simulator tests
'''
def __init__(self, state):
self.state = state
self._predict(state)
def getConfidence(self):
return self.confidence
def getPrediction(self, state, factors=None):
self._predict(state, factors)
return self.prediction
def _predict(self, state, factors = None):
geodata = self.state.getGeodata()
band = state.getBand(1)
rows, cols = geodata['ySize'], geodata['xSize']
# Let the prediction is: 1 -> 2, 2- >3, 3 -> 1
predicted_band = np.copy(band)
predicted_band[band == 1] = 2
predicted_band[band == 2] = 3
predicted_band[band == 3] = 1
# Let the confidence is 1/(1+row+col), where row is row number of the cell, col is column number of the cell.
confidence_band = np.zeros([rows, cols])
for i in xrange(cols):
for j in xrange(rows):
confidence_band[i,j] = 1.0/(1+i+j)
predicted_bands = [np.ma.array(data = predicted_band, mask = band.mask)]
confidence_bands = [np.ma.array(data = confidence_band, mask = band.mask)]
self.prediction = Raster()
self.prediction.create(predicted_bands, state.geodata)
self.confidence = Raster()
self.confidence.create(confidence_bands, state.geodata)
class Model(object):
'''
Simple predicting model for Simulator tests
'''
def __init__(self, state):
self._predict(state)
def getConfidence(self):
return self.confidence
def getPrediction(self, state, factors=None, calcTransitions=False):
self._predict(state, factors)
return self.prediction
def _predict(self, state, factors = None, calcTransitions=False):
geodata = state.getGeodata()
band = state.getBand(1)
rows, cols = geodata['ySize'], geodata['xSize']
# Let the new state is: 1 -> 2, 2- >3, 3 -> 1, then
# the prediction is 1->1, 2->5, 3->6
predicted_band = np.copy(band)
predicted_band[band == 1] = 1.0
predicted_band[band == 2] = 5.0
predicted_band[band == 3] = 6.0
# Let the confidence is 1/(1+row+col), where row is row number of the cell, col is column number of the cell.
confidence_band = np.zeros([rows, cols])
for i in xrange(cols):
for j in xrange(rows):
confidence_band[i,j] = 1.0/(1+i+j)
predicted_bands = [np.ma.array(data = predicted_band, mask = band.mask)]
confidence_bands = [np.ma.array(data = confidence_band, mask = band.mask)]
self.prediction = Raster()
self.prediction.create(predicted_bands, state.geodata)
self.confidence = Raster()
self.confidence.create(confidence_bands, state.geodata)
def _predict(self, state, factors = None):
geodata = self.state.getGeodata()
band = state.getBand(1)
rows, cols = geodata['ySize'], geodata['xSize']
# Let the prediction is: 1 -> 2, 2- >3, 3 -> 1
predicted_band = np.copy(band)
predicted_band[band == 1] = 2
predicted_band[band == 2] = 3
predicted_band[band == 3] = 1
# Let the confidence is 1/(1+row+col), where row is row number of the cell, col is column number of the cell.
confidence_band = np.zeros([rows, cols])
for i in xrange(cols):
for j in xrange(rows):
confidence_band[i,j] = 1.0/(1+i+j)
predicted_bands = [np.ma.array(data = predicted_band, mask = band.mask)]
confidence_bands = [np.ma.array(data = confidence_band, mask = band.mask)]
self.prediction = Raster()
self.prediction.create(predicted_bands, state.geodata)
self.confidence = Raster()
self.confidence.create(confidence_bands, state.geodata)
def setUp(self):
# Raster1:
#~ [1, 1, 3,],
#~ [3, 2, 1,],
#~ [0, 3, 1,]
self.raster1 = Raster('../../examples/multifact.tif')
self.raster1.resetMask([0])
self.X = np.array([
[1, 2, 3],
[3, 2, 1],
[0, 1, 1]
])
self.X = np.ma.array(self.X, mask=(self.X == 0))
self.raster2 = Raster()
self.raster2.create([self.X], self.raster1.getGeodata())
self.aa = AreaAnalyst(self.raster1, self.raster2)
self.crosstab = CrossTableManager(self.raster1, self.raster2)
# Simple model
self.model = Model(state=self.raster1)
def makeChangeMap(self):
rows, cols = self.geodata['ySize'], self.geodata['xSize']
band = np.zeros([rows, cols], dtype=np.int16)
f, s = self.first, self.second
if self.initRaster == None:
checkPersistent = False
else:
checkPersistent = True
t = self.initRaster.getBand(1)
raster = None
try:
self.rangeChanged.emit(self.tr("Creating change map %p%"), rows)
for i in xrange(rows):
for j in xrange(cols):
if (f.mask.shape == ()) or (not f.mask[i,j]):
r = f[i,j]
c = s[i,j]
# Percistent category is the category that is constant for all three rasters
if checkPersistent and (r==c) and (r==t[i,j]):
band[i, j] = self.persistentCategoryCode
else:
band[i, j] = self.encode(r, c)
self.updateProgress.emit()
bands = [np.ma.array(data = band, mask = f.mask, dtype=np.int16)]
raster = Raster()
raster.create(bands, self.geodata)
self.changeMap = raster
except MemoryError:
self.errorReport.emit(self.tr("The system out of memory during change map creating"))
raise
except:
self.errorReport.emit(self.tr("An unknown error occurs during change map creating"))
raise
finally:
self.processFinished.emit(raster)
def setUp(self):
self.output = Raster('../../examples/multifact.tif')
#~ [1,1,3]
#~ [3,2,1]
#~ [0,3,1]
self.output.resetMask([0])
self.state = self.output
self.factors = [Raster('../../examples/sites.tif'), Raster('../../examples/sites.tif')]
#~ [1,2,1],
#~ [1,2,1],
#~ [0,1,2]
self.output1 = Raster('../../examples/data.tif')
self.state1 = self.output1
self.factors1 = [Raster('../../examples/fact16.tif')]
def test_WoeManager(self):
aa = AreaAnalyst(self.sites, self.sites)
w1 = WoeManager([self.factor], aa)
p = w1.getPrediction(self.sites).getBand(1)
assert_array_equal(p, self.sites.getBand(1))
initState = Raster("../../examples/data.tif")
finalState = Raster("../../examples/data1.tif")
aa = AreaAnalyst(initState, finalState)
w = WoeManager([initState], aa)
p = w.getPrediction(initState).getBand(1)
# Calculate by hands:
# 1->1 transition raster:
r11 = [[1, 1, 0, 0], [0, 1, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]]
# 1->2 raster:
r12 = [[0, 0, 1, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]]
# 1->3 raster:
r13 = [[0, 0, 0, 1], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]]
# 2->1
r21 = [[0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]]
# 2->2
r22 = [[0, 0, 0, 0], [0, 0, 1, 0], [0, 0, 0, 0], [0, 0, 0, 0]]
# 2->3
r23 = [[0, 0, 0, 0], [0, 0, 0, 1], [1, 1, 1, 1], [0, 0, 0, 0]]
# 3->1
r31 = [[0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [1, 1, 0, 0]]
# 3->2
r32 = [[0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 1]]
# 3->3
r33 = [[0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 1, 0]]
geodata = initState.getGeodata()
sites = {"11": r11, "12": r12, "13": r13, "21": r21, "22": r22, "23": r23, "31": r31, "32": r32, "33": r33}
woeDict = {} # WoE of transitions
for k in sites.keys(): #
if k != "21": # !!! r21 is zero
x = Raster()
x.create([np.ma.array(data=sites[k])], geodata)
sites[k] = x
woeDict[k] = woe(initState.getBand(1), x.getBand(1))
# w1max = np.maximum(woeDict['11'], woeDict['12'], woeDict['13'])
# w2max = np.maximum(woeDict['22'], woeDict['23'])
# w3max = np.maximum(woeDict['31'], woeDict['32'], woeDict['33'])
# Answer is index of finalClass that maximizes weights of transiotion initClass -> finalClass
answer = [[1, 1, 1, 1], [1, 1, 3, 3], [3, 3, 3, 3], [1, 1, 1, 1]]
assert_array_equal(p, answer)
w = WoeManager([initState], aa, bins={0: [[2]]})
p = w.getPrediction(initState).getBand(1)
def test_save(self):
try:
filename = 'temp.tiff'
self.r1.save(filename)
r2 = Raster(filename)
self.assertEqual(r2.get_dtype(), self.r1.get_dtype())
self.assertEqual(r2.getBandsCount(), self.r1.getBandsCount())
for i in range(r2.getBandsCount()):
assert_array_equal(r2.getBand(i + 1), self.r1.getBand(i + 1))
finally:
os.remove(filename)
def setUp(self):
self.r1 = Raster('examples/multifact.tif')
self.r2 = Raster('examples/sites.tif')
self.r3 = Raster('examples/two_band.tif')
# r1
data1 = np.array([[1, 1, 3], [3, 2, 1], [0, 3, 1]])
# r2
data2 = np.array([[1, 2, 1], [1, 2, 1], [0, 1, 2]])
mask = [[False, False, False], [False, False, False],
[False, False, False]]
self.data1 = ma.array(data=data1, mask=mask)
self.data2 = ma.array(data=data2, mask=mask)
def test_coarse(self):
reference = Raster('../../examples/data.tif')
simulated = Raster('../../examples/data1.tif')
reference.resetMask([2])
simulated.resetMask([2])
eb = EBudget(reference, simulated)
eb.coarse(2)
# W
answer = np.array([[1.0, 0.25], [0.5, 0.25]])
np.testing.assert_array_equal(eb.W, answer)
# Rj
answer1 = np.array([[1.0, 1.0], [0, 0]])
answer3 = np.array([[0, 0], [1.0, 1.0]])
ans = {1: answer1, 3: answer3}
np.testing.assert_equal(eb.Rj, ans)
# Sj
answer1 = np.array([[3.0 / 4, 0.0], [1.0, 0]])
answer3 = np.array([[1.0 / 4, 1.0], [0, 1.0]])
ans = {1: answer1, 3: answer3}
np.testing.assert_equal(eb.Sj, ans)
eb.coarse(2)
# W
answer = np.array([[0.5]])
np.testing.assert_array_equal(eb.W, answer)
# Rj
answer1 = np.array([[(1 + 1.0 / 4) / 2]])
answer3 = np.array([[(1.0 / 2 + 1.0 / 4) / 2]])
ans = {1: answer1, 3: answer3}
np.testing.assert_equal(eb.Rj, ans)
# Sj
answer1 = np.array([[(3.0 / 4 + 0.5) / 2]])
answer3 = np.array([[(1.0 / 4 + 1.0 / 4 + 1.0 / 4) / 2]])
ans = {1: answer1, 3: answer3}
np.testing.assert_equal(eb.Sj, ans)
def setUp(self):
self.output = Raster('../../examples/multifact.tif')
#~ [1,1,3]
#~ [3,2,1]
#~ [0,3,1]
self.output.resetMask([0])
self.state = self.output
self.factors = [
Raster('../../examples/sites.tif'),
Raster('../../examples/sites.tif')
]
#~ [1,2,1],
#~ [1,2,1],
#~ [0,1,2]
self.output1 = Raster('../../examples/data.tif')
self.state1 = self.output1
self.factors1 = [Raster('../../examples/fact16.tif')]
def main(initRaster, finalRaster, factors):
print 'Start Reading Init Data...', clock()
initRaster = Raster(initRaster)
finalRaster = Raster(finalRaster)
factors = [Raster(rasterName) for rasterName in factors]
print 'Finish Reading Init Data', clock(), '\n'
print "Start Making CrossTable...", clock()
crosstab = CrossTableManager(initRaster, finalRaster)
#print crosstab.getTransitionStat()
print "Finish Making CrossTable", clock(), '\n'
# Create and Train Analyst
print 'Start creating AreaAnalyst...', clock()
analyst = AreaAnalyst(initRaster, finalRaster)
print 'Finish creating AreaAnalyst ...', clock(), '\n'
print 'Start Making Change Map...', clock()
analyst = AreaAnalyst(initRaster, finalRaster)
changeMap = analyst.getChangeMap()
print 'Finish Making Change Map', clock(), '\n'
#~ # Create and Train ANN Model
model = MlpManager(ns=1)
model.createMlp(initRaster, factors, changeMap, [10])
print 'Start Setting MLP Trainig Data...', clock()
model.setTrainingData(initRaster,
factors,
changeMap,
mode='Stratified',
samples=1000)
print 'Finish Setting Trainig Data', clock(), '\n'
print 'Start MLP Training...', clock()
model.train(20, valPercent=20)
print 'Finish Trainig', clock(), '\n'
# print 'Start ANN Prediction...', clock()
# predict = model.getPrediction(initRaster, factors, calcTransitions=True)
# confidence = model.getConfidence()
# potentials = model.getTransitionPotentials()
#~ # Create and Train LR Model
#~ model = LR(ns=0)
#~ print 'Start Setting LR Trainig Data...', clock()
#~ model.setState(initRaster)
#~ model.setFactors(factors)
#~ model.setOutput(changeMap)
#~ model.setMode('Stratified')
#~ model.setSamples(100)
#~ model.setTrainingData()
#~ print 'Finish Setting Trainig Data', clock(), '\n'
#~ print 'Start LR Training...', clock()
#~ model.train()
#~ print 'Finish Trainig', clock(), '\n'
#~
#~ print 'Start LR Prediction...', clock()
#~ predict = model.getPrediction(initRaster, factors, calcTransitions=True)
#~ print 'Finish LR Prediction...', clock(), '\n'
# Create and Train WoE Model
# print 'Start creating AreaAnalyst...', clock()
# analyst = AreaAnalyst(initRaster, finalRaster)
# print 'Finish creating AreaAnalyst ...', clock(), '\n'
# print 'Start creating WoE model...', clock()
# bins = {0: [[1000, 3000]], 1: [[200, 500, 1500]]}
# model = WoeManager(factors, analyst, bins= bins)
# model.train()
# print 'Finish creating WoE model...', clock(), '\n'
#~ # Create and Train MCE Model
#~ print 'Start creating MCE model...', clock()
#~ matrix = [
#~ [1, 6],
#~ [1.0/6, 1]
#~ ]
#~ model = MCE(factors, matrix, 2, 3, analyst)
#~ print 'Finish creating MCE model...', clock(), '\n'
# predict = model.getPrediction(initRaster, factors, calcTransitions=True)
# confidence = model.getConfidence()
# potentials = model.getTransitionPotentials()
# filename = 'predict.tif'
# confname = 'confidence.tif'
# trans_prefix='trans_'
# try:
# predict.save(filename)
# confidence.save(confname)
# if potentials != None:
# for k,v in potentials.iteritems():
# map = v.save(trans_prefix+str(k) + '.tif')
# finally:
# os.remove(filename)
# #pass
# print 'Finish Saving...', clock(), '\n'
# simulation
print 'Start Simulation...', clock()
simulator = Simulator(initRaster, factors, model, crosstab)
# Make 1 cycle of simulation:
simulator.setIterationCount(1)
simulator.simN()
monteCarloSim = simulator.getState() # Result of MonteCarlo simulation
errors = simulator.errorMap(finalRaster) # Risk class validation
riskFunct = simulator.getConfidence() # Risk function
try:
monteCarloSim.save('simulation_result.tiff')
errors.save('risk_validation.tiff')
riskFunct.save('risk_func.tiff')
finally:
pass
# os.remove('simulation_result.tiff')
# os.remove('risk_validation.tiff')
# os.remove('risk_func.tiff')
print 'Finish Simulation', clock(), '\n'
print 'Done', clock()
class TestLRManager(unittest.TestCase):
def setUp(self):
self.output = Raster('../../examples/multifact.tif')
#~ [1,1,3]
#~ [3,2,1]
#~ [0,3,1]
self.output.resetMask([0])
self.state = self.output
self.factors = [
Raster('../../examples/sites.tif'),
Raster('../../examples/sites.tif')
]
#~ [1,2,1],
#~ [1,2,1],
#~ [0,1,2]
self.output1 = Raster('../../examples/data.tif')
self.state1 = self.output1
self.factors1 = [Raster('../../examples/fact16.tif')]
def test_LR(self):
#~ data = [
#~ [3.0, 1.0, 3.0],
#~ [3.0, 1.0, 3.0],
#~ [0, 3.0, 1.0]
#~ ]
#~ result = np.ma.array(data = data, mask = (data==0))
lr = LR(ns=0) # 3-class problem
lr.setState(self.state)
lr.setFactors(self.factors)
lr.setOutput(self.output)
lr.setTrainingData()
lr.train()
predict = lr.getPrediction(self.state, self.factors)
predict = predict.getBand(1)
assert_array_equal(predict, self.output.getBand(1))
lr = LR(ns=1) # Two-class problem (it's because of boundary effect)
lr.setState(self.state1)
lr.setFactors(self.factors1)
lr.setOutput(self.output1)
lr.setTrainingData()
lr.train()
predict = lr.getPrediction(self.state1,
self.factors1,
calcTransitions=True)
predict = predict.getBand(1)
self.assertEquals(predict.dtype, np.uint8)
data = [
[0.0, 0.0, 0.0, 0.0],
[0.0, 1.0, 2.0, 0.0],
[0.0, 2.0, 2.0, 0.0],
[0.0, 0.0, 0.0, 0.0],
]
result = np.ma.array(data=data, mask=(data == 0))
assert_array_equal(predict, result)
# Confidence is zero
confid = lr.getConfidence()
self.assertEquals(confid.getBand(1).dtype, np.uint8)
# Transition Potentials
potentials = lr.getTransitionPotentials()
cats = self.output.getBandGradation(1)
for cat in [1.0, 2.0]:
map = potentials[cat]
self.assertEquals(map.getBand(1).dtype, np.uint8)
def setUp(self):
self.init = Raster('../../examples/init.tif', maskVals={1: [255]})
self.final = Raster('../../examples/final.tif', maskVals={1: [255]})
class TestMlpManager (unittest.TestCase):
def setUp(self):
self.factors = [Raster('../../examples/multifact.tif')]
self.output = Raster('../../examples/sites.tif')
#~ sites.tif is 1-band 3x3 raster:
#~ [1,2,1],
#~ [1,2,1],
#~ [0,1,2]
self.factors2 = [Raster('../../examples/multifact.tif'), Raster('../../examples/multifact.tif')]
self.factors3 = [Raster('../../examples/two_band.tif')]
self.factors4 = [Raster('../../examples/two_band.tif'), Raster('../../examples/multifact.tif')]
self.output1 = Raster('../../examples/data.tif')
self.state1 = self.output1
self.factors1 = [Raster('../../examples/fact16.tif')]
def test_MlpManager(self):
mng = MlpManager(ns=1)
mng.createMlp(self.output, self.factors2, self.output, [10])
assert_array_equal(mng.getMlpTopology(), [2*9 + 2*9, 10, 3])
mng = MlpManager()
mng.createMlp(self.output, self.factors, self.output, [10])
assert_array_equal(mng.getMlpTopology(), [2*1 + 1*1, 10, 3])
def test_setTrainingData(self):
mng = MlpManager()
mng.createMlp(self.output, self.factors, self.output, [10])
stat = self.factors[0].getBandStat(1) # mean & std
m,s = stat['mean'], stat['std']
mng.setTrainingData(self.output, self.factors, self.output, shuffle=False)
min, max = mng.sigmin, mng.sigmax
data = np.array(
[
((0,3), [0, 1], (1.0-m)/s, [min, max, min]),
((1,3), [0, 0], (1.0-m)/s, [min, min, max]),
((2,3), [0, 1], (3.0-m)/s, [min, max, min]),
((0,2), [0, 1], (3.0-m)/s, [min, max, min]),
((1,2), [0, 0], (2.0-m)/s, [min, min, max]),
((2,2), [0, 1], (1.0-m)/s, [min, max, min]),
((0,1), [1, 0], (0.0-m)/s, [max, min, min]),
((1,1), [0, 1], (3.0-m)/s, [min, max, min]),
((2,1), [0, 0], (1.0-m)/s, [min, min, max]),
],
dtype=[('coords', float, 2),('state', float, (2,)), ('factors', float, (1,)), ('output', float, 3)]
)
self.assertEqual(mng.data.shape, (9,))
for i in range(len(data)):
assert_array_equal(data[i]['coords'], mng.data[i]['coords'])
assert_array_equal(data[i]['factors'], mng.data[i]['factors'])
assert_array_equal(data[i]['output'], mng.data[i]['output'])
# two input rasters
mng = MlpManager(ns=1)
mng.createMlp(self.output, self.factors2, self.output, [10])
mng.setTrainingData(self.output, self.factors2, self.output)
data = [
{
'factors': (np.array([ 1., 1., 3., 3., 2., 1., 0., 3., 1.,
1., 1., 3., 3., 2., 1., 0., 3., 1.]) - stat['mean'])/stat['std'],
'output': np.array([min, min, max]),
'state': np.array([0,1, 0,0, 0,1, 0,1, 0,0, 0,1, 1,0, 0,1, 0,0])
}
]
self.assertEqual(mng.data.shape, (1,))
assert_array_equal(data[0]['factors'], mng.data[0]['factors'])
assert_array_equal(data[0]['output'], mng.data[0]['output'])
assert_array_equal(data[0]['state'], mng.data[0]['state'])
# Multiband input
mng = MlpManager(ns=1)
mng.createMlp(self.output, self.factors3, self.output, [10])
stat1 = self.factors3[0].getBandStat(1) # mean & std
m1,s1 = stat1['mean'], stat1['std']
stat2 = self.factors3[0].getBandStat(2) # mean & std
m2,s2 = stat2['mean'], stat2['std']
mng.setTrainingData(self.output, self.factors3, self.output)
data = [
{
'factors': np.array([ (1.-m1)/s1, (2.-m1)/s1, (1.-m1)/s1, (1.-m1)/s1, (2.-m1)/s1, (1.-m1)/s1, (0.-m1)/s1, (1.-m1)/s1, (2.-m1)/s1,
(1.-m2)/s2, (1.-m2)/s2, (3.-m2)/s2, (3.-m2)/s2, (2.-m2)/s2, (1.-m2)/s2, (0.-m2)/s2, (3.-m2)/s2, (1.-m2)/s2]),
'output': np.array([min, min, max]),
'state': np.array([0,1, 0,0, 0,1, 0,1, 0,0, 0,1, 1,0, 0,1, 0,0])
}
]
self.assertEqual(mng.data.shape, (1,))
assert_array_equal(data[0]['factors'], mng.data[0]['factors'])
assert_array_equal(data[0]['output'], mng.data[0]['output'])
assert_array_equal(data[0]['state'], mng.data[0]['state'])
# Complex case:
mng = MlpManager(ns=1)
mng.createMlp(self.output, self.factors4, self.output, [10])
stat1 = self.factors4[0].getBandStat(1) # mean & std
m1,s1 = stat1['mean'], stat1['std']
stat2 = self.factors4[0].getBandStat(2) # mean & std
m2,s2 = stat2['mean'], stat2['std']
stat3 = self.factors4[1].getBandStat(1)
m3,s3 = stat3['mean'], stat2['std']
mng.setTrainingData(self.output, self.factors4, self.output)
data = [
{
'factors': np.array([ (1.-m1)/s1, (2.-m1)/s1, (1.-m1)/s1, (1.-m1)/s1, (2.-m1)/s1, (1.-m1)/s1, (0.-m1)/s1, (1.-m1)/s1, (2.-m1)/s1,
(1.-m2)/s2, (1.-m2)/s2, (3.-m2)/s2, (3.-m2)/s2, (2.-m2)/s2, (1.-m2)/s2, (0.-m2)/s2, (3.-m2)/s2, (1.-m2)/s2,
(1.-m3)/s3, (1.-m3)/s3, (3.-m3)/s3, (3.-m3)/s3, (2.-m3)/s3, (1.-m3)/s3, (0.-m3)/s3, (3.-m3)/s3, (1.-m3)/s3]),
'output': np.array([min, min, max]),
'state': np.array([0,1, 0,0, 0,1, 0,1, 0,0, 0,1, 1,0, 0,1, 0,0])
}
]
self.assertEqual(mng.data.shape, (1,))
assert_array_equal(data[0]['factors'], mng.data[0]['factors'])
assert_array_equal(data[0]['output'], mng.data[0]['output'])
assert_array_equal(data[0]['state'], mng.data[0]['state'])
def test_train(self):
mng = MlpManager()
mng.createMlp(self.output, self.factors, self.output, [10])
mng.setTrainingData(self.output, self.factors, self.output)
mng.train(1, valPercent=50)
val = mng.getMinValError()
tr = mng.getTrainError()
mng.train(20, valPercent=50, continue_train=True)
self.assertGreaterEqual(val, mng.getMinValError())
mng = MlpManager(ns=1)
mng.createMlp(self.state1, self.factors1, self.output1, [10])
mng.setTrainingData(self.state1, self.factors1, self.output1)
mng.train(1, valPercent=20)
predict = mng.getPrediction(self.state1, self.factors1)
mask = predict.getBand(1).mask
self.assertTrue(not all(mask.flatten()))
def test_predict(self):
mng = MlpManager()
mng.createMlp(self.output, self.factors, self.output, [10])
weights = mng.copyWeights()
# Set weights=0
# then the output must be sigmoid(0)
layers = []
for layer in weights:
shape = layer.shape
layers.append(np.zeros(shape))
mng.setMlpWeights(layers)
mng._predict(self.output, self.factors)
# Prediction ( the output must be sigmoid(0) )
raster = mng.getPrediction(self.output, self.factors, calcTransitions=True)
assert_array_equal(raster.getBand(1), sigmoid(0)*np.ones((3,3)))
# Confidence is zero
confid = mng.getConfidence()
self.assertEquals(confid.getBand(1).dtype, np.uint8)
assert_array_equal(confid.getBand(1), np.zeros((3,3)))
# Transition Potentials (is (sigmoid(0) - sigmin)/sigrange )
potentials = mng.getTransitionPotentials()
cats = self.output.getBandGradation(1)
for cat in cats:
map = potentials[cat]
self.assertEquals(map.getBand(1).dtype, np.uint8)
assert_array_equal(map.getBand(1), 50*np.ones((3,3)))
def test_normalize(self):
multifact = [
[1, 1, 3],
[3, 2, 1],
[0, 3, 1],
]
# Normalize using std and mean
r1 = Raster('examples/multifact.tif')
r1.normalize()
r1.denormalize()
assert_array_equal(r1.getBand(1), multifact)
# Normalize using min and max
r1 = Raster('examples/multifact.tif')
r1.normalize(mode='maxmin')
r1.denormalize()
assert_array_equal(r1.getBand(1), multifact)
# Two normalization procedures
r1 = Raster('examples/multifact.tif')
r1.normalize()
r1.normalize(mode='maxmin')
r1.denormalize()
assert_array_equal(r1.getBand(1), multifact)
r1 = Raster('examples/multifact.tif')
r1.normalize(mode='maxmin')
r1.normalize()
r1.denormalize()
assert_array_equal(r1.getBand(1), multifact)
def test_isContinues(self):
rast = Raster('examples/multifact.tif')
self.assertFalse(rast.isCountinues(bandNo=1))
rast = Raster('examples/dist_roads.tif')
self.assertTrue(rast.isCountinues(bandNo=1))
