def runPycno(idsdataset,
polygonvaluesdataset,
rastergeo,
niter=100,
converge=0.001,
tempfileid=None):
print('| PYCNOPHYLACTIC INTERPOLATION')
pycnodataset = massp.runMassPreserving(idsdataset, polygonvaluesdataset,
rastergeo, tempfileid)[0]
oldpycnodataset = pycnodataset
idpolvalues = npu.polygonValuesByID(polygonvaluesdataset, idsdataset)
pycnomask = idsdataset
pycnomask[~np.isnan(pycnomask)] = 1
for it in range(1, niter + 1):
print('| - Iteration', it)
# Calculate the mean of the cells in the 3 by 3 neighborhood
mask = np.array([[0, 1, 0], [1, 0, 1], [0, 1, 0]])
mask = np.expand_dims(mask, axis=2)
pycnodataset = ndimage.generic_filter(pycnodataset,
np.nanmean,
footprint=mask,
mode='constant',
cval=np.NaN)
# Summarizes the values within each polygon
stats = npu.statsByID(pycnodataset, idsdataset, 'sum')
# Divide the true polygon values by the estimated polygon values (= ratio)
polygonratios = {
k: idpolvalues[k] / stats[k]
for k in stats.keys() & idpolvalues
}
# Multiply ratio by the different cells within each polygon
for polid in polygonratios:
pycnodataset[idsdataset == polid] = (
pycnodataset[idsdataset == polid] * polygonratios[polid])
pycnodataset = pycnodataset * pycnomask
# Check if the algorithm has converged
error = np.nanmean(abs(pycnodataset - oldpycnodataset))
rangeds = np.nanmax(oldpycnodataset) - np.nanmin(oldpycnodataset)
stopcrit = converge # * rangeds
print('Error:', error)
if ((it > 1) and (error < stopcrit)):
break
else:
oldpycnodataset = pycnodataset
if tempfileid:
tempfile = 'tempfilepycno_' + tempfileid + '.tif'
osgu.writeRaster(pycnodataset[:, :, 0], rastergeo, tempfile)
return pycnodataset, rastergeo
def runMassPreserving(idsdataset,
polygonvaluesdataset,
rastergeo,
tempfileid=None):
print('|| MASS-PRESERVING AREAL WEIGHTING')
unique, counts = np.unique(idsdataset[~np.isnan(idsdataset)],
return_counts=True)
counts = dict(zip(unique, counts))
for polid in counts:
idsdataset[idsdataset == polid] = counts[polid]
masspdataset = polygonvaluesdataset / idsdataset
if tempfileid:
tempfile = 'tempfilemp_' + tempfileid + '.tif'
osgu.writeRaster(masspdataset[:, :, 0], rastergeo, tempfile)
return masspdataset, rastergeo
def runDissever(fshape,
ancdatasets,
ancnames,
yraster=None,
rastergeo=None,
perc2evaluate=0.1,
poly2agg='NUTSIII',
method='lm',
cnnmod='unet',
patchsize=7,
epochspi=1,
batchsize=1024,
lrate=0.001,
filters=[2, 4, 8, 16, 32],
lweights=[1 / 2, 1 / 2],
extdataset=None,
p=1,
min_iter=3,
max_iter=100,
converge=2,
tempfileid=None,
verbose=False):
print('| DISSEVER')
nrowsds = ancdatasets[:, :, 0].shape[1]
ncolsds = ancdatasets[:, :, 0].shape[0]
idsdataset = osgu.ogr2raster(fshape,
attr='ID',
template=[rastergeo, nrowsds, ncolsds])[0]
if yraster:
disseverdataset, rastergeo = osgu.readRaster(yraster)
idpolvalues = npu.statsByID(disseverdataset, idsdataset, 'sum')
else:
polygonvaluesdataset, rastergeo = osgu.ogr2raster(
fshape, attr='VALUE', template=[rastergeo, nrowsds, ncolsds])
idpolvalues = npu.polygonValuesByID(polygonvaluesdataset, idsdataset)
disseverdataset, rastergeo = pycno.runPycno(idsdataset,
polygonvaluesdataset,
rastergeo, tempfileid)
if method.startswith('ap'):
adjpolygons = gput.computeNeighbors(fshape,
polyg=poly2agg,
verbose=True)
dissmask = np.copy(idsdataset)
dissmask[~np.isnan(dissmask)] = 1
ancvarsmask = np.dstack([dissmask] * ancdatasets.shape[2])
olddisseverdataset = disseverdataset
if method.endswith('cnn'):
# Replace NaN's by 0
ancdatasets[np.isnan(ancdatasets)] = 0
disseverdataset[np.isnan(disseverdataset)] = 0
padd = True if cnnmod.endswith('lenet') else False
cnnobj = ku.compilecnnmodel(
cnnmod, [patchsize, patchsize, ancdatasets.shape[2]],
lrate,
filters=filters,
lweights=lweights)
if cnnmod.startswith('2r'):
ancpatches = ku.createpatches(ancdatasets,
patchsize,
padding=padd,
stride=1)
disspatches = ku.createpatches(disseverdataset,
patchsize,
padding=padd,
stride=1,
disseverds=True)
else:
ancpatches = ku.createpatches(ancdatasets,
patchsize,
padding=padd,
stride=1)
disspatches = ku.createpatches(disseverdataset,
patchsize,
padding=padd,
stride=1,
disseverds=True)
disspatcheso = disspatches
# Replace NaN zones by Nan
disseverdataset = disseverdataset * dissmask
ancdatasets = ancdatasets * ancvarsmask
strat = True
if method.startswith('ap'):
if strat:
print('entrou')
numpols = round((perc2evaluate / 2) * (len(adjpolygons)))
pquartilles = gput.polygonsQuartilles(idpolvalues, numpols)
adjpairs = gput.createAdjPairs(adjpolygons,
perc2evaluate / 2,
strat=pquartilles,
verbose=True)
else:
adjpairs = gput.createAdjPairs(adjpolygons,
perc2evaluate / 2,
verbose=True)
initadjpairs = adjpairs
if (verbose):
print('Fixed adjacent pairs (' + str(len(initadjpairs)) + ') -',
initadjpairs)
lasterror = -np.inf
lowesterror = np.inf
history = []
for k in range(1, max_iter + 1):
print('| - Iteration', k)
previdpolvalues = idpolvalues
if method.startswith('ap'):
if strat:
print('started createadjpairs')
pquartilles = gput.polygonsQuartilles(idpolvalues, numpols)
adjpairs = gput.createAdjPairs(adjpolygons,
perc2evaluate / 2,
strat=pquartilles,
initadjpairs=initadjpairs,
verbose=True)
print('ended createadjpairs')
else:
# adjpairs = gput.createAdjPairs(adjpolygons, 0, initadjpairs=initadjpairs)
adjpairs = gput.createAdjPairs(adjpolygons,
perc2evaluate / 2,
initadjpairs=initadjpairs)
if (verbose):
print('Adjacent pairs (' + str(len(adjpairs)) + ') -',
adjpairs)
newshape, newpairs = gput.dissolvePairs(fshape, adjpairs)
idsdataset = osgu.ogr2raster(
newshape, attr='ID', template=[rastergeo, nrowsds, ncolsds])[0]
idsdataset2e = osgu.ogr2raster(
fshape, attr='ID', template=[rastergeo, nrowsds, ncolsds])[0]
osgu.removeShapefile(newshape)
# Edit idpolvalues
pairslist = [item for t in adjpairs for item in t]
ids2keep = list(set(previdpolvalues.keys()) - set(pairslist))
idpolvalues = dict((k, previdpolvalues[k]) for k in ids2keep)
idpolvalues2e = dict((k, previdpolvalues[k]) for k in pairslist)
for newpair in newpairs:
idpolvalues[newpair] = previdpolvalues[newpairs[newpair][
0]] + previdpolvalues[newpairs[newpair][1]]
if method.endswith('cnn'):
npatches = ancpatches.shape[0]
nsamples = round(npatches * p)
idsamples = np.random.choice(npatches, nsamples, replace=False)
print('| -- Fitting the model')
historycnn = caret.fitcnn(ancpatches,
disspatches,
disspatcheso,
idsamples,
cnnmod=cnnmod,
cnnobj=cnnobj,
epochs=epochspi,
batchsize=batchsize,
extdataset=extdataset)
loss = historycnn.history['loss']
print('| -- Predicting new values')
disseverdataset = caret.predictcnn(cnnobj,
cnnmod,
ancpatches,
disspatches,
disseverdataset.shape,
batchsize=batchsize)
else:
disssize = disseverdataset.shape[0] * disseverdataset.shape[1]
nsamples = round(disssize * p)
idsamples = np.random.choice(disssize, nsamples, replace=False)
print('| -- Fitting the model')
mod = caret.fit(ancdatasets, disseverdataset, idsamples, method)
print('| -- Predicting new values')
# Replace NaN's by 0 and predict
ancdatasets[np.isnan(ancdatasets)] = 0
disseverdataset = caret.predict(mod, ancdatasets)
disseverdataset = np.expand_dims(disseverdataset, axis=2)
# Replace NaN zones by Nan
disseverdataset = disseverdataset * dissmask
ancdatasets = ancdatasets * ancvarsmask
# Avoid negative counts
disseverdataset[disseverdataset < 0] = 0
if verbose: print('| -- Computing adjustement factor')
stats = npu.statsByID(disseverdataset, idsdataset, 'sum')
if method.startswith('ap'):
stats2e = npu.statsByID(disseverdataset, idsdataset2e, 'sum')
stats2e = dict((k, stats2e[k]) for k in pairslist)
# Horrible hack, avoid division by 0
for s in stats:
stats[s] = stats[s] + 0.00001
polygonratios = {
k: idpolvalues[k] / stats[k]
for k in stats.keys() & idpolvalues
}
idpolvalues = previdpolvalues
#print('** Mean of adj. factor', np.array(list(polygonratios.values())).mean())
for polid in polygonratios:
disseverdataset[idsdataset == polid] = (
disseverdataset[idsdataset == polid] * polygonratios[polid])
osgu.writeRaster(
disseverdataset[:, :, 0], rastergeo,
'tempfiledissever_livebirths' + 'pfv_' + str(k) + 'it.tif')
if method.startswith('ap'):
# Compute disaggregation error for 10% of the municipalities
actual2e = list(idpolvalues2e.values())
predicted2e = list(stats2e.values())
range2e = max(actual2e) - min(actual2e)
nrmse2e = np.sqrt(metrics.mean_squared_error(
actual2e, predicted2e)) / range2e
nmae2e = metrics.mean_absolute_error(actual2e,
predicted2e) / range2e
filenamenrmse2e = 'nrmse2epfv_livebirths' + '.txt'
filenamenmae2e = 'nmae2epfv_livebirths' + '.txt'
if os.path.exists(filenamenrmse2e):
with open(filenamenrmse2e, 'a') as myfile:
myfile.write(str(k) + ': ' + str(nrmse2e) + '\n')
else:
with open(filenamenrmse2e, 'w+') as myfile:
myfile.write(str(k) + ': ' + str(nrmse2e) + '\n')
if os.path.exists(filenamenmae2e):
with open(filenamenmae2e, 'a') as myfile:
myfile.write(str(k) + ': ' + str(nmae2e) + '\n')
else:
with open(filenamenmae2e, 'w+') as myfile:
myfile.write(str(k) + ': ' + str(nmae2e) + '\n')
# Check if the algorithm has converged
error = np.nanmean(abs(disseverdataset - olddisseverdataset))
history.append([error, loss
]) if method.endswith('cnn') else history.append(error)
errorrat = (error / lasterror) if lasterror > 0 else np.inf
lasterror = error
print('Error:', error)
if k >= min_iter:
if errorrat < converge:
if error < lowesterror:
lowesterror = error
lowesterriter = k
lowesterrdisseverdataset = np.copy(disseverdataset)
else:
if k == min_iter:
lowesterriter = k
else:
disseverdataset = lowesterrdisseverdataset
print('Retaining model fitted at iteration', lowesterriter)
break
olddisseverdataset = disseverdataset
if tempfileid:
tempfile = 'tempfiledissever_' + tempfileid + '.tif'
osgu.writeRaster(disseverdataset, rastergeo, tempfile)
return disseverdataset[:, :, 0], rastergeo, history
for indicator in indicators:
print('--- Running dasymetric mapping for the indicator', indicator[0])
fshape = os.path.join('Shapefiles', (indicator[2] + '13_cont.shp'))
fcsv = os.path.join('Statistics', indicator[0], (indicator[2] + '.csv'))
fancdataset = 'Rasters/ghs15_200m.tif'
osgu.removeAttrFromShapefile(fshape, ['ID', 'VALUE'])
osgu.addAttr2Shapefile(fshape,
fcsv, [indicator[2].upper()],
encoding='UTF-8')
tempfileid = None #None
idsdataset = osgu.ogr2raster(fshape,
attr='ID',
template=[rastergeo, nrowsds, ncolsds])[0]
polygonvaluesdataset, rastergeo = osgu.ogr2raster(
fshape, attr='VALUE', template=[rastergeo, nrowsds, ncolsds])
ancdataset = osgu.readRaster(fancdataset)[0]
tddataset, rastergeo = dm.rundasymmapping(idsdataset,
polygonvaluesdataset,
ancdataset,
rastergeo,
tempfileid=tempfileid)
osgu.writeRaster(tddataset[:, :, 0], rastergeo,
'td_' + indicator[0] + '.tif')
osgu.removeAttrFromShapefile(fshape, ['ID', 'VALUE'])
print('\n--- Running dissever leveraging a', meth, 'model')
seed(42)
set_random_seed(42)
dissdataset, rastergeo, history = dissever.runDissever(fshape, ancdatasets, ancnames=ancnames,
min_iter=3, max_iter=4,
perc2evaluate=p2e, poly2agg=admboundary2.upper(),
rastergeo=rastergeo, method=meth,
p=psamp,
yraster=yraster,
verbose=True)
print('- Writing raster to disk...')
niter = len(history)-1
osgu.writeRaster(dissdataset, rastergeo,
'dissever_tdnl_' + str(p2e) + '_'
+ indicator + '_' + meth + '_psamp=' + str(psamp) + '_' + str(niter) + 'iter.tif')
else:
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
session = tf.Session(config=config)
for cnnm in cnnmodel:
for lweights in lossweights:
for filt in filters:
for psize in patchsize:
for epiter in epochspi:
for bsize in batchsize:
for lrate in learningrate:
nrowsds = ds.shape[1]
ncolsds = ds.shape[0]
for indicator in indicators:
print('--- Running pycnophylactic interpolation for the indicator',
indicator[0])
fshape = os.path.join('Shapefiles', (indicator[2] + '13_cont.shp'))
fcsv = os.path.join('Statistics', indicator[0], (indicator[2] + '.csv'))
osgu.removeAttrFromShapefile(fshape, ['ID', 'VALUE'])
osgu.addAttr2Shapefile(fshape,
fcsv, [indicator[1].upper(), indicator[2].upper()],
encoding='UTF-8')
tempfileid = indicator[0]
idsdataset = osgu.ogr2raster(fshape,
attr='ID',
template=[rastergeo, nrowsds, ncolsds])[0]
polygonvaluesdataset, rastergeo = osgu.ogr2raster(
fshape, attr='VALUE', template=[rastergeo, nrowsds, ncolsds])
pycnodataset, rastergeo = pycno.runPycno(idsdataset,
polygonvaluesdataset,
rastergeo,
tempfileid=tempfileid)
osgu.writeRaster(pycnodataset[:, :, 0], rastergeo,
'pycnointerpolation_' + indicator[0] + '.tif')
osgu.removeAttrFromShapefile(fshape, ['ID', 'VALUE'])