def runPycno(idsdataset, polygonvaluesdataset, rastergeo, niter=100, converge=0.001, tempfileid=None): print('| PYCNOPHYLACTIC INTERPOLATION') pycnodataset = massp.runMassPreserving(idsdataset, polygonvaluesdataset, rastergeo, tempfileid)[0] oldpycnodataset = pycnodataset idpolvalues = npu.polygonValuesByID(polygonvaluesdataset, idsdataset) pycnomask = idsdataset pycnomask[~np.isnan(pycnomask)] = 1 for it in range(1, niter + 1): print('| - Iteration', it) # Calculate the mean of the cells in the 3 by 3 neighborhood mask = np.array([[0, 1, 0], [1, 0, 1], [0, 1, 0]]) mask = np.expand_dims(mask, axis=2) pycnodataset = ndimage.generic_filter(pycnodataset, np.nanmean, footprint=mask, mode='constant', cval=np.NaN) # Summarizes the values within each polygon stats = npu.statsByID(pycnodataset, idsdataset, 'sum') # Divide the true polygon values by the estimated polygon values (= ratio) polygonratios = { k: idpolvalues[k] / stats[k] for k in stats.keys() & idpolvalues } # Multiply ratio by the different cells within each polygon for polid in polygonratios: pycnodataset[idsdataset == polid] = ( pycnodataset[idsdataset == polid] * polygonratios[polid]) pycnodataset = pycnodataset * pycnomask # Check if the algorithm has converged error = np.nanmean(abs(pycnodataset - oldpycnodataset)) rangeds = np.nanmax(oldpycnodataset) - np.nanmin(oldpycnodataset) stopcrit = converge # * rangeds print('Error:', error) if ((it > 1) and (error < stopcrit)): break else: oldpycnodataset = pycnodataset if tempfileid: tempfile = 'tempfilepycno_' + tempfileid + '.tif' osgu.writeRaster(pycnodataset[:, :, 0], rastergeo, tempfile) return pycnodataset, rastergeo
def runMassPreserving(idsdataset, polygonvaluesdataset, rastergeo, tempfileid=None): print('|| MASS-PRESERVING AREAL WEIGHTING') unique, counts = np.unique(idsdataset[~np.isnan(idsdataset)], return_counts=True) counts = dict(zip(unique, counts)) for polid in counts: idsdataset[idsdataset == polid] = counts[polid] masspdataset = polygonvaluesdataset / idsdataset if tempfileid: tempfile = 'tempfilemp_' + tempfileid + '.tif' osgu.writeRaster(masspdataset[:, :, 0], rastergeo, tempfile) return masspdataset, rastergeo
def runDissever(fshape, ancdatasets, ancnames, yraster=None, rastergeo=None, perc2evaluate=0.1, poly2agg='NUTSIII', method='lm', cnnmod='unet', patchsize=7, epochspi=1, batchsize=1024, lrate=0.001, filters=[2, 4, 8, 16, 32], lweights=[1 / 2, 1 / 2], extdataset=None, p=1, min_iter=3, max_iter=100, converge=2, tempfileid=None, verbose=False): print('| DISSEVER') nrowsds = ancdatasets[:, :, 0].shape[1] ncolsds = ancdatasets[:, :, 0].shape[0] idsdataset = osgu.ogr2raster(fshape, attr='ID', template=[rastergeo, nrowsds, ncolsds])[0] if yraster: disseverdataset, rastergeo = osgu.readRaster(yraster) idpolvalues = npu.statsByID(disseverdataset, idsdataset, 'sum') else: polygonvaluesdataset, rastergeo = osgu.ogr2raster( fshape, attr='VALUE', template=[rastergeo, nrowsds, ncolsds]) idpolvalues = npu.polygonValuesByID(polygonvaluesdataset, idsdataset) disseverdataset, rastergeo = pycno.runPycno(idsdataset, polygonvaluesdataset, rastergeo, tempfileid) if method.startswith('ap'): adjpolygons = gput.computeNeighbors(fshape, polyg=poly2agg, verbose=True) dissmask = np.copy(idsdataset) dissmask[~np.isnan(dissmask)] = 1 ancvarsmask = np.dstack([dissmask] * ancdatasets.shape[2]) olddisseverdataset = disseverdataset if method.endswith('cnn'): # Replace NaN's by 0 ancdatasets[np.isnan(ancdatasets)] = 0 disseverdataset[np.isnan(disseverdataset)] = 0 padd = True if cnnmod.endswith('lenet') else False cnnobj = ku.compilecnnmodel( cnnmod, [patchsize, patchsize, ancdatasets.shape[2]], lrate, filters=filters, lweights=lweights) if cnnmod.startswith('2r'): ancpatches = ku.createpatches(ancdatasets, patchsize, padding=padd, stride=1) disspatches = ku.createpatches(disseverdataset, patchsize, padding=padd, stride=1, disseverds=True) else: ancpatches = ku.createpatches(ancdatasets, patchsize, padding=padd, stride=1) disspatches = ku.createpatches(disseverdataset, patchsize, padding=padd, stride=1, disseverds=True) disspatcheso = disspatches # Replace NaN zones by Nan disseverdataset = disseverdataset * dissmask ancdatasets = ancdatasets * ancvarsmask strat = True if method.startswith('ap'): if strat: print('entrou') numpols = round((perc2evaluate / 2) * (len(adjpolygons))) pquartilles = gput.polygonsQuartilles(idpolvalues, numpols) adjpairs = gput.createAdjPairs(adjpolygons, perc2evaluate / 2, strat=pquartilles, verbose=True) else: adjpairs = gput.createAdjPairs(adjpolygons, perc2evaluate / 2, verbose=True) initadjpairs = adjpairs if (verbose): print('Fixed adjacent pairs (' + str(len(initadjpairs)) + ') -', initadjpairs) lasterror = -np.inf lowesterror = np.inf history = [] for k in range(1, max_iter + 1): print('| - Iteration', k) previdpolvalues = idpolvalues if method.startswith('ap'): if strat: print('started createadjpairs') pquartilles = gput.polygonsQuartilles(idpolvalues, numpols) adjpairs = gput.createAdjPairs(adjpolygons, perc2evaluate / 2, strat=pquartilles, initadjpairs=initadjpairs, verbose=True) print('ended createadjpairs') else: # adjpairs = gput.createAdjPairs(adjpolygons, 0, initadjpairs=initadjpairs) adjpairs = gput.createAdjPairs(adjpolygons, perc2evaluate / 2, initadjpairs=initadjpairs) if (verbose): print('Adjacent pairs (' + str(len(adjpairs)) + ') -', adjpairs) newshape, newpairs = gput.dissolvePairs(fshape, adjpairs) idsdataset = osgu.ogr2raster( newshape, attr='ID', template=[rastergeo, nrowsds, ncolsds])[0] idsdataset2e = osgu.ogr2raster( fshape, attr='ID', template=[rastergeo, nrowsds, ncolsds])[0] osgu.removeShapefile(newshape) # Edit idpolvalues pairslist = [item for t in adjpairs for item in t] ids2keep = list(set(previdpolvalues.keys()) - set(pairslist)) idpolvalues = dict((k, previdpolvalues[k]) for k in ids2keep) idpolvalues2e = dict((k, previdpolvalues[k]) for k in pairslist) for newpair in newpairs: idpolvalues[newpair] = previdpolvalues[newpairs[newpair][ 0]] + previdpolvalues[newpairs[newpair][1]] if method.endswith('cnn'): npatches = ancpatches.shape[0] nsamples = round(npatches * p) idsamples = np.random.choice(npatches, nsamples, replace=False) print('| -- Fitting the model') historycnn = caret.fitcnn(ancpatches, disspatches, disspatcheso, idsamples, cnnmod=cnnmod, cnnobj=cnnobj, epochs=epochspi, batchsize=batchsize, extdataset=extdataset) loss = historycnn.history['loss'] print('| -- Predicting new values') disseverdataset = caret.predictcnn(cnnobj, cnnmod, ancpatches, disspatches, disseverdataset.shape, batchsize=batchsize) else: disssize = disseverdataset.shape[0] * disseverdataset.shape[1] nsamples = round(disssize * p) idsamples = np.random.choice(disssize, nsamples, replace=False) print('| -- Fitting the model') mod = caret.fit(ancdatasets, disseverdataset, idsamples, method) print('| -- Predicting new values') # Replace NaN's by 0 and predict ancdatasets[np.isnan(ancdatasets)] = 0 disseverdataset = caret.predict(mod, ancdatasets) disseverdataset = np.expand_dims(disseverdataset, axis=2) # Replace NaN zones by Nan disseverdataset = disseverdataset * dissmask ancdatasets = ancdatasets * ancvarsmask # Avoid negative counts disseverdataset[disseverdataset < 0] = 0 if verbose: print('| -- Computing adjustement factor') stats = npu.statsByID(disseverdataset, idsdataset, 'sum') if method.startswith('ap'): stats2e = npu.statsByID(disseverdataset, idsdataset2e, 'sum') stats2e = dict((k, stats2e[k]) for k in pairslist) # Horrible hack, avoid division by 0 for s in stats: stats[s] = stats[s] + 0.00001 polygonratios = { k: idpolvalues[k] / stats[k] for k in stats.keys() & idpolvalues } idpolvalues = previdpolvalues #print('** Mean of adj. factor', np.array(list(polygonratios.values())).mean()) for polid in polygonratios: disseverdataset[idsdataset == polid] = ( disseverdataset[idsdataset == polid] * polygonratios[polid]) osgu.writeRaster( disseverdataset[:, :, 0], rastergeo, 'tempfiledissever_livebirths' + 'pfv_' + str(k) + 'it.tif') if method.startswith('ap'): # Compute disaggregation error for 10% of the municipalities actual2e = list(idpolvalues2e.values()) predicted2e = list(stats2e.values()) range2e = max(actual2e) - min(actual2e) nrmse2e = np.sqrt(metrics.mean_squared_error( actual2e, predicted2e)) / range2e nmae2e = metrics.mean_absolute_error(actual2e, predicted2e) / range2e filenamenrmse2e = 'nrmse2epfv_livebirths' + '.txt' filenamenmae2e = 'nmae2epfv_livebirths' + '.txt' if os.path.exists(filenamenrmse2e): with open(filenamenrmse2e, 'a') as myfile: myfile.write(str(k) + ': ' + str(nrmse2e) + '\n') else: with open(filenamenrmse2e, 'w+') as myfile: myfile.write(str(k) + ': ' + str(nrmse2e) + '\n') if os.path.exists(filenamenmae2e): with open(filenamenmae2e, 'a') as myfile: myfile.write(str(k) + ': ' + str(nmae2e) + '\n') else: with open(filenamenmae2e, 'w+') as myfile: myfile.write(str(k) + ': ' + str(nmae2e) + '\n') # Check if the algorithm has converged error = np.nanmean(abs(disseverdataset - olddisseverdataset)) history.append([error, loss ]) if method.endswith('cnn') else history.append(error) errorrat = (error / lasterror) if lasterror > 0 else np.inf lasterror = error print('Error:', error) if k >= min_iter: if errorrat < converge: if error < lowesterror: lowesterror = error lowesterriter = k lowesterrdisseverdataset = np.copy(disseverdataset) else: if k == min_iter: lowesterriter = k else: disseverdataset = lowesterrdisseverdataset print('Retaining model fitted at iteration', lowesterriter) break olddisseverdataset = disseverdataset if tempfileid: tempfile = 'tempfiledissever_' + tempfileid + '.tif' osgu.writeRaster(disseverdataset, rastergeo, tempfile) return disseverdataset[:, :, 0], rastergeo, history
for indicator in indicators: print('--- Running dasymetric mapping for the indicator', indicator[0]) fshape = os.path.join('Shapefiles', (indicator[2] + '13_cont.shp')) fcsv = os.path.join('Statistics', indicator[0], (indicator[2] + '.csv')) fancdataset = 'Rasters/ghs15_200m.tif' osgu.removeAttrFromShapefile(fshape, ['ID', 'VALUE']) osgu.addAttr2Shapefile(fshape, fcsv, [indicator[2].upper()], encoding='UTF-8') tempfileid = None #None idsdataset = osgu.ogr2raster(fshape, attr='ID', template=[rastergeo, nrowsds, ncolsds])[0] polygonvaluesdataset, rastergeo = osgu.ogr2raster( fshape, attr='VALUE', template=[rastergeo, nrowsds, ncolsds]) ancdataset = osgu.readRaster(fancdataset)[0] tddataset, rastergeo = dm.rundasymmapping(idsdataset, polygonvaluesdataset, ancdataset, rastergeo, tempfileid=tempfileid) osgu.writeRaster(tddataset[:, :, 0], rastergeo, 'td_' + indicator[0] + '.tif') osgu.removeAttrFromShapefile(fshape, ['ID', 'VALUE'])
print('\n--- Running dissever leveraging a', meth, 'model') seed(42) set_random_seed(42) dissdataset, rastergeo, history = dissever.runDissever(fshape, ancdatasets, ancnames=ancnames, min_iter=3, max_iter=4, perc2evaluate=p2e, poly2agg=admboundary2.upper(), rastergeo=rastergeo, method=meth, p=psamp, yraster=yraster, verbose=True) print('- Writing raster to disk...') niter = len(history)-1 osgu.writeRaster(dissdataset, rastergeo, 'dissever_tdnl_' + str(p2e) + '_' + indicator + '_' + meth + '_psamp=' + str(psamp) + '_' + str(niter) + 'iter.tif') else: config = tf.ConfigProto() config.gpu_options.allow_growth = True session = tf.Session(config=config) for cnnm in cnnmodel: for lweights in lossweights: for filt in filters: for psize in patchsize: for epiter in epochspi: for bsize in batchsize: for lrate in learningrate:
nrowsds = ds.shape[1] ncolsds = ds.shape[0] for indicator in indicators: print('--- Running pycnophylactic interpolation for the indicator', indicator[0]) fshape = os.path.join('Shapefiles', (indicator[2] + '13_cont.shp')) fcsv = os.path.join('Statistics', indicator[0], (indicator[2] + '.csv')) osgu.removeAttrFromShapefile(fshape, ['ID', 'VALUE']) osgu.addAttr2Shapefile(fshape, fcsv, [indicator[1].upper(), indicator[2].upper()], encoding='UTF-8') tempfileid = indicator[0] idsdataset = osgu.ogr2raster(fshape, attr='ID', template=[rastergeo, nrowsds, ncolsds])[0] polygonvaluesdataset, rastergeo = osgu.ogr2raster( fshape, attr='VALUE', template=[rastergeo, nrowsds, ncolsds]) pycnodataset, rastergeo = pycno.runPycno(idsdataset, polygonvaluesdataset, rastergeo, tempfileid=tempfileid) osgu.writeRaster(pycnodataset[:, :, 0], rastergeo, 'pycnointerpolation_' + indicator[0] + '.tif') osgu.removeAttrFromShapefile(fshape, ['ID', 'VALUE'])