def get_inhabitants(polygon, name):
"""The inhabitants of 2016 are used."""
table = 'ew'
cfg_data = cfg.get_dict(table)
ew_fn = os.path.join(cfg.get('paths', 'fis_broker'), cfg_data['table'],
'shp', cfg_data['table'] + '.shp')
logging.debug("Reading {0}".format(ew_fn))
if not os.path.isfile(ew_fn):
ew_fn = download.download_maps(single=table)
ew = geometries.load(fullname=ew_fn)
ew['centroid_column'] = ew.representative_point()
ew = ew.set_geometry('centroid_column')
neu = geometries.spatial_join_with_buffer(
ew,
polygon,
name=name,
limit=0,
)
grp = neu.groupby(name).sum()
grp['frac'] = grp['EW'].div(grp.sum()['EW']).multiply(100).round(1)
return grp
def process_alkis_buildings(shapefile_out, table, remove_non_heated=True):
"""
Parameters
----------
shapefile_out
table
remove_non_heated
Returns
-------
"""
path = os.path.join(cfg.get("paths", "fis_broker"), table, "shp")
shapefile_in = os.path.join(path, table + ".shp")
# Download shp_file if it does not exist
if not os.path.isfile(shapefile_in):
shapefile_in = download.download_maps(single="alkis")
geo_table = gpd.read_file(shapefile_in)
# Removing parts of the Alkis table:
# Bauart_sch == 0 : Data sets with Bauart_sch > 0 are building parts
# LageZurErd != 1200 : Remove underground buildings
logging.info(
"Length of data set before removing parts: {0}".format(len(geo_table))
)
geo_table = geo_table[geo_table["BAT"].isnull()]
geo_table = geo_table[geo_table["OFL"] != 1200]
# Remove all data sets that are marked es non-heated in the alkis heat
# factor table if remove_non_heated is set to True.
if remove_non_heated is True:
filename_heat_factor = os.path.join(
cfg.get("paths", "data_berlin"),
cfg.get("oeq", "alkis_heat_factor_table"),
)
heat_factor = pd.read_csv(filename_heat_factor, index_col=[0])
non_heated = list(heat_factor.loc[heat_factor.heat_factor == 0].index)
geo_table = geo_table[~geo_table["BEZGFK"].isin(non_heated)]
logging.info(
"Length of data set after removing parts: {0}".format(len(geo_table))
)
# Calculate the perimeter and area of the polygons and add it as columns
logging.info("Calculate perimeter and area of each polygon...")
geo_table = geo_table.to_crs({"init": "epsg:3035"})
geo_table["area"] = geo_table["geometry"].area
geo_table["perimeter"] = geo_table["geometry"].length
geo_table = geo_table.to_crs({"init": "epsg:4326"})
# Dump table as new shape_file
logging.info("Dump new table to shp-file.")
geo_table.to_file(shapefile_out)
return shapefile_out
def process_alkis_buildings(shapefile_out, table, remove_non_heated=True):
"""
Parameters
----------
shapefile_out
table
remove_non_heated
Returns
-------
"""
path = os.path.join(cfg.get('paths', 'fis_broker'), table, 'shp')
shapefile_in = os.path.join(path, table + '.shp')
# Download shp_file if it does not exist
if not os.path.isfile(shapefile_in):
shapefile_in = download.download_maps(single='alkis')
geo_table = gpd.read_file(shapefile_in)
# Removing parts of the Alkis table:
# Bauart_sch == 0 : Data sets with Bauart_sch > 0 are building parts
# LageZurErd != 1200 : Remove underground buildings
logging.info("Length of data set before removing parts: {0}".format(
len(geo_table)))
geo_table = geo_table[geo_table['Bauart_sch'] == 0]
geo_table = geo_table[geo_table['LageZurErd'] != 1200]
# Remove all data sets that are marked es non-heated in the alkis heat
# factor table if remove_non_heated is set to True.
if remove_non_heated is True:
filename_heat_factor = os.path.join(
cfg.get('paths', 'data_berlin'),
cfg.get('oeq', 'alkis_heat_factor_table'))
heat_factor = pd.read_csv(filename_heat_factor, index_col=[0])
non_heated = list(heat_factor.loc[heat_factor.heat_factor == 0].index)
geo_table = geo_table[~geo_table['Gebaeudefu'].isin(non_heated)]
logging.info("Length of data set after removing parts: {0}".format(
len(geo_table)))
# Calculate the perimeter and area of the polygons and add it as columns
logging.info("Calculate perimeter and area of each polygon...")
geo_table = geo_table.to_crs({'init': 'epsg:3035'})
geo_table['area'] = geo_table['geometry'].area
geo_table['perimeter'] = geo_table['geometry'].length
geo_table = geo_table.to_crs({'init': 'epsg:4326'})
# Dump table as new shape_file
logging.info("Dump new table to shp-file.")
geo_table.to_file(shapefile_out)
return shapefile_out
def installed_pv_capacity():
table = cfg.get('pv_map', 'table')
path = os.path.join(cfg.get('paths', 'fis_broker'), table, 'shp')
shapefile = os.path.join(path, table + '.shp')
if not os.path.isfile(shapefile):
new_shapefile = download.download_maps(single='pv_map')
if new_shapefile != shapefile:
msg = "Wrong path will download this file every time {0} : {1}"
logging.error(msg.format(shapefile, new_shapefile))
shapefile = new_shapefile
pv_cap = gpd.read_file(shapefile)
pv_cap['spatial_na'] = pv_cap['gml_id'].str.split('.', expand=True)[1]
pv_cap.set_index('spatial_na', inplace=True)
return round(pv_cap.loc['090517'].BZR_GLEIST / 1000, 3)
def merge_maps():
gdf = {}
table = 's_wfs_alkis_gebaeudeflaechen'
path = os.path.join(cfg.get('paths', 'fis_broker'), table, 'shp')
shapefile_alkis = os.path.join(path, table + '_prepared' + '.shp')
if not os.path.isfile(shapefile_alkis):
shapefile_alkis = process_alkis_buildings(shapefile_alkis, table)
tables = download.get_map_config()
# Filename and path for output files
filename_poly_layer = os.path.join(
cfg.get('paths', 'fis_broker'),
cfg.get('fis_broker', 'merged_blocks_polygon'))
# Columns to use
cols = {
'block': ['gml_id', 'PLR', 'STAT', 'STR_FLGES'],
'nutz': ['STSTRNAME', 'TYPKLAR', 'WOZ_NAME'],
'ew': ['EW_HA']
}
logging.info("Read tables to be joined: {0}.".format(tuple(cols.keys())))
for t in ['block', 'nutz', 'ew']:
tables[t]['path'] = os.path.join(cfg.get('paths', 'fis_broker'),
tables[t]['table'], 'shp',
tables[t]['table'] + '.shp')
logging.debug("Reading {0}".format(tables[t]['path']))
if not os.path.isfile(tables[t]['path']):
tables[t]['path'] = download.download_maps(single=t)
gdf[t] = gpd.read_file(tables[t]['path'])[cols[t] + ['geometry']]
logging.info("Spatial join of all tables...")
gdf['block'].rename(columns={'gml_id': 'SCHL5'}, inplace=True)
# Convert geometry to representative points to simplify the join
gdf['block']['geometry'] = gdf['block'].representative_point()
gdf['block'] = gpd.sjoin(gdf['block'],
gdf['nutz'],
how='inner',
op='within')
del gdf['block']['index_right']
gdf['block'] = gpd.sjoin(gdf['block'], gdf['ew'], how='left', op='within')
del gdf['block']['index_right']
del gdf['block']['geometry']
# Merge with polygon layer to dump polygons instead of points.
gdf['block'] = pd.DataFrame(gdf['block'])
polygons = gpd.read_file(tables['block']['path'])[['gml_id', 'geometry']]
polygons.rename(columns={'gml_id': 'SCHL5'}, inplace=True)
polygons = polygons.merge(gdf['block'], on='SCHL5')
polygons = polygons.set_geometry('geometry')
logging.info("Dump polygon layer to {0}...".format(filename_poly_layer))
polygons.to_file(filename_poly_layer)
logging.info("Read alkis table...")
alkis = gpd.read_file(shapefile_alkis)
logging.info("Join alkis buildings with block data...")
alkis = alkis[[
'AnzahlDerO', 'area', 'perimeter', 'Gebaeudefu', 'gml_id', 'geometry'
]]
block_j = polygons[[
'SCHL5', 'PLR', 'STAT', 'TYPKLAR', 'EW_HA', 'geometry'
]]
alkis['geometry'] = alkis.representative_point()
alkis = gpd.sjoin(alkis, block_j, how='left', op='within')
del alkis['index_right']
# Join the alkis data with the map of the heating system fraction
logging.info("Join alkis buildings with heiz data...")
geoheiz_obj = reegis_tools.geometries.Geometry()
geoheiz_obj.load_csv(cfg.get('paths', 'data_berlin'),
cfg.get('fis_broker', 'heating_systems_csv'))
geoheiz_obj.df = geoheiz_obj.df.loc[geoheiz_obj.df['geometry'].notnull()]
geoheiz_obj.df = geoheiz_obj.df.rename(columns={'block': 'heiz_block'})
geoheiz_obj.create_geo_df()
geoheiz = geoheiz_obj.gdf
geoheiz = geoheiz[geoheiz.geometry.is_valid]
alkis = gpd.sjoin(alkis, geoheiz, how='left', op='within')
del alkis['index_right']
logging.info("Add block data for non-matching points using buffers.")
remain = len(alkis.loc[alkis['PLR'].isnull()])
logging.info(
"This will take some time. Number of points: {0}".format(remain))
# I think it is possible to make this faster and more elegant but I do not
# not have the time to think about it. As it has to be done only once it
# is not really time-sensitive.
for row in alkis.loc[alkis['PLR'].isnull()].iterrows():
idx = int(row[0])
point = row[1].geometry
intersec = False
n = 0
block_id = 0
while not intersec and n < 500:
bi = block_j.loc[block_j.intersects(point.buffer(n / 100000))]
if len(bi) > 0:
intersec = True
bi = bi.iloc[0]
block_id = bi['SCHL5']
del bi['geometry']
alkis.loc[idx, bi.index] = bi
n += 1
remain -= 1
if intersec:
logging.info(
"Block found for {0}: {1}, Buffer: {2}. Remains: {3}".format(
alkis.loc[idx, 'gml_id'][-12:], block_id[-16:], n, remain))
else:
warnings.warn(
"{0} does not intersect with any region. Please check".format(
row[1]))
logging.info(
"Check: Number of buildings without PLR attribute: {0}".format(
len(alkis.loc[alkis['PLR'].isnull()])))
# Merge with polygon layer to dump polygons instead of points.
logging.info("Merge new alkis layer with alkis polygon layer.")
alkis = pd.DataFrame(alkis)
del alkis['geometry']
alkis_poly = gpd.read_file(shapefile_alkis)[['gml_id', 'geometry']]
alkis_poly = alkis_poly.merge(alkis, on='gml_id')
alkis_poly = alkis_poly.set_geometry('geometry')
logging.info("Dump new alkis layer with additional block data.")
filename_shp = os.path.join(cfg.get('paths', 'fis_broker'),
cfg.get('fis_broker', 'alkis_joined_shp'))
alkis_poly.to_file(filename_shp)
return filename_shp
def merge_maps():
gdf = {}
table = "s_wfs_alkis_gebaeudeflaechen"
path = os.path.join(cfg.get("paths", "fis_broker"), table, "shp")
shapefile_alkis = os.path.join(path, table + "_prepared" + ".shp")
if not os.path.isfile(shapefile_alkis):
shapefile_alkis = process_alkis_buildings(shapefile_alkis, table)
tables = download.get_map_config()
# Filename and path for output files
filename_poly_layer = os.path.join(
cfg.get("paths", "fis_broker"),
cfg.get("fis_broker", "merged_blocks_polygon"),
)
# Columns to use
cols = {
"block": ["gml_id", "PLR", "STAT", "STR_FLGES"],
"nutz": ["STSTRNAME", "TYPKLAR", "WOZ_NAME"],
"ew": ["EW_HA"],
}
logging.info("Read tables to be joined: {0}.".format(tuple(cols.keys())))
for t in ["block", "nutz", "ew"]:
tables[t]["path"] = os.path.join(
cfg.get("paths", "fis_broker"),
tables[t]["table"],
"shp",
tables[t]["table"] + ".shp",
)
logging.debug("Reading {0}".format(tables[t]["path"]))
if not os.path.isfile(tables[t]["path"]):
tables[t]["path"] = download.download_maps(single=t)
gdf[t] = gpd.read_file(tables[t]["path"])[cols[t] + ["geometry"]]
logging.info("Spatial join of all tables...")
gdf["block"].rename(columns={"gml_id": "SCHL5"}, inplace=True)
# Convert geometry to representative points to simplify the join
gdf["block"]["geometry"] = gdf["block"].representative_point()
gdf["block"] = gpd.sjoin(
gdf["block"], gdf["nutz"], how="inner", op="within"
)
del gdf["block"]["index_right"]
gdf["block"] = gpd.sjoin(gdf["block"], gdf["ew"], how="left", op="within")
del gdf["block"]["index_right"]
del gdf["block"]["geometry"]
# Merge with polygon layer to dump polygons instead of points.
gdf["block"] = pd.DataFrame(gdf["block"])
polygons = gpd.read_file(tables["block"]["path"])[["gml_id", "geometry"]]
polygons.rename(columns={"gml_id": "SCHL5"}, inplace=True)
polygons = polygons.merge(gdf["block"], on="SCHL5")
polygons = polygons.set_geometry("geometry")
logging.info("Dump polygon layer to {0}...".format(filename_poly_layer))
polygons.to_file(filename_poly_layer)
logging.info("Read alkis table...")
alkis = gpd.read_file(shapefile_alkis)
logging.info("Join alkis buildings with block data...")
alkis = alkis[
["AOG", "area", "perimeter", "BEZGFK", "GFK", "gml_id", "geometry"]
]
block_j = polygons[
["SCHL5", "PLR", "STAT", "TYPKLAR", "EW_HA", "geometry"]
]
alkis["geometry"] = alkis.representative_point()
alkis = gpd.sjoin(alkis, block_j, how="left", op="within")
del alkis["index_right"]
# Join the alkis data with the map of the heating system fraction
logging.info("Join alkis buildings with heiz data...")
geoheiz = geometries.load_csv(
cfg.get("paths", "data_berlin"),
cfg.get("fis_broker", "heating_systems_csv"),
)
geoheiz = geoheiz.loc[geoheiz["geometry"].notnull()]
geoheiz = geoheiz.rename(columns={"block": "heiz_block"})
geoheiz = geometries.create_geo_df(geoheiz)
geoheiz = geoheiz[geoheiz.geometry.is_valid]
alkis = gpd.sjoin(alkis, geoheiz, how="left", op="within")
del alkis["index_right"]
logging.info("Add block data for non-matching points using buffers.")
remain = len(alkis.loc[alkis["PLR"].isnull()])
logging.info(
"This will take some time. Number of points: {0}".format(remain)
)
# I think it is possible to make this faster and more elegant but I do not
# not have the time to think about it. As it has to be done only once it
# is not really time-sensitive.
for row in alkis.loc[alkis["PLR"].isnull()].iterrows():
idx = int(row[0])
point = row[1].geometry
intersec = False
n = 0
block_id = 0
while not intersec and n < 500:
bi = block_j.loc[block_j.intersects(point.buffer(n / 100000))]
if len(bi) > 0:
intersec = True
bi = bi.iloc[0]
block_id = bi["SCHL5"]
del bi["geometry"]
alkis.loc[idx, bi.index] = bi
n += 1
remain -= 1
if intersec:
logging.info(
"Block found for {0}: {1}, Buffer: {2}. Remains: {3}".format(
alkis.loc[idx, "gml_id"][-12:], block_id[-16:], n, remain
)
)
else:
warnings.warn(
"{0} does not intersect with any region. Please check".format(
row[1]
)
)
logging.info(
"Check: Number of buildings without PLR attribute: {0}".format(
len(alkis.loc[alkis["PLR"].isnull()])
)
)
# Merge with polygon layer to dump polygons instead of points.
logging.info("Merge new alkis layer with alkis polygon layer.")
alkis = pd.DataFrame(alkis)
del alkis["geometry"]
alkis_poly = gpd.read_file(shapefile_alkis)[["gml_id", "geometry"]]
alkis_poly = alkis_poly.merge(alkis, on="gml_id")
alkis_poly = alkis_poly.set_geometry("geometry")
logging.info("Dump new alkis layer with additional block data.")
filename_shp = os.path.join(
cfg.get("paths", "fis_broker"),
cfg.get("fis_broker", "alkis_joined_shp"),
)
alkis_poly.to_file(filename_shp)
return filename_shp