def test_noise_graph_update(self):
in_graph_file = 'data/test_graph.graphml'
out_graph_file = 'temp/test_graph_noises.graphml'
data_extent_file = 'data/HMA.geojson'
noise_csv_dir = 'noise_csv/'
data_extent: Polygon = geom_utils.project_geom(
gpd.read_file(data_extent_file)['geometry'][0])
graph = ig_utils.read_graphml(in_graph_file, log)
noise_graph_update.set_default_and_na_edge_noises(
graph, data_extent, log)
noise_graph_update.noise_graph_update(graph, noise_csv_dir, log)
ig_utils.export_to_graphml(graph, out_graph_file)
graph = ig_utils.read_graphml(out_graph_file)
self.assertEqual(graph.ecount(), 3702)
for edge in graph.es:
attrs = edge.attributes()
# check that edge IDs are correct
self.assertEqual(edge.index, attrs[E.id_ig.value])
if isinstance(attrs[E.geometry.value], LineString):
# note: this will fail if some of the edges are outside the noise data extent
self.assertNotEqual(edge[E.noises.value], None)
self.assertIsInstance(edge[E.noises.value], dict)
self.assertNotEqual(edge[E.noise_source.value], None)
self.assertIsInstance(edge[E.noise_source.value], str)
else:
# for edges without geometry the noise attributes should be nodata
self.assertEqual(edge[E.noises.value], None)
self.assertEqual(edge[E.noise_source.value], None)
# if edge noises are nodata then also noise source must be nodata
if edge[E.noises.value] == None:
self.assertEqual(edge[E.noise_source.value], None)
# if edge noises are not nodata but {} then noise source must also be just '' (not nodata)
if edge[E.noises.value] == {}:
self.assertEqual(edge[E.noise_source.value], '')
# if edge has noises it must also have noise source
if edge[E.noises.value]:
self.assertNotEqual(edge[E.noise_source.value], '')
self.assertNotEqual(edge[E.noise_source.value], None)
# if edge has noise source it must have also noises
if edge[E.noise_source.value]:
self.assertNotEqual(edge[E.noises.value], '')
self.assertNotEqual(edge[E.noises.value], None)
def test_updates_noises_from_csv_to_graph():
in_graph_file = f'{base_dir}/data/test_graph.graphml'
out_graph_file = f'{base_dir}/temp/test_graph_noises.graphml'
data_extent_file = f'{base_dir}/data/HMA.geojson'
noise_csv_dir = f'{base_dir}/noise_csv/'
data_extent: Polygon = geom_utils.project_geom(gpd.read_file(data_extent_file)['geometry'][0])
graph = ig_utils.read_graphml(in_graph_file)
noise_graph_update.set_default_and_na_edge_noises(graph, data_extent)
noise_graph_update.noise_graph_update(graph, noise_csv_dir)
ig_utils.export_to_graphml(graph, out_graph_file)
graph = ig_utils.read_graphml(out_graph_file)
assert graph.ecount() == 3702
for edge in graph.es:
attrs = edge.attributes()
# check that edge IDs are correct
assert edge.index == attrs[E.id_ig.value]
if isinstance(attrs[E.geometry.value], LineString):
# note: this will fail if some of the edges are outside the noise data extent
assert edge[E.noises.value] is not None
assert isinstance(edge[E.noises.value], dict)
assert edge[E.noise_source.value] is not None
assert isinstance(edge[E.noise_source.value], str)
else:
# for edges without geometry the noise attributes should be nodata
assert edge[E.noises.value] is None
assert edge[E.noise_source.value] is None
# if edge noises are nodata then also noise source must be nodata
if edge[E.noises.value] is None:
assert edge[E.noise_source.value] is None
# if edge noises are not nodata but {} then noise source must also be just '' (not nodata)
if edge[E.noises.value] == {}:
assert edge[E.noise_source.value] == ''
# if edge has noises it must also have noise source
if edge[E.noises.value]:
assert edge[E.noise_source.value] != ''
assert edge[E.noise_source.value] is not None
# if edge has noise source it must have also noises
if edge[E.noise_source.value]:
assert edge[E.noises.value] != ''
assert edge[E.noises.value] is not None
def __init__(self, logger: Logger, graph_file: str, routing_conf: RoutingConf):
"""Initializes a graph (and related features) used by green_paths_app and aqi_processor_app.
Args:
subset: A boolean variable indicating whether a subset of the graph should be loaded
(subset is for testing / developing).
"""
self.log = logger
self.log.info(f'Loading graph from file: {graph_file}')
start_time = time.time()
self.graph = ig_utils.read_graphml(graph_file)
self.routing_conf = routing_conf
self.ecount = self.graph.ecount()
self.vcount = self.graph.vcount()
self.log.info(f'Graph of {self.graph.ecount()} edges read')
self.__edge_gdf = self.__get_edge_gdf()
self.__edge_sindex = self.__edge_gdf.sindex
self.__node_gdf = ig_utils.get_node_gdf(self.graph, drop_na_geoms=True)
self.__nodes_sind = self.__node_gdf.sindex
if conf.cycling_enabled:
edge_cost_factory.set_biking_costs(self.graph, self.log)
if conf.quiet_paths_enabled:
edge_cost_factory.set_noise_costs_to_edges(self.graph, routing_conf)
self.log.info('Noise costs set')
if conf.gvi_paths_enabled:
edge_cost_factory.set_gvi_costs_to_graph(self.graph, routing_conf)
self.log.info('GVI costs set')
self.graph.es[E.aqi.value] = None # set default AQI value to None
self.log.duration(start_time, 'Graph initialized', log_level='info')
self.__path_edge_cache: Dict[int, PathEdge] = {}
def test_add_sampling_points(self):
graph = ig_utils.read_graphml('data/test_graph.graphml')
gdf = ig_utils.get_edge_gdf(graph)
# start_time = time.time()
gdf = utils.add_sampling_points_to_gdf(gdf, 2)
# log.duration(start_time, 'added sampling points')
sampling_points_list = list(gdf['sampling_points'])
self.assertEqual(
len([sps for sps in sampling_points_list if sps != None]), 3522)
self.assertEqual(
len([sps for sps in sampling_points_list if sps == None]), 180)
# test that all sample points are on the line geometries
for edge in gdf.itertuples():
sampling_points = getattr(edge, 'sampling_points')
if (sampling_points == None): continue
line_geom = getattr(edge, 'geometry')
for sp in sampling_points:
self.assertAlmostEqual(sp.distance(line_geom), 0, 5)
# validate sampling point gdf (exploaded from edge gdf with sampling points)
sampling_gdf = utils.explode_sampling_point_gdf(gdf, 'sampling_points')
self.assertGreater(len(sampling_gdf), len(gdf))
self.assertEqual(len(sampling_gdf), 58554)
# check that the total representative length of each set of sampling points equals the length of the respective edge
sps_by_edge = sampling_gdf.groupby('edge_id')
for edge in gdf.itertuples():
if (edge.sampling_points != None):
edge_sps = sps_by_edge.get_group(edge.Index)
sampling_length_sum = edge_sps['sample_len'].sum()
self.assertAlmostEqual(sampling_length_sum,
edge.geometry.length, 5)
def test_adds_sampling_points_to_edge_gdf():
graph = ig_utils.read_graphml(f'{base_dir}/data/test_graph.graphml')
gdf = ig_utils.get_edge_gdf(graph)
# start_time = time.time()
gdf = noise_join_utils.add_sampling_points_to_gdf(gdf, 2)
# log.duration(start_time, 'added sampling points')
sampling_points_list = list(gdf['sampling_points'])
assert len([sps for sps in sampling_points_list if sps != None]) == 3522
assert len([sps for sps in sampling_points_list if sps == None]) == 180
# test that all sample points are on the line geometries
for edge in gdf.itertuples():
sampling_points = getattr(edge, 'sampling_points')
if not sampling_points:
continue
line_geom = getattr(edge, 'geometry')
for sp in sampling_points:
assert round(sp.distance(line_geom), 1) == 0, 5
# validate sampling point gdf (exploaded from edge gdf with sampling points)
sampling_gdf = noise_join_utils.explode_sampling_point_gdf(gdf, 'sampling_points')
assert len(sampling_gdf) > len(gdf)
assert len(sampling_gdf) == 58554
# check that the total representative length of each set of sampling points equals the length of the respective edge
sps_by_edge = sampling_gdf.groupby('edge_id')
for edge in gdf.itertuples():
if (edge.sampling_points != None):
edge_sps = sps_by_edge.get_group(edge.Index)
sampling_length_sum = edge_sps['sample_len'].sum()
assert round(sampling_length_sum, 2) == round(edge.geometry.length, 2)
def test_joins_noises_to_graph_edges():
graph = ig_utils.read_graphml(f'{base_dir}/data/test_graph.graphml')
edge_gdf = ig_utils.get_edge_gdf(graph, attrs=[E.id_ig, E.length])
edge_gdf[E.id_ig.name] = edge_gdf.index
# read noise data
noise_layer_names = [layer for layer in fiona.listlayers(f'{base_dir}/data/noise_data_processed.gpkg')]
noise_layers = {name: gpd.read_file(f'{base_dir}/data/noise_data_processed.gpkg', layer=name) for name in noise_layer_names}
noise_layers = {name: gdf.rename(columns={'db_low': name}) for name, gdf in noise_layers.items()}
# read nodata zone: narrow area between noise surfaces of different municipalities
nodata_layer = gpd.read_file(f'{base_dir}/data/extents.gpkg', layer='municipal_boundaries')
edge_noises = noise_graph_join.noise_graph_join(
edge_gdf=edge_gdf,
sampling_interval=3,
noise_layers=noise_layers,
nodata_layer=nodata_layer
)
assert edge_noises[E.id_ig.name].nunique() == 3522
edge_noises_df = pd.merge(edge_gdf, edge_noises, how='inner', on=E.id_ig.name)
edge_noises_df['total_noise_len'] = [round(sum(noises.values()), 4) for noises in edge_noises_df['noises']]
def validate_edge_noises(row):
assert round(row['total_noise_len'], 1) <= round(row['length'], 1)
edge_noises_df.apply(lambda row: validate_edge_noises(row), axis=1)
assert round(edge_noises_df['total_noise_len'].mean(), 2) == 33.20
# test frequency of different main noise sources
noise_sources = dict(Counter(list(edge_noises_df[E.noise_source.name])))
assert noise_sources == {'road': 2322, 'train': 1198, '': 2}
def main(conf: GraphNoiseJoinConf):
data_extent: Polygon = geom_utils.project_geom(
gpd.read_file(conf.noise_data_extent_fp)['geometry'][0])
graph = ig_utils.read_graphml(conf.graph_in_fp, log)
set_default_and_na_edge_noises(graph, data_extent)
noise_graph_update(graph, conf.noise_data_csv_dir)
ig_utils.export_to_graphml(graph, conf.graph_out_fp)
log.info(f'exported graph of {graph.ecount()} edges')
log.info('all done')
def main(conf: GraphNoiseJoinConf):
graph = ig_utils.read_graphml(conf.graph_in_fp)
log.info(f'read graph of {graph.ecount()} edges')
edge_gdf = ig_utils.get_edge_gdf(graph, attrs=[E.id_ig])
edge_gdf = edge_gdf.sort_values(E.id_ig.name)
# read noise data
noise_layer_names = [
layer for layer in fiona.listlayers(conf.noise_data_fp)
]
noise_layers = {
name: gpd.read_file(conf.noise_data_fp, layer=name)
for name in noise_layer_names
}
noise_layers = {
name: gdf.rename(columns={'db_low': name})
for name, gdf in noise_layers.items()
}
log.info(f'read {len(noise_layers)} noise layers')
# read nodata zone: narrow area between noise surfaces of different municipalities
nodata_layer = gpd.read_file(conf.nodata_fp, layer=conf.nodata_layer_name)
# process chunks of edges together by dividing gdf to parts
processing_size = 50000
split_gdf_count = math.ceil(len(edge_gdf) / processing_size)
gdfs = np.array_split(edge_gdf, split_gdf_count)
# get max id of previously processed edges
max_processed_id = get_previously_processed_max_id(conf.noise_data_csv_dir)
if max_processed_id > 0:
log.info(
f'found previously processed edges up to edge id {max_processed_id}'
)
for idx, gdf in enumerate(gdfs):
if gdf[E.id_ig.name].max() <= max_processed_id:
log.info(
f'skipping {idx+1} of {len(gdfs)} edge gdfs (processed before)'
)
continue
else:
log.info(f'processing {idx+1} of {len(gdfs)} edge gdfs')
edge_noises = noise_graph_join(
edge_gdf=gdf,
sampling_interval=3,
noise_layers=noise_layers,
nodata_layer=nodata_layer,
b_debug=False,
debug_gpkg='debug/noise_join_debug.gpkg')
export_edge_noise_csv(edge_noises, conf.noise_data_csv_dir)
def test_graph_to_gdf(self):
graph = ig_utils.read_graphml('data/test_graph.graphml',
log=Logger(printing=True))
# test read graph to wgs gdf
gdf = ig_utils.get_edge_gdf(graph,
id_attr=Edge.id_ig,
attrs=[Edge.length],
geom_attr=Edge.geom_wgs)
gdf['geom_length'] = [geom.length for geom in gdf[Edge.geom_wgs.name]]
self.assertAlmostEqual(gdf['geom_length'].mean(), 0.000429, 6)
# test read to projected gdf
gdf = ig_utils.get_edge_gdf(graph,
id_attr=Edge.id_ig,
attrs=[Edge.length],
geom_attr=Edge.geometry)
gdf['geom_length'] = [geom.length for geom in gdf[Edge.geometry.name]]
self.assertAlmostEqual(gdf['geom_length'].mean(), 31.65, 2)
def test_gets_graph_data_as_gdf():
graph = ig_utils.read_graphml(conf.igraph_out_file)
# test read graph to wgs gdf
gdf = ig_utils.get_edge_gdf(graph,
id_attr=Edge.id_ig,
attrs=[Edge.length],
geom_attr=Edge.geom_wgs,
drop_na_geoms=True)
gdf['geom_length'] = [geom.length for geom in gdf[Edge.geom_wgs.name]]
assert round(gdf['geom_length'].mean(), 6) == 0.000451
# test read to projected gdf
gdf = ig_utils.get_edge_gdf(graph,
id_attr=Edge.id_ig,
attrs=[Edge.length],
geom_attr=Edge.geometry,
drop_na_geoms=True)
gdf['geom_length'] = [geom.length for geom in gdf[Edge.geometry.name]]
assert round(gdf['geom_length'].mean(), 2) == 33.27
def test_read_igraph(self):
graph = ig_utils.read_graphml('temp/test_graph.graphml',
log=Logger(printing=True))
self.assertEqual(graph.ecount(), 3702)
self.assertEqual(graph.vcount(), 1328)
attr_names = list(graph.vs[0].attributes().keys())
for attr in attr_names:
self.assertIn(attr, [e.value for e in Node],
'no unknown node attributes allowed')
attr_names = list(graph.es[0].attributes().keys())
for attr in attr_names:
self.assertIn(attr, [e.value for e in Edge],
'no unknown edge attributes allowed')
for n in graph.vs:
attrs = n.attributes()
self.assertEqual(attrs[Node.id_ig.value], n.index)
self.assertIsInstance(attrs[Node.id_ig.value], int)
self.assertIsInstance(attrs[Node.id_otp.value], str)
self.assertIsInstance(attrs[Node.name_otp.value], str)
self.assertIsInstance(attrs[Node.geometry.value], Point)
self.assertIsInstance(attrs[Node.geom_wgs.value], Point)
self.assertIsInstance(attrs[Node.traversable_walking.value], bool)
self.assertIsInstance(attrs[Node.traversable_biking.value], bool)
self.assertIsInstance(attrs[Node.traffic_light.value], bool)
for e in graph.es:
attrs = e.attributes()
self.assertEqual(attrs[Edge.id_ig.value], e.index)
self.assertIsInstance(attrs[Edge.id_ig.value], int)
self.assertIsInstance(attrs[Edge.id_otp.value], str)
self.assertIsInstance(attrs[Edge.name_otp.value], str)
self.assertIsInstance(attrs[Edge.geometry.value],
(LineString, GeometryCollection))
self.assertIsInstance(attrs[Edge.geom_wgs.value],
(LineString, GeometryCollection))
self.assertIsInstance(attrs[Edge.length.value], float)
self.assertIsInstance(attrs[Edge.edge_class.value], str)
self.assertIsInstance(attrs[Edge.street_class.value], str)
self.assertIsInstance(attrs[Edge.is_stairs.value], bool)
self.assertIsInstance(attrs[Edge.is_no_thru_traffic.value], bool)
self.assertIsInstance(attrs[Edge.allows_walking.value], bool)
self.assertIsInstance(attrs[Edge.allows_biking.value], bool)
self.assertIsInstance(attrs[Edge.traversable_walking.value], bool)
self.assertIsInstance(attrs[Edge.traversable_biking.value], bool)
self.assertIsInstance(attrs[Edge.bike_safety_factor.value], float)
def test_reads_the_created_igraph():
graph = ig_utils.read_graphml(conf.igraph_out_file)
assert graph.ecount() == 3702
assert graph.vcount() == 1328
attr_names = list(graph.vs[0].attributes().keys())
for attr in attr_names:
assert attr in [e.value
for e in Node] # no unknown node attributes allowed
attr_names = list(graph.es[0].attributes().keys())
for attr in attr_names:
assert attr in [e.value
for e in Edge] # no unknown edge attributes allowed
for n in graph.vs:
attrs = n.attributes()
assert attrs[Node.id_ig.value] == n.index
assert isinstance(attrs[Node.id_ig.value], int)
assert isinstance(attrs[Node.id_otp.value], str)
assert isinstance(attrs[Node.name_otp.value], str)
assert isinstance(attrs[Node.geometry.value], Point)
assert isinstance(attrs[Node.geom_wgs.value], Point)
assert isinstance(attrs[Node.traversable_walking.value], bool)
assert isinstance(attrs[Node.traversable_biking.value], bool)
assert isinstance(attrs[Node.traffic_light.value], bool)
for e in graph.es:
attrs = e.attributes()
assert attrs[Edge.id_ig.value] == e.index
assert isinstance(attrs[Edge.id_ig.value], int)
assert isinstance(attrs[Edge.id_otp.value], str)
assert isinstance(attrs[Edge.name_otp.value], str)
assert isinstance(attrs[Edge.geometry.value],
(LineString, GeometryCollection))
assert isinstance(attrs[Edge.geom_wgs.value],
(LineString, GeometryCollection))
assert isinstance(attrs[Edge.length.value], float)
assert isinstance(attrs[Edge.edge_class.value], str)
assert isinstance(attrs[Edge.street_class.value], str)
assert isinstance(attrs[Edge.is_stairs.value], bool)
assert isinstance(attrs[Edge.is_no_thru_traffic.value], bool)
assert isinstance(attrs[Edge.allows_walking.value], bool)
assert isinstance(attrs[Edge.allows_biking.value], bool)
assert isinstance(attrs[Edge.traversable_walking.value], bool)
assert isinstance(attrs[Edge.traversable_biking.value], bool)
assert isinstance(attrs[Edge.bike_safety_factor.value], float)
def graph_export(conf: GraphExportConf, ):
in_graph = fr'{conf.base_dir}/graph_in/{conf.graph_id}.graphml'
out_graph = fr'{conf.base_dir}/graph_out/{conf.graph_id}.graphml'
out_graph_research = fr'{conf.base_dir}/graph_out/{conf.graph_id}_r.graphml'
out_graph_research_hel = fr'{conf.base_dir}/graph_out/{conf.graph_id}_r_hel-clip.graphml'
out_geojson_noise_gvi = fr'{conf.base_dir}/graph_out/{conf.graph_id}_noise_gvi.geojson'
out_geojson = fr'{conf.base_dir}/graph_out/{conf.graph_id}.geojson'
hel_extent = gpd.read_file(conf.hel_extent_fp)
out_node_attrs = [N.geometry]
out_edge_attrs = [
E.id_ig, E.uv, E.id_way, E.geometry, E.geom_wgs, E.length,
E.allows_biking, E.is_stairs, E.bike_safety_factor, E.noises, E.gvi
]
if not conf.with_noise_data:
out_edge_attrs.remove(E.noises)
if not conf.with_greenery_data:
out_edge_attrs.remove(E.gvi)
log.info(f'Reading graph file: {in_graph}')
graph = ig_utils.read_graphml(in_graph)
edge_gdf = ig_utils.get_edge_gdf(graph,
attrs=[E.id_ig, E.length],
ig_attrs=['source', 'target'])
set_uv(graph, edge_gdf)
set_way_ids(graph, edge_gdf)
graph.es[E.bike_safety_factor.value] = [
round(v, 2) if (v and np.isfinite(v)) else 1
for v in graph.es[E.bike_safety_factor.value]
]
# set combined GVI to GVI attribute & export graph
graph.es[E.gvi.value] = list(graph.es[E.gvi_comb_gsv_veg.value])
ig_utils.export_to_graphml(graph,
out_graph,
n_attrs=out_node_attrs,
e_attrs=out_edge_attrs)
# create GeoJSON files for vector tiles
geojson = utils.create_geojson(graph)
utils.write_geojson(geojson, out_geojson, overwrite=True, id_attr=True)
utils.write_geojson(geojson,
out_geojson_noise_gvi,
overwrite=True,
db_prop=True,
gvi_prop=True)
# for research use, set combined GVI that omits low vegetation to GVI attribute and export graph
graph.es[E.gvi.value] = list(graph.es[E.gvi_comb_gsv_high_veg.value])
ig_utils.export_to_graphml(graph,
out_graph_research,
n_attrs=out_node_attrs,
e_attrs=out_edge_attrs)
# export clip of the graph by the extent of Helsinki
node_gdf = ig_utils.get_node_gdf(graph, attrs=[N.id_ig])
# replace geometry with buffered one (500 m)
hel_extent['geometry'] = [
geom.buffer(500) for geom in hel_extent['geometry']
]
inside_hel = gpd.sjoin(node_gdf, hel_extent)
inside_hel_ids = list(inside_hel[N.id_ig.name])
outside_hel_ids = [
id_ig for id_ig in list(node_gdf[N.id_ig.name])
if id_ig not in inside_hel_ids
]
graph.delete_vertices(outside_hel_ids)
# delete isolated nodes
del_node_ids = [v.index for v in graph.vs.select(_degree_eq=0)]
graph.delete_vertices(del_node_ids)
# reassign igraph indexes to edge and node attributes
graph.es[E.id_ig.value] = [e.index for e in graph.es]
graph.vs[N.id_ig.value] = [v.index for v in graph.vs]
# recalculate uv_id edge attributes
edge_gdf = ig_utils.get_edge_gdf(graph, ig_attrs=['source', 'target'])
set_uv(graph, edge_gdf)
# export clipped graph
ig_utils.export_to_graphml(graph,
out_graph_research_hel,
n_attrs=out_node_attrs,
e_attrs=out_edge_attrs)
import pytest
from aqi_updater import aq_processing
from aqi_updater.aqi_updater import AqiUpdater
from common.igraph import Edge as E
from aqi_updater.tests.conftest import test_data_dir, aqi_updates_dir
import common.igraph as ig_utils
import rasterio
import numpy as np
import pandas as pd
import json
graph = ig_utils.read_graphml(fr'{test_data_dir}kumpula.graphml')
aqi_updater = AqiUpdater(graph, test_data_dir, aqi_updates_dir)
aqi_updater.create_aqi_update_csv('aqi_2020-10-10T08.tif')
aqi_updater.finish_aqi_update()
def test_extract_rt_aqi_from_zip():
zip_file = 'allPollutants_2021-02-26T14.zip'
aq_file = aq_processing.extract_zipped_aq_file(test_data_dir, zip_file)
assert aq_file == 'allPollutants_2021-02-26T14.nc'
def test_convert_aqi_nc_to_tif():
aq_nc_file = 'allPollutants_2021-02-26T14.nc'
aq_tif = aq_processing.convert_aq_nc_to_tif(test_data_dir, aq_nc_file)
assert aq_tif == 'aqi_2021-02-26T14.tif'
def test_fillna_in_aqi_raster():
aq_tif = 'aqi_2021-02-26T14.tif'
import pandas as pd
graph_dir = r'graphs'
graph_id = r'kumpula'
# graph_id = r'hma_r_hel-clip'
aqi_update_fp = fr'aqi_updates/yearly_2019_aqi_avg_sum_{graph_id}.csv'
out_csv_fp = fr'examples/{graph_id}_edges.csv'
edge_attrs_in = [E.id_ig, E.id_way, E.length, E.gvi, E.aqi,
E.noises] # geometry is read by default
edge_attrs_out = [
E.id_ig.name, E.length.name, E.gvi.name, E.aqi.name, E.noises.name, 'mdB'
] # only these are exported to CSV
graph = ig_utils.read_graphml(fr'{graph_dir}/{graph_id}.graphml')
edges = ig_utils.get_edge_gdf(graph, attrs=edge_attrs_in, drop_na_geoms=True)
# edges = edges.drop_duplicates(E.id_way.name) # keep only edges unique by geometry
# ensure sum of noise exposure is length by adding missing exposures to 40dB
edges[E.noises.name] = edges.apply(
lambda row: noise_exps.add_db_40_exp_to_noises(row[E.noises.name], row[
E.length.name]),
axis=1)
edges['mdB'] = edges.apply(lambda row: noise_exps.get_mean_noise_level(
row[E.noises.name], row[E.length.name]),
axis=1)
# stringify noises dict
edges[E.noises.name] = [str(noises) for noises in edges[E.noises.name]]
# join AQI to edge data
def get_previously_processed_max_id(csv_dir: str):
csv_files = os.listdir(csv_dir)
max_ids = [int(name.split('_')[0]) for name in csv_files]
return max(max_ids) if max_ids else 0
def export_edge_noise_csv(edge_noises: pd.DataFrame, out_dir: str):
max_id = edge_noises[E.id_ig.name].max()
csv_name = f'{max_id}_edge_noises.csv'
edge_noises.to_csv(out_dir + csv_name)
if (__name__ == '__main__'):
log = Logger(printing=True, log_file='noise_graph_join.log', level='debug')
graph = ig_utils.read_graphml('data/hma.graphml')
log.info(f'read graph of {graph.ecount()} edges')
edge_gdf = ig_utils.get_edge_gdf(graph, attrs=[E.id_ig])
edge_gdf = edge_gdf.sort_values(E.id_ig.name)
# read noise data
noise_layer_names = [
layer for layer in fiona.listlayers('data/noise_data_processed.gpkg')
]
noise_layers = {
name: gpd.read_file('data/noise_data_processed.gpkg', layer=name)
for name in noise_layer_names
}
noise_layers = {
name: gdf.rename(columns={'db_low': name})
for name, gdf in noise_layers.items()
def graph():
graph = ig_utils.read_graphml(fr'{test_data_dir}kumpula.graphml')
yield graph
import logging
import os
import time
import traceback
from aqi_updater.aqi_fetcher import AqiFetcher
from aqi_updater.aqi_updater import AqiUpdater
import aqi_updater.configuration
import common.igraph as ig_utils
log = logging.getLogger('main')
graph_subset = eval(os.getenv('GRAPH_SUBSET', 'False'))
graph = ig_utils.read_graphml(
'graphs/kumpula.graphml' if graph_subset else 'graphs/hma.graphml')
aqi_fetcher = AqiFetcher('aqi_cache/')
aqi_updater = AqiUpdater(graph, 'aqi_cache/', 'aqi_updates/')
def fetch_process_aqi_data():
try:
aqi_fetcher.fetch_process_current_aqi_data()
log.info('AQI fetch & processing succeeded')
except Exception:
log.error(traceback.format_exc())
log.error(
f'Failed to process AQI data to {aqi_fetcher.wip_aqi_tif}, retrying in 30s'
)
time.sleep(30)
finally:
aqi_fetcher.finish_aqi_fetch()
graph_id = 'hma_r_hel-clip' # 'hma'
aqi_tif_name = r'yearly_2019_aqi_avg_sum.tiff'
mean_aqi_tif = fr'aq_data_import/data/{aqi_tif_name}'
fillna_aqi = False
graph_file = fr'graphs/{graph_id}.graphml'
aq_update_out_file = fr'aqi_updates/yearly_2019_aqi_avg_sum_{graph_id}.csv'
aq_attr_name = 'aqi'
if fillna_aqi:
aq_processing.fillna_in_raster(r'aq_data_import/data/',
aqi_tif_name,
na_val=1.001,
log=log)
graph = ig_utils.read_graphml(graph_file)
edge_point_gdf = aq_sampling.get_sampling_point_gdf_from_graph(graph)
log.info(f'Loaded {len(edge_point_gdf)} edges for AQI sampling')
sampling_gdf = edge_point_gdf.drop_duplicates(E.id_way.name)
log.info(f'Created {len(sampling_gdf)} sampling points')
aqi_sample_df = aq_sampling.sample_aq_to_point_gdf(sampling_gdf, mean_aqi_tif,
aq_attr_name)
aqi_sample_df = aq_sampling.validate_aqi_sample_df(aqi_sample_df, aq_attr_name,
log)
final_edge_aqi_samples = aq_sampling.merge_edge_aq_samples(
edge_point_gdf, aqi_sample_df, aq_attr_name, log)
log.info(f'Combined AQI samples for {len(final_edge_aqi_samples)} edges')
log.info(
f'found {real_edge_count - len(edges_within)} edges of {real_edge_count} outside noise data extent'
)
# set noise attributes of edges within the data extent to default values (no noise)
for edge in edges_within.itertuples():
graph.es[getattr(edge, E.id_ig.name)][E.noises.value] = {}
graph.es[getattr(edge, E.id_ig.name)][E.noise_source.value] = ''
if (__name__ == '__main__'):
log = Logger(printing=True,
log_file='noise_graph_update.log',
level='debug')
in_graph_file = 'data/hma.graphml'
out_graph_file = 'out_graph/hma.graphml'
data_extent_file = 'data/HMA.geojson'
noise_csv_dir = 'out_csv/'
data_extent: Polygon = geom_utils.project_geom(
gpd.read_file(data_extent_file)['geometry'][0])
graph = ig_utils.read_graphml(in_graph_file, log)
set_default_and_na_edge_noises(graph, data_extent, log)
noise_graph_update(graph, noise_csv_dir, log)
ig_utils.export_to_graphml(graph, out_graph_file)
log.info(f'exported graph of {graph.ecount()} edges')
log.info('all done')
def graph() -> Graph:
g = ig_utils.read_graphml(test_graph_fp)
g.es[E.id_way.value] = list(g.es[E.id_ig.value])
yield g
def main(conf: GraphGreenViewJoinConf):
edge_table_db_name = conf.db_edge_table
execute_sql = db.get_sql_executor(log)
db_tables = db.get_db_table_names(execute_sql)
# load GSV GVI points from GPKG
gsv_gvi_gdf = load_gsv_gvi_gdf(conf.greenery_points_fp)
# load street network graph from GraphML
graph = ig_utils.read_graphml(conf.graph_file_in)
log.info(f'Read graph of {graph.ecount()} edges')
# load edge_gdf
edge_gdf: GeoDataFrame = ig_utils.get_edge_gdf(
graph, attrs=[E.id_ig, E.length, E.id_way])
edge_gdf = edge_gdf.drop_duplicates(E.id_way.name, keep='first')
# drop edges without geometry
edge_gdf = edge_gdf[edge_gdf['geometry'].apply(
lambda geom: isinstance(geom, LineString))]
log.info(f'Subset edge_gdf to {len(edge_gdf)} unique geometries')
# export edges to db if not there yet for land cover overlay analysis
if edge_table_db_name not in db_tables:
# add simplified buffers to edge_gdf
edges_2_db = edge_gdf.copy()
log.info('Calculating 30m buffers from edge geometries')
edges_2_db['b30'] = [
geom.buffer(30, resolution=3) for geom in edges_2_db['geometry']
]
edges_2_db = edges_2_db.rename(columns={
'geometry': 'line_geom',
'b30': 'geometry'
})
edges_2_db = edges_2_db.set_geometry('geometry')
log.info('Writing edges to PostGIS')
write_to_postgis = db.get_db_writer(log)
write_to_postgis(edges_2_db[[E.id_way.name, 'geometry']],
edge_table_db_name)
log.info(
'Wrote graph edges to db, run land_cover_overlay_analysis.py next')
exit()
else:
log.info(
f'Edges were already exported to db table: {edge_table_db_name}')
# get mean GSV GVI per edge
gsv_gvi_list_by_way_id = get_gsv_gvi_list_by_way_id(edge_gdf, gsv_gvi_gdf)
mean_gsv_gvi_by_way_id = get_mean_gsv_gvi_by_way_id(
gsv_gvi_list_by_way_id, edge_gdf)
# fetch low and high vegetation shares from db per edge buffer (way ID)
low_veg_share_by_way_id = lc_analysis.get_low_veg_share_by_way_id(
conf.db_low_veg_share_table)
high_veg_share_by_way_id = lc_analysis.get_high_veg_share_by_way_id(
conf.db_high_veg_share_table)
graph = update_gvi_attributes_to_graph(graph, mean_gsv_gvi_by_way_id,
low_veg_share_by_way_id,
high_veg_share_by_way_id)
ig_utils.export_to_graphml(graph, conf.graph_file_out)
log.info(f'Exported graph to file {conf.graph_file_out}')
import sys
sys.path.append('..')
import common.igraph as ig_utils
from common.igraph import Edge as E, Node as N
import geopandas as gpd
import utils
subset = False
graph_name = 'kumpula' if subset else 'hma'
graph = ig_utils.read_graphml(fr'graph_in/{graph_name}.graphml')
hel_extent = gpd.read_file(r'hel.geojson')
out_graph = fr'graph_out/{graph_name}.graphml'
out_graph_research = fr'graph_out/{graph_name}_r.graphml'
out_graph_research_hel = fr'graph_out/{graph_name}_r_hel-clip.graphml'
out_geojson_noise_gvi = fr'graph_out/{graph_name}_noise_gvi.geojson'
out_geojson = fr'graph_out/{graph_name}.geojson'
out_node_attrs = [N.geometry]
out_edge_attrs = [
E.id_ig, E.uv, E.id_way, E.geometry, E.geom_wgs, E.length, E.length_b,
E.noises, E.gvi
]
def set_biking_lengths(graph, edge_gdf):
for edge in edge_gdf.itertuples():
length = getattr(edge, E.length.name)
biking_length = length * getattr(
edge, E.bike_safety_factor.name) if length != 0.0 else 0.0
