def test_noise_graph_update(self): in_graph_file = 'data/test_graph.graphml' out_graph_file = 'temp/test_graph_noises.graphml' data_extent_file = 'data/HMA.geojson' noise_csv_dir = 'noise_csv/' data_extent: Polygon = geom_utils.project_geom( gpd.read_file(data_extent_file)['geometry'][0]) graph = ig_utils.read_graphml(in_graph_file, log) noise_graph_update.set_default_and_na_edge_noises( graph, data_extent, log) noise_graph_update.noise_graph_update(graph, noise_csv_dir, log) ig_utils.export_to_graphml(graph, out_graph_file) graph = ig_utils.read_graphml(out_graph_file) self.assertEqual(graph.ecount(), 3702) for edge in graph.es: attrs = edge.attributes() # check that edge IDs are correct self.assertEqual(edge.index, attrs[E.id_ig.value]) if isinstance(attrs[E.geometry.value], LineString): # note: this will fail if some of the edges are outside the noise data extent self.assertNotEqual(edge[E.noises.value], None) self.assertIsInstance(edge[E.noises.value], dict) self.assertNotEqual(edge[E.noise_source.value], None) self.assertIsInstance(edge[E.noise_source.value], str) else: # for edges without geometry the noise attributes should be nodata self.assertEqual(edge[E.noises.value], None) self.assertEqual(edge[E.noise_source.value], None) # if edge noises are nodata then also noise source must be nodata if edge[E.noises.value] == None: self.assertEqual(edge[E.noise_source.value], None) # if edge noises are not nodata but {} then noise source must also be just '' (not nodata) if edge[E.noises.value] == {}: self.assertEqual(edge[E.noise_source.value], '') # if edge has noises it must also have noise source if edge[E.noises.value]: self.assertNotEqual(edge[E.noise_source.value], '') self.assertNotEqual(edge[E.noise_source.value], None) # if edge has noise source it must have also noises if edge[E.noise_source.value]: self.assertNotEqual(edge[E.noises.value], '') self.assertNotEqual(edge[E.noises.value], None)
def test_updates_noises_from_csv_to_graph(): in_graph_file = f'{base_dir}/data/test_graph.graphml' out_graph_file = f'{base_dir}/temp/test_graph_noises.graphml' data_extent_file = f'{base_dir}/data/HMA.geojson' noise_csv_dir = f'{base_dir}/noise_csv/' data_extent: Polygon = geom_utils.project_geom(gpd.read_file(data_extent_file)['geometry'][0]) graph = ig_utils.read_graphml(in_graph_file) noise_graph_update.set_default_and_na_edge_noises(graph, data_extent) noise_graph_update.noise_graph_update(graph, noise_csv_dir) ig_utils.export_to_graphml(graph, out_graph_file) graph = ig_utils.read_graphml(out_graph_file) assert graph.ecount() == 3702 for edge in graph.es: attrs = edge.attributes() # check that edge IDs are correct assert edge.index == attrs[E.id_ig.value] if isinstance(attrs[E.geometry.value], LineString): # note: this will fail if some of the edges are outside the noise data extent assert edge[E.noises.value] is not None assert isinstance(edge[E.noises.value], dict) assert edge[E.noise_source.value] is not None assert isinstance(edge[E.noise_source.value], str) else: # for edges without geometry the noise attributes should be nodata assert edge[E.noises.value] is None assert edge[E.noise_source.value] is None # if edge noises are nodata then also noise source must be nodata if edge[E.noises.value] is None: assert edge[E.noise_source.value] is None # if edge noises are not nodata but {} then noise source must also be just '' (not nodata) if edge[E.noises.value] == {}: assert edge[E.noise_source.value] == '' # if edge has noises it must also have noise source if edge[E.noises.value]: assert edge[E.noise_source.value] != '' assert edge[E.noise_source.value] is not None # if edge has noise source it must have also noises if edge[E.noise_source.value]: assert edge[E.noises.value] != '' assert edge[E.noises.value] is not None
def __init__(self, logger: Logger, graph_file: str, routing_conf: RoutingConf): """Initializes a graph (and related features) used by green_paths_app and aqi_processor_app. Args: subset: A boolean variable indicating whether a subset of the graph should be loaded (subset is for testing / developing). """ self.log = logger self.log.info(f'Loading graph from file: {graph_file}') start_time = time.time() self.graph = ig_utils.read_graphml(graph_file) self.routing_conf = routing_conf self.ecount = self.graph.ecount() self.vcount = self.graph.vcount() self.log.info(f'Graph of {self.graph.ecount()} edges read') self.__edge_gdf = self.__get_edge_gdf() self.__edge_sindex = self.__edge_gdf.sindex self.__node_gdf = ig_utils.get_node_gdf(self.graph, drop_na_geoms=True) self.__nodes_sind = self.__node_gdf.sindex if conf.cycling_enabled: edge_cost_factory.set_biking_costs(self.graph, self.log) if conf.quiet_paths_enabled: edge_cost_factory.set_noise_costs_to_edges(self.graph, routing_conf) self.log.info('Noise costs set') if conf.gvi_paths_enabled: edge_cost_factory.set_gvi_costs_to_graph(self.graph, routing_conf) self.log.info('GVI costs set') self.graph.es[E.aqi.value] = None # set default AQI value to None self.log.duration(start_time, 'Graph initialized', log_level='info') self.__path_edge_cache: Dict[int, PathEdge] = {}
def test_add_sampling_points(self): graph = ig_utils.read_graphml('data/test_graph.graphml') gdf = ig_utils.get_edge_gdf(graph) # start_time = time.time() gdf = utils.add_sampling_points_to_gdf(gdf, 2) # log.duration(start_time, 'added sampling points') sampling_points_list = list(gdf['sampling_points']) self.assertEqual( len([sps for sps in sampling_points_list if sps != None]), 3522) self.assertEqual( len([sps for sps in sampling_points_list if sps == None]), 180) # test that all sample points are on the line geometries for edge in gdf.itertuples(): sampling_points = getattr(edge, 'sampling_points') if (sampling_points == None): continue line_geom = getattr(edge, 'geometry') for sp in sampling_points: self.assertAlmostEqual(sp.distance(line_geom), 0, 5) # validate sampling point gdf (exploaded from edge gdf with sampling points) sampling_gdf = utils.explode_sampling_point_gdf(gdf, 'sampling_points') self.assertGreater(len(sampling_gdf), len(gdf)) self.assertEqual(len(sampling_gdf), 58554) # check that the total representative length of each set of sampling points equals the length of the respective edge sps_by_edge = sampling_gdf.groupby('edge_id') for edge in gdf.itertuples(): if (edge.sampling_points != None): edge_sps = sps_by_edge.get_group(edge.Index) sampling_length_sum = edge_sps['sample_len'].sum() self.assertAlmostEqual(sampling_length_sum, edge.geometry.length, 5)
def test_adds_sampling_points_to_edge_gdf(): graph = ig_utils.read_graphml(f'{base_dir}/data/test_graph.graphml') gdf = ig_utils.get_edge_gdf(graph) # start_time = time.time() gdf = noise_join_utils.add_sampling_points_to_gdf(gdf, 2) # log.duration(start_time, 'added sampling points') sampling_points_list = list(gdf['sampling_points']) assert len([sps for sps in sampling_points_list if sps != None]) == 3522 assert len([sps for sps in sampling_points_list if sps == None]) == 180 # test that all sample points are on the line geometries for edge in gdf.itertuples(): sampling_points = getattr(edge, 'sampling_points') if not sampling_points: continue line_geom = getattr(edge, 'geometry') for sp in sampling_points: assert round(sp.distance(line_geom), 1) == 0, 5 # validate sampling point gdf (exploaded from edge gdf with sampling points) sampling_gdf = noise_join_utils.explode_sampling_point_gdf(gdf, 'sampling_points') assert len(sampling_gdf) > len(gdf) assert len(sampling_gdf) == 58554 # check that the total representative length of each set of sampling points equals the length of the respective edge sps_by_edge = sampling_gdf.groupby('edge_id') for edge in gdf.itertuples(): if (edge.sampling_points != None): edge_sps = sps_by_edge.get_group(edge.Index) sampling_length_sum = edge_sps['sample_len'].sum() assert round(sampling_length_sum, 2) == round(edge.geometry.length, 2)
def test_joins_noises_to_graph_edges(): graph = ig_utils.read_graphml(f'{base_dir}/data/test_graph.graphml') edge_gdf = ig_utils.get_edge_gdf(graph, attrs=[E.id_ig, E.length]) edge_gdf[E.id_ig.name] = edge_gdf.index # read noise data noise_layer_names = [layer for layer in fiona.listlayers(f'{base_dir}/data/noise_data_processed.gpkg')] noise_layers = {name: gpd.read_file(f'{base_dir}/data/noise_data_processed.gpkg', layer=name) for name in noise_layer_names} noise_layers = {name: gdf.rename(columns={'db_low': name}) for name, gdf in noise_layers.items()} # read nodata zone: narrow area between noise surfaces of different municipalities nodata_layer = gpd.read_file(f'{base_dir}/data/extents.gpkg', layer='municipal_boundaries') edge_noises = noise_graph_join.noise_graph_join( edge_gdf=edge_gdf, sampling_interval=3, noise_layers=noise_layers, nodata_layer=nodata_layer ) assert edge_noises[E.id_ig.name].nunique() == 3522 edge_noises_df = pd.merge(edge_gdf, edge_noises, how='inner', on=E.id_ig.name) edge_noises_df['total_noise_len'] = [round(sum(noises.values()), 4) for noises in edge_noises_df['noises']] def validate_edge_noises(row): assert round(row['total_noise_len'], 1) <= round(row['length'], 1) edge_noises_df.apply(lambda row: validate_edge_noises(row), axis=1) assert round(edge_noises_df['total_noise_len'].mean(), 2) == 33.20 # test frequency of different main noise sources noise_sources = dict(Counter(list(edge_noises_df[E.noise_source.name]))) assert noise_sources == {'road': 2322, 'train': 1198, '': 2}
def main(conf: GraphNoiseJoinConf): data_extent: Polygon = geom_utils.project_geom( gpd.read_file(conf.noise_data_extent_fp)['geometry'][0]) graph = ig_utils.read_graphml(conf.graph_in_fp, log) set_default_and_na_edge_noises(graph, data_extent) noise_graph_update(graph, conf.noise_data_csv_dir) ig_utils.export_to_graphml(graph, conf.graph_out_fp) log.info(f'exported graph of {graph.ecount()} edges') log.info('all done')
def main(conf: GraphNoiseJoinConf): graph = ig_utils.read_graphml(conf.graph_in_fp) log.info(f'read graph of {graph.ecount()} edges') edge_gdf = ig_utils.get_edge_gdf(graph, attrs=[E.id_ig]) edge_gdf = edge_gdf.sort_values(E.id_ig.name) # read noise data noise_layer_names = [ layer for layer in fiona.listlayers(conf.noise_data_fp) ] noise_layers = { name: gpd.read_file(conf.noise_data_fp, layer=name) for name in noise_layer_names } noise_layers = { name: gdf.rename(columns={'db_low': name}) for name, gdf in noise_layers.items() } log.info(f'read {len(noise_layers)} noise layers') # read nodata zone: narrow area between noise surfaces of different municipalities nodata_layer = gpd.read_file(conf.nodata_fp, layer=conf.nodata_layer_name) # process chunks of edges together by dividing gdf to parts processing_size = 50000 split_gdf_count = math.ceil(len(edge_gdf) / processing_size) gdfs = np.array_split(edge_gdf, split_gdf_count) # get max id of previously processed edges max_processed_id = get_previously_processed_max_id(conf.noise_data_csv_dir) if max_processed_id > 0: log.info( f'found previously processed edges up to edge id {max_processed_id}' ) for idx, gdf in enumerate(gdfs): if gdf[E.id_ig.name].max() <= max_processed_id: log.info( f'skipping {idx+1} of {len(gdfs)} edge gdfs (processed before)' ) continue else: log.info(f'processing {idx+1} of {len(gdfs)} edge gdfs') edge_noises = noise_graph_join( edge_gdf=gdf, sampling_interval=3, noise_layers=noise_layers, nodata_layer=nodata_layer, b_debug=False, debug_gpkg='debug/noise_join_debug.gpkg') export_edge_noise_csv(edge_noises, conf.noise_data_csv_dir)
def test_graph_to_gdf(self): graph = ig_utils.read_graphml('data/test_graph.graphml', log=Logger(printing=True)) # test read graph to wgs gdf gdf = ig_utils.get_edge_gdf(graph, id_attr=Edge.id_ig, attrs=[Edge.length], geom_attr=Edge.geom_wgs) gdf['geom_length'] = [geom.length for geom in gdf[Edge.geom_wgs.name]] self.assertAlmostEqual(gdf['geom_length'].mean(), 0.000429, 6) # test read to projected gdf gdf = ig_utils.get_edge_gdf(graph, id_attr=Edge.id_ig, attrs=[Edge.length], geom_attr=Edge.geometry) gdf['geom_length'] = [geom.length for geom in gdf[Edge.geometry.name]] self.assertAlmostEqual(gdf['geom_length'].mean(), 31.65, 2)
def test_gets_graph_data_as_gdf(): graph = ig_utils.read_graphml(conf.igraph_out_file) # test read graph to wgs gdf gdf = ig_utils.get_edge_gdf(graph, id_attr=Edge.id_ig, attrs=[Edge.length], geom_attr=Edge.geom_wgs, drop_na_geoms=True) gdf['geom_length'] = [geom.length for geom in gdf[Edge.geom_wgs.name]] assert round(gdf['geom_length'].mean(), 6) == 0.000451 # test read to projected gdf gdf = ig_utils.get_edge_gdf(graph, id_attr=Edge.id_ig, attrs=[Edge.length], geom_attr=Edge.geometry, drop_na_geoms=True) gdf['geom_length'] = [geom.length for geom in gdf[Edge.geometry.name]] assert round(gdf['geom_length'].mean(), 2) == 33.27
def test_read_igraph(self): graph = ig_utils.read_graphml('temp/test_graph.graphml', log=Logger(printing=True)) self.assertEqual(graph.ecount(), 3702) self.assertEqual(graph.vcount(), 1328) attr_names = list(graph.vs[0].attributes().keys()) for attr in attr_names: self.assertIn(attr, [e.value for e in Node], 'no unknown node attributes allowed') attr_names = list(graph.es[0].attributes().keys()) for attr in attr_names: self.assertIn(attr, [e.value for e in Edge], 'no unknown edge attributes allowed') for n in graph.vs: attrs = n.attributes() self.assertEqual(attrs[Node.id_ig.value], n.index) self.assertIsInstance(attrs[Node.id_ig.value], int) self.assertIsInstance(attrs[Node.id_otp.value], str) self.assertIsInstance(attrs[Node.name_otp.value], str) self.assertIsInstance(attrs[Node.geometry.value], Point) self.assertIsInstance(attrs[Node.geom_wgs.value], Point) self.assertIsInstance(attrs[Node.traversable_walking.value], bool) self.assertIsInstance(attrs[Node.traversable_biking.value], bool) self.assertIsInstance(attrs[Node.traffic_light.value], bool) for e in graph.es: attrs = e.attributes() self.assertEqual(attrs[Edge.id_ig.value], e.index) self.assertIsInstance(attrs[Edge.id_ig.value], int) self.assertIsInstance(attrs[Edge.id_otp.value], str) self.assertIsInstance(attrs[Edge.name_otp.value], str) self.assertIsInstance(attrs[Edge.geometry.value], (LineString, GeometryCollection)) self.assertIsInstance(attrs[Edge.geom_wgs.value], (LineString, GeometryCollection)) self.assertIsInstance(attrs[Edge.length.value], float) self.assertIsInstance(attrs[Edge.edge_class.value], str) self.assertIsInstance(attrs[Edge.street_class.value], str) self.assertIsInstance(attrs[Edge.is_stairs.value], bool) self.assertIsInstance(attrs[Edge.is_no_thru_traffic.value], bool) self.assertIsInstance(attrs[Edge.allows_walking.value], bool) self.assertIsInstance(attrs[Edge.allows_biking.value], bool) self.assertIsInstance(attrs[Edge.traversable_walking.value], bool) self.assertIsInstance(attrs[Edge.traversable_biking.value], bool) self.assertIsInstance(attrs[Edge.bike_safety_factor.value], float)
def test_reads_the_created_igraph(): graph = ig_utils.read_graphml(conf.igraph_out_file) assert graph.ecount() == 3702 assert graph.vcount() == 1328 attr_names = list(graph.vs[0].attributes().keys()) for attr in attr_names: assert attr in [e.value for e in Node] # no unknown node attributes allowed attr_names = list(graph.es[0].attributes().keys()) for attr in attr_names: assert attr in [e.value for e in Edge] # no unknown edge attributes allowed for n in graph.vs: attrs = n.attributes() assert attrs[Node.id_ig.value] == n.index assert isinstance(attrs[Node.id_ig.value], int) assert isinstance(attrs[Node.id_otp.value], str) assert isinstance(attrs[Node.name_otp.value], str) assert isinstance(attrs[Node.geometry.value], Point) assert isinstance(attrs[Node.geom_wgs.value], Point) assert isinstance(attrs[Node.traversable_walking.value], bool) assert isinstance(attrs[Node.traversable_biking.value], bool) assert isinstance(attrs[Node.traffic_light.value], bool) for e in graph.es: attrs = e.attributes() assert attrs[Edge.id_ig.value] == e.index assert isinstance(attrs[Edge.id_ig.value], int) assert isinstance(attrs[Edge.id_otp.value], str) assert isinstance(attrs[Edge.name_otp.value], str) assert isinstance(attrs[Edge.geometry.value], (LineString, GeometryCollection)) assert isinstance(attrs[Edge.geom_wgs.value], (LineString, GeometryCollection)) assert isinstance(attrs[Edge.length.value], float) assert isinstance(attrs[Edge.edge_class.value], str) assert isinstance(attrs[Edge.street_class.value], str) assert isinstance(attrs[Edge.is_stairs.value], bool) assert isinstance(attrs[Edge.is_no_thru_traffic.value], bool) assert isinstance(attrs[Edge.allows_walking.value], bool) assert isinstance(attrs[Edge.allows_biking.value], bool) assert isinstance(attrs[Edge.traversable_walking.value], bool) assert isinstance(attrs[Edge.traversable_biking.value], bool) assert isinstance(attrs[Edge.bike_safety_factor.value], float)
def graph_export(conf: GraphExportConf, ): in_graph = fr'{conf.base_dir}/graph_in/{conf.graph_id}.graphml' out_graph = fr'{conf.base_dir}/graph_out/{conf.graph_id}.graphml' out_graph_research = fr'{conf.base_dir}/graph_out/{conf.graph_id}_r.graphml' out_graph_research_hel = fr'{conf.base_dir}/graph_out/{conf.graph_id}_r_hel-clip.graphml' out_geojson_noise_gvi = fr'{conf.base_dir}/graph_out/{conf.graph_id}_noise_gvi.geojson' out_geojson = fr'{conf.base_dir}/graph_out/{conf.graph_id}.geojson' hel_extent = gpd.read_file(conf.hel_extent_fp) out_node_attrs = [N.geometry] out_edge_attrs = [ E.id_ig, E.uv, E.id_way, E.geometry, E.geom_wgs, E.length, E.allows_biking, E.is_stairs, E.bike_safety_factor, E.noises, E.gvi ] if not conf.with_noise_data: out_edge_attrs.remove(E.noises) if not conf.with_greenery_data: out_edge_attrs.remove(E.gvi) log.info(f'Reading graph file: {in_graph}') graph = ig_utils.read_graphml(in_graph) edge_gdf = ig_utils.get_edge_gdf(graph, attrs=[E.id_ig, E.length], ig_attrs=['source', 'target']) set_uv(graph, edge_gdf) set_way_ids(graph, edge_gdf) graph.es[E.bike_safety_factor.value] = [ round(v, 2) if (v and np.isfinite(v)) else 1 for v in graph.es[E.bike_safety_factor.value] ] # set combined GVI to GVI attribute & export graph graph.es[E.gvi.value] = list(graph.es[E.gvi_comb_gsv_veg.value]) ig_utils.export_to_graphml(graph, out_graph, n_attrs=out_node_attrs, e_attrs=out_edge_attrs) # create GeoJSON files for vector tiles geojson = utils.create_geojson(graph) utils.write_geojson(geojson, out_geojson, overwrite=True, id_attr=True) utils.write_geojson(geojson, out_geojson_noise_gvi, overwrite=True, db_prop=True, gvi_prop=True) # for research use, set combined GVI that omits low vegetation to GVI attribute and export graph graph.es[E.gvi.value] = list(graph.es[E.gvi_comb_gsv_high_veg.value]) ig_utils.export_to_graphml(graph, out_graph_research, n_attrs=out_node_attrs, e_attrs=out_edge_attrs) # export clip of the graph by the extent of Helsinki node_gdf = ig_utils.get_node_gdf(graph, attrs=[N.id_ig]) # replace geometry with buffered one (500 m) hel_extent['geometry'] = [ geom.buffer(500) for geom in hel_extent['geometry'] ] inside_hel = gpd.sjoin(node_gdf, hel_extent) inside_hel_ids = list(inside_hel[N.id_ig.name]) outside_hel_ids = [ id_ig for id_ig in list(node_gdf[N.id_ig.name]) if id_ig not in inside_hel_ids ] graph.delete_vertices(outside_hel_ids) # delete isolated nodes del_node_ids = [v.index for v in graph.vs.select(_degree_eq=0)] graph.delete_vertices(del_node_ids) # reassign igraph indexes to edge and node attributes graph.es[E.id_ig.value] = [e.index for e in graph.es] graph.vs[N.id_ig.value] = [v.index for v in graph.vs] # recalculate uv_id edge attributes edge_gdf = ig_utils.get_edge_gdf(graph, ig_attrs=['source', 'target']) set_uv(graph, edge_gdf) # export clipped graph ig_utils.export_to_graphml(graph, out_graph_research_hel, n_attrs=out_node_attrs, e_attrs=out_edge_attrs)
import pytest from aqi_updater import aq_processing from aqi_updater.aqi_updater import AqiUpdater from common.igraph import Edge as E from aqi_updater.tests.conftest import test_data_dir, aqi_updates_dir import common.igraph as ig_utils import rasterio import numpy as np import pandas as pd import json graph = ig_utils.read_graphml(fr'{test_data_dir}kumpula.graphml') aqi_updater = AqiUpdater(graph, test_data_dir, aqi_updates_dir) aqi_updater.create_aqi_update_csv('aqi_2020-10-10T08.tif') aqi_updater.finish_aqi_update() def test_extract_rt_aqi_from_zip(): zip_file = 'allPollutants_2021-02-26T14.zip' aq_file = aq_processing.extract_zipped_aq_file(test_data_dir, zip_file) assert aq_file == 'allPollutants_2021-02-26T14.nc' def test_convert_aqi_nc_to_tif(): aq_nc_file = 'allPollutants_2021-02-26T14.nc' aq_tif = aq_processing.convert_aq_nc_to_tif(test_data_dir, aq_nc_file) assert aq_tif == 'aqi_2021-02-26T14.tif' def test_fillna_in_aqi_raster(): aq_tif = 'aqi_2021-02-26T14.tif'
import pandas as pd graph_dir = r'graphs' graph_id = r'kumpula' # graph_id = r'hma_r_hel-clip' aqi_update_fp = fr'aqi_updates/yearly_2019_aqi_avg_sum_{graph_id}.csv' out_csv_fp = fr'examples/{graph_id}_edges.csv' edge_attrs_in = [E.id_ig, E.id_way, E.length, E.gvi, E.aqi, E.noises] # geometry is read by default edge_attrs_out = [ E.id_ig.name, E.length.name, E.gvi.name, E.aqi.name, E.noises.name, 'mdB' ] # only these are exported to CSV graph = ig_utils.read_graphml(fr'{graph_dir}/{graph_id}.graphml') edges = ig_utils.get_edge_gdf(graph, attrs=edge_attrs_in, drop_na_geoms=True) # edges = edges.drop_duplicates(E.id_way.name) # keep only edges unique by geometry # ensure sum of noise exposure is length by adding missing exposures to 40dB edges[E.noises.name] = edges.apply( lambda row: noise_exps.add_db_40_exp_to_noises(row[E.noises.name], row[ E.length.name]), axis=1) edges['mdB'] = edges.apply(lambda row: noise_exps.get_mean_noise_level( row[E.noises.name], row[E.length.name]), axis=1) # stringify noises dict edges[E.noises.name] = [str(noises) for noises in edges[E.noises.name]] # join AQI to edge data
def get_previously_processed_max_id(csv_dir: str): csv_files = os.listdir(csv_dir) max_ids = [int(name.split('_')[0]) for name in csv_files] return max(max_ids) if max_ids else 0 def export_edge_noise_csv(edge_noises: pd.DataFrame, out_dir: str): max_id = edge_noises[E.id_ig.name].max() csv_name = f'{max_id}_edge_noises.csv' edge_noises.to_csv(out_dir + csv_name) if (__name__ == '__main__'): log = Logger(printing=True, log_file='noise_graph_join.log', level='debug') graph = ig_utils.read_graphml('data/hma.graphml') log.info(f'read graph of {graph.ecount()} edges') edge_gdf = ig_utils.get_edge_gdf(graph, attrs=[E.id_ig]) edge_gdf = edge_gdf.sort_values(E.id_ig.name) # read noise data noise_layer_names = [ layer for layer in fiona.listlayers('data/noise_data_processed.gpkg') ] noise_layers = { name: gpd.read_file('data/noise_data_processed.gpkg', layer=name) for name in noise_layer_names } noise_layers = { name: gdf.rename(columns={'db_low': name}) for name, gdf in noise_layers.items()
def graph(): graph = ig_utils.read_graphml(fr'{test_data_dir}kumpula.graphml') yield graph
import logging import os import time import traceback from aqi_updater.aqi_fetcher import AqiFetcher from aqi_updater.aqi_updater import AqiUpdater import aqi_updater.configuration import common.igraph as ig_utils log = logging.getLogger('main') graph_subset = eval(os.getenv('GRAPH_SUBSET', 'False')) graph = ig_utils.read_graphml( 'graphs/kumpula.graphml' if graph_subset else 'graphs/hma.graphml') aqi_fetcher = AqiFetcher('aqi_cache/') aqi_updater = AqiUpdater(graph, 'aqi_cache/', 'aqi_updates/') def fetch_process_aqi_data(): try: aqi_fetcher.fetch_process_current_aqi_data() log.info('AQI fetch & processing succeeded') except Exception: log.error(traceback.format_exc()) log.error( f'Failed to process AQI data to {aqi_fetcher.wip_aqi_tif}, retrying in 30s' ) time.sleep(30) finally: aqi_fetcher.finish_aqi_fetch()
graph_id = 'hma_r_hel-clip' # 'hma' aqi_tif_name = r'yearly_2019_aqi_avg_sum.tiff' mean_aqi_tif = fr'aq_data_import/data/{aqi_tif_name}' fillna_aqi = False graph_file = fr'graphs/{graph_id}.graphml' aq_update_out_file = fr'aqi_updates/yearly_2019_aqi_avg_sum_{graph_id}.csv' aq_attr_name = 'aqi' if fillna_aqi: aq_processing.fillna_in_raster(r'aq_data_import/data/', aqi_tif_name, na_val=1.001, log=log) graph = ig_utils.read_graphml(graph_file) edge_point_gdf = aq_sampling.get_sampling_point_gdf_from_graph(graph) log.info(f'Loaded {len(edge_point_gdf)} edges for AQI sampling') sampling_gdf = edge_point_gdf.drop_duplicates(E.id_way.name) log.info(f'Created {len(sampling_gdf)} sampling points') aqi_sample_df = aq_sampling.sample_aq_to_point_gdf(sampling_gdf, mean_aqi_tif, aq_attr_name) aqi_sample_df = aq_sampling.validate_aqi_sample_df(aqi_sample_df, aq_attr_name, log) final_edge_aqi_samples = aq_sampling.merge_edge_aq_samples( edge_point_gdf, aqi_sample_df, aq_attr_name, log) log.info(f'Combined AQI samples for {len(final_edge_aqi_samples)} edges')
log.info( f'found {real_edge_count - len(edges_within)} edges of {real_edge_count} outside noise data extent' ) # set noise attributes of edges within the data extent to default values (no noise) for edge in edges_within.itertuples(): graph.es[getattr(edge, E.id_ig.name)][E.noises.value] = {} graph.es[getattr(edge, E.id_ig.name)][E.noise_source.value] = '' if (__name__ == '__main__'): log = Logger(printing=True, log_file='noise_graph_update.log', level='debug') in_graph_file = 'data/hma.graphml' out_graph_file = 'out_graph/hma.graphml' data_extent_file = 'data/HMA.geojson' noise_csv_dir = 'out_csv/' data_extent: Polygon = geom_utils.project_geom( gpd.read_file(data_extent_file)['geometry'][0]) graph = ig_utils.read_graphml(in_graph_file, log) set_default_and_na_edge_noises(graph, data_extent, log) noise_graph_update(graph, noise_csv_dir, log) ig_utils.export_to_graphml(graph, out_graph_file) log.info(f'exported graph of {graph.ecount()} edges') log.info('all done')
def graph() -> Graph: g = ig_utils.read_graphml(test_graph_fp) g.es[E.id_way.value] = list(g.es[E.id_ig.value]) yield g
def main(conf: GraphGreenViewJoinConf): edge_table_db_name = conf.db_edge_table execute_sql = db.get_sql_executor(log) db_tables = db.get_db_table_names(execute_sql) # load GSV GVI points from GPKG gsv_gvi_gdf = load_gsv_gvi_gdf(conf.greenery_points_fp) # load street network graph from GraphML graph = ig_utils.read_graphml(conf.graph_file_in) log.info(f'Read graph of {graph.ecount()} edges') # load edge_gdf edge_gdf: GeoDataFrame = ig_utils.get_edge_gdf( graph, attrs=[E.id_ig, E.length, E.id_way]) edge_gdf = edge_gdf.drop_duplicates(E.id_way.name, keep='first') # drop edges without geometry edge_gdf = edge_gdf[edge_gdf['geometry'].apply( lambda geom: isinstance(geom, LineString))] log.info(f'Subset edge_gdf to {len(edge_gdf)} unique geometries') # export edges to db if not there yet for land cover overlay analysis if edge_table_db_name not in db_tables: # add simplified buffers to edge_gdf edges_2_db = edge_gdf.copy() log.info('Calculating 30m buffers from edge geometries') edges_2_db['b30'] = [ geom.buffer(30, resolution=3) for geom in edges_2_db['geometry'] ] edges_2_db = edges_2_db.rename(columns={ 'geometry': 'line_geom', 'b30': 'geometry' }) edges_2_db = edges_2_db.set_geometry('geometry') log.info('Writing edges to PostGIS') write_to_postgis = db.get_db_writer(log) write_to_postgis(edges_2_db[[E.id_way.name, 'geometry']], edge_table_db_name) log.info( 'Wrote graph edges to db, run land_cover_overlay_analysis.py next') exit() else: log.info( f'Edges were already exported to db table: {edge_table_db_name}') # get mean GSV GVI per edge gsv_gvi_list_by_way_id = get_gsv_gvi_list_by_way_id(edge_gdf, gsv_gvi_gdf) mean_gsv_gvi_by_way_id = get_mean_gsv_gvi_by_way_id( gsv_gvi_list_by_way_id, edge_gdf) # fetch low and high vegetation shares from db per edge buffer (way ID) low_veg_share_by_way_id = lc_analysis.get_low_veg_share_by_way_id( conf.db_low_veg_share_table) high_veg_share_by_way_id = lc_analysis.get_high_veg_share_by_way_id( conf.db_high_veg_share_table) graph = update_gvi_attributes_to_graph(graph, mean_gsv_gvi_by_way_id, low_veg_share_by_way_id, high_veg_share_by_way_id) ig_utils.export_to_graphml(graph, conf.graph_file_out) log.info(f'Exported graph to file {conf.graph_file_out}')
import sys sys.path.append('..') import common.igraph as ig_utils from common.igraph import Edge as E, Node as N import geopandas as gpd import utils subset = False graph_name = 'kumpula' if subset else 'hma' graph = ig_utils.read_graphml(fr'graph_in/{graph_name}.graphml') hel_extent = gpd.read_file(r'hel.geojson') out_graph = fr'graph_out/{graph_name}.graphml' out_graph_research = fr'graph_out/{graph_name}_r.graphml' out_graph_research_hel = fr'graph_out/{graph_name}_r_hel-clip.graphml' out_geojson_noise_gvi = fr'graph_out/{graph_name}_noise_gvi.geojson' out_geojson = fr'graph_out/{graph_name}.geojson' out_node_attrs = [N.geometry] out_edge_attrs = [ E.id_ig, E.uv, E.id_way, E.geometry, E.geom_wgs, E.length, E.length_b, E.noises, E.gvi ] def set_biking_lengths(graph, edge_gdf): for edge in edge_gdf.itertuples(): length = getattr(edge, E.length.name) biking_length = length * getattr( edge, E.bike_safety_factor.name) if length != 0.0 else 0.0