def test_hill_branch_wt_diversity(primal_graph):
for distances, betas in network_generator():
G = primal_graph.copy()
data_dict = mock.mock_data_dict(G)
landuse_labels = mock.mock_categorical_data(len(data_dict))
# easy version
N_easy = networks.NetworkLayerFromNX(G, distances=distances)
D_easy = layers.DataLayerFromDict(data_dict)
D_easy.assign_to_network(N_easy, max_dist=500)
D_easy.hill_branch_wt_diversity(landuse_labels, qs=[0, 1, 2])
# custom version
N_full = networks.NetworkLayerFromNX(G, distances=distances)
D_full = layers.DataLayerFromDict(data_dict)
D_full.assign_to_network(N_full, max_dist=500)
D_full.compute_landuses(landuse_labels,
mixed_use_keys=['hill_branch_wt'],
qs=[0, 1, 2])
# compare
for d in distances:
for q in [0, 1, 2]:
assert np.allclose(
N_easy.metrics['mixed_uses']['hill_branch_wt'][q][d],
N_full.metrics['mixed_uses']['hill_branch_wt'][q][d],
atol=0.001,
rtol=0)
def test_check_data_map(primal_graph):
N = networks.NetworkLayerFromNX(primal_graph, distances=[500])
data_dict = mock.mock_data_dict(primal_graph)
data_uids, data_map = layers.data_map_from_dict(data_dict)
# should throw error if not assigned
with pytest.raises(ValueError):
checks.check_data_map(data_map)
# should work if flag set to False
checks.check_data_map(data_map, check_assigned=False)
# assign then check that it runs as intended
data_map = data.assign_to_network(data_map,
N._node_data,
N._edge_data,
N._node_edge_map,
max_dist=400)
checks.check_data_map(data_map)
# catch zero length data arrays
empty_2d_arr = np.full((0, 4), np.nan)
with pytest.raises(ValueError):
checks.check_data_map(empty_2d_arr)
# catch invalid dimensionality
with pytest.raises(ValueError):
checks.check_data_map(data_map[:, :-1])
def test_check_network_maps(primal_graph):
# network maps
N = networks.NetworkLayerFromNX(primal_graph, distances=[500])
# from cityseer.tools import plot
# plot.plot_networkX_primal_or_dual(primal=G)
# plot.plot_graph_maps(N.uids, N._node_data, N._edge_data)
# catch zero length node and edge arrays
empty_node_arr = np.full((0, 5), np.nan)
with pytest.raises(ValueError):
checks.check_network_maps(empty_node_arr, N._edge_data,
N._node_edge_map)
empty_edge_arr = np.full((0, 4), np.nan)
with pytest.raises(ValueError):
checks.check_network_maps(N._node_data, empty_edge_arr,
N._node_edge_map)
# check that malformed node and data maps throw errors
with pytest.raises(ValueError):
checks.check_network_maps(N._node_data[:, :-1], N._edge_data,
N._node_edge_map)
with pytest.raises(ValueError):
checks.check_network_maps(N._node_data, N._edge_data[:, :-1],
N._node_edge_map)
# catch problematic edge map values
for x in [np.nan, -1]:
# missing start node
corrupted_edges = N._edge_data.copy()
corrupted_edges[0, 0] = x
with pytest.raises(AssertionError):
checks.check_network_maps(N._node_data, corrupted_edges,
N._node_edge_map)
# missing end node
corrupted_edges = N._edge_data.copy()
corrupted_edges[0, 1] = x
with pytest.raises(KeyError):
checks.check_network_maps(N._node_data, corrupted_edges,
N._node_edge_map)
# invalid length
corrupted_edges = N._edge_data.copy()
corrupted_edges[0, 2] = x
with pytest.raises(ValueError):
checks.check_network_maps(N._node_data, corrupted_edges,
N._node_edge_map)
# invalid angle_sum
corrupted_edges = N._edge_data.copy()
corrupted_edges[0, 3] = x
with pytest.raises(ValueError):
checks.check_network_maps(N._node_data, corrupted_edges,
N._node_edge_map)
# invalid imp_factor
corrupted_edges = N._edge_data.copy()
corrupted_edges[0, 4] = x
with pytest.raises(ValueError):
checks.check_network_maps(N._node_data, corrupted_edges,
N._node_edge_map)
def test_compute_accessibilities(primal_graph):
for distances, betas in network_generator():
G = primal_graph.copy()
data_dict = mock.mock_data_dict(G)
landuse_labels = mock.mock_categorical_data(len(data_dict))
# easy version
N_easy = networks.NetworkLayerFromNX(G, distances=distances)
D_easy = layers.DataLayerFromDict(data_dict)
D_easy.assign_to_network(N_easy, max_dist=500)
D_easy.compute_accessibilities(landuse_labels, ['c'])
# custom version
N_full = networks.NetworkLayerFromNX(G, distances=distances)
D_full = layers.DataLayerFromDict(data_dict)
D_full.assign_to_network(N_full, max_dist=500)
D_full.compute_landuses(landuse_labels, accessibility_keys=['c'])
# compare
for d in distances:
for wt in ['weighted', 'non_weighted']:
assert np.allclose(N_easy.metrics['accessibility'][wt]['c'][d],
N_full.metrics['accessibility'][wt]['c'][d],
atol=0.001,
rtol=0)
def test_metrics_to_dict(primal_graph):
# create a network layer and run some metrics
N = networks.NetworkLayerFromNX(primal_graph, distances=[500, 1000])
# check with no metrics
metrics_dict = N.metrics_to_dict()
dict_check(metrics_dict, N)
# check with centrality metrics
N.node_centrality(measures=['node_harmonic'])
metrics_dict = N.metrics_to_dict()
dict_check(metrics_dict, N)
# check with data metrics
data_dict = mock.mock_data_dict(primal_graph)
landuse_labels = mock.mock_categorical_data(len(data_dict))
numerical_data = mock.mock_numerical_data(len(data_dict))
metrics_dict = N.metrics_to_dict()
dict_check(metrics_dict, N)
def test_Network_Layer_From_nX(primal_graph):
node_uids, node_data, edge_data, node_edge_map = graphs.graph_maps_from_nX(primal_graph)
x_arr = node_data[:, 0]
y_arr = node_data[:, 1]
betas = np.array([0.04, 0.02])
distances = networks.distance_from_beta(betas)
# test Network_Layer_From_NetworkX's class
for d, b in zip([distances, None], [None, betas]):
for angular in [True, False]:
N = networks.NetworkLayerFromNX(primal_graph, distances=d, betas=b)
assert np.allclose(N.uids, node_uids, atol=0.001, rtol=0)
assert np.allclose(N._node_data, node_data, atol=0.001, rtol=0)
assert np.allclose(N._edge_data, edge_data, atol=0.001, rtol=0)
assert np.allclose(N.distances, distances, atol=0.001,
rtol=0) # inferred automatically when only betas provided
assert np.allclose(N.betas, betas, atol=0.001,
rtol=0) # inferred automatically when only distances provided
assert N._min_threshold_wt == checks.def_min_thresh_wt
assert np.allclose(N.node_x_arr, x_arr, atol=0.001, rtol=0)
assert np.allclose(N.node_y_arr, y_arr, atol=0.001, rtol=0)
assert np.allclose(N.node_live_arr, node_data[:, 2], atol=0.001, rtol=0)
assert np.allclose(N.edge_lengths_arr, edge_data[:, 2], atol=0.001, rtol=0)
assert np.allclose(N.edge_angles_arr, edge_data[:, 3], atol=0.001, rtol=0)
assert np.allclose(N.edge_impedance_factors_arr, edge_data[:, 4], atol=0.001, rtol=0)
assert np.allclose(N.edge_in_bearings_arr, edge_data[:, 5], atol=0.001, rtol=0)
assert np.allclose(N.edge_out_bearings_arr, edge_data[:, 6], atol=0.001, rtol=0)
# check alternate min_threshold_wt gets passed through successfully
alt_min = 0.02
alt_distances = networks.distance_from_beta(betas, min_threshold_wt=alt_min)
N = networks.NetworkLayerFromNX(primal_graph, betas=betas, min_threshold_wt=alt_min)
assert np.allclose(N.distances, alt_distances, atol=0.001, rtol=0)
# check for malformed signatures
with pytest.raises(TypeError):
networks.NetworkLayerFromNX('boo', distances=distances)
with pytest.raises(ValueError):
networks.NetworkLayerFromNX(primal_graph) # no betas or distances
with pytest.raises(ValueError):
networks.NetworkLayerFromNX(primal_graph, distances=None, betas=None)
with pytest.raises(ValueError):
networks.NetworkLayerFromNX(primal_graph, distances=[])
with pytest.raises(ValueError):
networks.NetworkLayerFromNX(primal_graph, betas=[])
def test_to_networkX(primal_graph):
# also see test_graphs.test_networkX_from_graph_maps for underlying graph maps version
# check round trip to and from graph maps results in same graph
# explicitly set live and weight params for equality checks
# graph_maps_from_networkX generates these implicitly if missing
G = graphs.nX_decompose(primal_graph, decompose_max=20)
for n in G.nodes():
G.nodes[n]['live'] = bool(np.random.randint(0, 1))
for s, e, k in G.edges(keys=True):
G[s][e][k]['imp_factor'] = np.random.randint(0, 2)
# add random data to check persistence at other end
baa_node = None
for n in G.nodes():
baa_node = n
G.nodes[n]['boo'] = 'baa'
break
boo_edge = None
for s, e, k in G.edges(keys=True):
boo_edge = (s, e)
G[s][e][k]['baa'] = 'boo'
break
# test with metrics
N = networks.NetworkLayerFromNX(G, distances=[500])
N.node_centrality(measures=['node_harmonic'])
metrics_dict = N.metrics_to_dict()
G_round_trip = N.to_networkX()
for n, d in G.nodes(data=True):
assert G_round_trip.nodes[n]['x'] == d['x']
assert G_round_trip.nodes[n]['y'] == d['y']
assert G_round_trip.nodes[n]['live'] == d['live']
for s, e, k, d in G.edges(keys=True, data=True):
assert G_round_trip[s][e][k]['geom'] == d['geom']
assert G_round_trip[s][e][k]['imp_factor'] == d['imp_factor']
# check that metrics came through
for uid, metrics in metrics_dict.items():
assert G_round_trip.nodes[uid]['metrics'] == metrics
# check data persistence
assert G_round_trip.nodes[baa_node]['boo'] == 'baa'
assert G_round_trip[boo_edge[0]][boo_edge[1]][0]['baa'] == 'boo'
def test_find_nearest(primal_graph):
N = networks.NetworkLayerFromNX(primal_graph, distances=[100])
# generate some data
data_dict = mock.mock_data_dict(primal_graph)
D = layers.DataLayerFromDict(data_dict)
# test the filter - iterating each point in data map
for d in D._data:
d_x = d[0]
d_y = d[1]
# find the closest point on the network
min_idx, min_dist = data.find_nearest(d_x, d_y, N.node_x_arr, N.node_y_arr, max_dist=500)
# check that no other indices are nearer
for i, n in enumerate(N._node_data):
n_x = n[0]
n_y = n[1]
dist = np.sqrt((d_x - n_x) ** 2 + (d_y - n_y) ** 2)
if i == min_idx:
assert round(dist, 8) == round(min_dist, 8)
else:
assert dist > min_dist
def test_compute_centrality(primal_graph):
"""
Underlying methods also tested via test_networks.test_network_centralities
"""
betas = np.array([0.01, 0.005])
distances = networks.distance_from_beta(betas)
# generate data structures
N = networks.NetworkLayerFromNX(primal_graph, distances=distances)
node_data = N._node_data
edge_data = N._edge_data
node_edge_map = N._node_edge_map
# CHECK NODE BASED
node_measures = ['node_density',
'node_farness',
'node_cycles',
'node_harmonic',
'node_beta',
'node_betweenness',
'node_betweenness_beta']
node_measures_ang = ['node_harmonic_angular',
'node_betweenness_angular']
# check measures against underlying method
N = networks.NetworkLayerFromNX(primal_graph, distances=distances)
N.node_centrality(measures=['node_density'])
# test against underlying method
measures_data = centrality.local_node_centrality(node_data,
edge_data,
node_edge_map,
distances,
betas,
measure_keys=('node_density',))
for d_idx, d_key in enumerate(distances):
assert np.allclose(N.metrics['centrality']['node_density'][d_key], measures_data[0][d_idx])
# also check the number of returned types for a few assortments of metrics
np.random.shuffle(node_measures) # in place
# not necessary to do all labels, first few should do
for min_idx in range(3):
measure_keys = np.array(node_measures[min_idx:])
N = networks.NetworkLayerFromNX(primal_graph, distances=distances)
N.node_centrality(measures=node_measures)
# test against underlying method
measures_data = centrality.local_node_centrality(node_data,
edge_data,
node_edge_map,
distances,
betas,
measure_keys=tuple(measure_keys))
for m_idx, measure_name in enumerate(measure_keys):
for d_idx, d_key in enumerate(distances):
assert np.allclose(N.metrics['centrality'][measure_name][d_key],
measures_data[m_idx][d_idx], atol=0.001, rtol=0)
# check that angular gets passed through
N_ang = networks.NetworkLayerFromNX(primal_graph, distances=[2000])
N_ang.node_centrality(measures=['node_harmonic_angular'],
angular=True)
N = networks.NetworkLayerFromNX(primal_graph, distances=[2000])
N.node_centrality(measures=['node_harmonic'],
angular=False)
assert not np.allclose(N_ang.metrics['centrality']['node_harmonic_angular'][2000],
N.metrics['centrality']['node_harmonic'][2000], atol=0.001, rtol=0)
assert not np.allclose(N_ang.metrics['centrality']['node_harmonic_angular'][2000],
N.metrics['centrality']['node_harmonic'][2000], atol=0.001, rtol=0)
# check that typos, duplicates, and mixed angular / non-angular are caught
with pytest.raises(ValueError):
N.node_centrality(measures=['spelling_typo'])
with pytest.raises(ValueError):
N.node_centrality(measures=['node_density', 'node_density'])
with pytest.raises(ValueError):
N.node_centrality(measures=['node_density', 'node_harmonic_angular'])
# CHECK SEGMENTISED
segment_measures = ['segment_density',
'segment_harmonic',
'segment_beta',
'segment_betweenness']
segment_measures_ang = ['segment_harmonic_hybrid',
'segment_betweeness_hybrid']
# check measures against underlying method
N = networks.NetworkLayerFromNX(primal_graph, distances=distances)
N.segment_centrality(measures=['segment_density'])
# test against underlying method
measures_data = centrality.local_segment_centrality(node_data,
edge_data,
node_edge_map,
distances,
betas,
measure_keys=('segment_density',))
for d_idx, d_key in enumerate(distances):
assert np.allclose(N.metrics['centrality']['segment_density'][d_key], measures_data[0][d_idx])
# also check the number of returned types for a few assortments of metrics
np.random.shuffle(segment_measures) # in place
# not necessary to do all labels, first few should do
for min_idx in range(3):
measure_keys = np.array(segment_measures[min_idx:])
N = networks.NetworkLayerFromNX(primal_graph,
distances=distances)
N.segment_centrality(measures=segment_measures)
# test against underlying method
measures_data = centrality.local_segment_centrality(node_data,
edge_data,
node_edge_map,
distances,
betas,
measure_keys=tuple(measure_keys))
for m_idx, measure_name in enumerate(measure_keys):
for d_idx, d_key in enumerate(distances):
assert np.allclose(N.metrics['centrality'][measure_name][d_key],
measures_data[m_idx][d_idx], atol=0.001, rtol=0)
# check that angular gets passed through
N_ang = networks.NetworkLayerFromNX(primal_graph, distances=[2000])
N_ang.segment_centrality(measures=['segment_harmonic_hybrid'],
angular=True)
N = networks.NetworkLayerFromNX(primal_graph, distances=[2000])
N.segment_centrality(measures=['segment_harmonic'],
angular=False)
assert not np.allclose(N_ang.metrics['centrality']['segment_harmonic_hybrid'][2000],
N.metrics['centrality']['segment_harmonic'][2000], atol=0.001, rtol=0)
assert not np.allclose(N_ang.metrics['centrality']['segment_harmonic_hybrid'][2000],
N.metrics['centrality']['segment_harmonic'][2000], atol=0.001, rtol=0)
# check that typos, duplicates, and mixed angular / non-angular are caught
with pytest.raises(ValueError):
N.segment_centrality(measures=['spelling_typo'])
with pytest.raises(ValueError):
N.segment_centrality(measures=['segment_density', 'segment_density'])
with pytest.raises(ValueError):
N.segment_centrality(measures=['segment_density', 'segment_harmonic_hybrid'])
# check that the deprecated method raises:
with pytest.raises(DeprecationWarning):
N.compute_centrality()
async def accessibility_calc(db_config,
nodes_table,
links_table,
city_pop_id,
distances,
boundary_table='analysis.city_boundaries_150',
data_table='os.poi',
data_where=None,
rdm_flag=False,
dual_flag=False):
if dual_flag or rdm_flag:
if city_pop_id > 1:
logger.warning(
'Only do dual or randomised metrics for city_pop_id = 1')
return
if dual_flag:
nodes_table += '_dual'
links_table += '_dual'
if rdm_flag:
data_table += '_randomised'
logger.info(
f'Starting LU calcs for city id: {city_pop_id} on network table '
f'{nodes_table} and data table {data_table}')
logger.info(f'Loading network data')
G = await postGIS_to_networkX(db_config, nodes_table, links_table,
city_pop_id)
N = networks.NetworkLayerFromNX(G, distances)
logger.info(f'Loading POI data from data table: {data_table}')
data_dict = await postGIS_to_landuses_dict(db_config,
data_table,
'urn',
'class_code',
boundary_table,
city_pop_id,
max_dist=max(distances),
data_where=data_where)
data_uids, data_map = layers.data_map_from_dict(data_dict)
# derive the landuse labels, classes, encodings
landuse_labels = [v['class'] for v in data_dict.values()]
landuse_classes, landuse_encodings = layers.encode_categorical(
landuse_labels)
logger.info(f'Generating disparity weights matrix')
cl_disparity_wt_matrix = disparity_wt_matrix(landuse_classes)
logger.info('Creating data layer')
D = layers.DataLayer(data_uids, data_map)
start = time.localtime()
logger.info('Assigning data points to the network')
D.assign_to_network(N, max_dist=400)
# generate the accessibility codes Class
# this deduces codes and squashes results into categories
logger.info('Generating POI accessibility codes')
Acc_codes = Accessibility_Codes(landuse_classes,
len(N.uids),
distances,
compact=(dual_flag or rdm_flag))
mixed_use_metrics = [
'hill', 'hill_branch_wt', 'hill_pairwise_wt',
'hill_pairwise_disparity', 'shannon', 'gini_simpson',
'raos_pairwise_disparity'
]
# if dual or rdm only do first two
if dual_flag or rdm_flag:
mixed_use_metrics = mixed_use_metrics[:2]
cl_disparity_wt_matrix = None
# compute
logger.info('Computing landuses')
D.compute_aggregated(landuse_labels=landuse_labels,
mixed_use_keys=mixed_use_metrics,
accessibility_keys=Acc_codes.all_codes,
cl_disparity_wt_matrix=cl_disparity_wt_matrix,
qs=[0, 1, 2])
time_duration = datetime.timedelta(seconds=time.mktime(time.localtime()) -
time.mktime(start))
logger.info(f'Algo duration: {time_duration}')
# squash the accessibility data
logger.info('Squashing accessibility data')
Acc_codes.set_metrics(N.metrics['accessibility'])
mu_q_keys = [
'hill', 'hill_branch_wt', 'hill_pairwise_wt', 'hill_pairwise_disparity'
]
if dual_flag or rdm_flag:
mu_q_keys = mu_q_keys[:2]
mu_keys = ['shannon', 'gini_simpson', 'raos_pairwise_disparity']
if dual_flag or rdm_flag:
mu_keys = []
if not dual_flag and not rdm_flag:
ac_keys = [
'accommodation', 'eating', 'drinking', 'commercial', 'tourism',
'entertainment', 'government', 'manufacturing', 'retail_food',
'retail_other', 'transport', 'health', 'education', 'parks',
'cultural', 'sports', 'total'
]
else:
ac_keys = [
'eating', 'drinking', 'commercial', 'retail_food', 'retail_other',
'transport', 'total'
]
# aggregate the data
logger.info('Aggregating results')
bulk_data = []
for idx, uid in enumerate(N.uids):
# first check that this is a live node (i.e. within the original city boundary)
if not N.live[idx]:
continue
node_data = [uid]
# mixed-use keys requiring q values
for mu_key in mu_q_keys:
for q_key, q_val in N.metrics['mixed_uses'][mu_key].items():
inner_data = []
for d_key, d_val in q_val.items():
inner_data.append(d_val[idx])
node_data.append(inner_data)
# mixed-use keys not requiring q values
for mu_key in mu_keys:
inner_data = []
for d_key, d_val in N.metrics['mixed_uses'][mu_key].items():
inner_data.append(d_val[idx])
node_data.append(inner_data)
# accessibility keys
for ac_key in ac_keys:
inner_data = []
for d_key, d_val in Acc_codes.metrics['weighted'][ac_key].items():
inner_data.append(d_val[idx])
node_data.append(inner_data)
# also write non-weighted variants of the following
if ac_key in [
'eating', 'commercial', 'retail_food', 'retail_other',
'total'
]:
inner_data = []
for d_key, d_val in Acc_codes.metrics['non_weighted'][
ac_key].items():
inner_data.append(d_val[idx])
node_data.append(inner_data)
bulk_data.append(tuple(node_data))
logger.info('Writing results to database')
db_con = await asyncpg.connect(**db_config)
if not dual_flag and not rdm_flag:
measure_cols = [
'mu_hill_0', 'mu_hill_1', 'mu_hill_2', 'mu_hill_branch_wt_0',
'mu_hill_branch_wt_1', 'mu_hill_branch_wt_2',
'mu_hill_pairwise_wt_0', 'mu_hill_pairwise_wt_1',
'mu_hill_pairwise_wt_2', 'mu_hill_dispar_wt_0',
'mu_hill_dispar_wt_1', 'mu_hill_dispar_wt_2', 'mu_shannon',
'mu_gini', 'mu_raos', 'ac_accommodation', 'ac_eating',
'ac_eating_nw', 'ac_drinking', 'ac_commercial', 'ac_commercial_nw',
'ac_tourism', 'ac_entertainment', 'ac_government',
'ac_manufacturing', 'ac_retail_food', 'ac_retail_food_nw',
'ac_retail_other', 'ac_retail_other_nw', 'ac_transport',
'ac_health', 'ac_education', 'ac_parks', 'ac_cultural',
'ac_sports', 'ac_total', 'ac_total_nw'
]
else:
measure_cols = [
'mu_hill_0', 'mu_hill_1', 'mu_hill_2', 'mu_hill_branch_wt_0',
'mu_hill_branch_wt_1', 'mu_hill_branch_wt_2', 'ac_eating',
'ac_eating_nw', 'ac_drinking', 'ac_commercial', 'ac_commercial_nw',
'ac_retail_food', 'ac_retail_food_nw', 'ac_retail_other',
'ac_retail_other_nw', 'ac_transport', 'ac_total', 'ac_total_nw'
]
# add the _rdm extension if necessary
if rdm_flag:
measure_cols = [m + '_rdm' for m in measure_cols]
# create the columns
col_strings = []
counter = 2
for measure_col in measure_cols:
await db_con.execute(f'''
ALTER TABLE {nodes_table}
ADD COLUMN IF NOT EXISTS {measure_col} real[];
''')
col_strings.append(f'{measure_col} = ${counter}')
counter += 1
await db_con.executemany(
f'UPDATE {nodes_table} SET ' + ', '.join(col_strings) +
' WHERE id = $1;', bulk_data)
await db_con.close()
async def centrality_shortest(db_config, nodes_table, links_table, city_pop_id, distances):
logger.info(f'Loading graph for city: {city_pop_id} derived from table: {nodes_table}')
G = await postGIS_to_networkX(db_config, nodes_table, links_table, city_pop_id)
if len(G) == 0:
return
logger.info(f'Generating node map and edge map')
N = networks.NetworkLayerFromNX(G, distances=distances)
logger.info('Calculating shortest-path node centralities')
start = time.localtime()
node_measures = [
'node_density',
'node_farness',
'node_cycles',
'node_harmonic',
'node_beta',
'node_betweenness',
'node_betweenness_beta'
]
N.node_centrality(measures=node_measures)
time_duration = datetime.timedelta(
seconds=time.mktime(time.localtime()) - time.mktime(start))
logger.info(f'Algo duration: {time_duration}')
logger.info('Calculating shortest-path segment centralities')
start = time.localtime()
segment_measures = [
'segment_density',
'segment_harmonic',
'segment_beta',
'segment_betweenness'
]
N.segment_centrality(measures=segment_measures)
time_duration = datetime.timedelta(
seconds=time.mktime(time.localtime()) - time.mktime(start))
logger.info(f'Algo duration: {time_duration}')
logger.info('Calculating simplest-path node centralities')
start = time.localtime()
angular_node_measures = [
'node_harmonic_angular',
'node_betweenness_angular'
]
N.node_centrality(measures=angular_node_measures, angular=True)
time_duration = datetime.timedelta(
seconds=time.mktime(time.localtime()) - time.mktime(start))
logger.info(f'Algo duration: {time_duration}')
logger.info('Calculating simplest-path segment centralities')
start = time.localtime()
angular_segment_measures = [
'segment_harmonic_hybrid',
'segment_betweeness_hybrid'
]
N.segment_centrality(measures=angular_segment_measures, angular=True)
time_duration = datetime.timedelta(
seconds=time.mktime(time.localtime()) - time.mktime(start))
logger.info(f'Algo duration: {time_duration}')
# Quite slow writing to database so do all distances at once
logger.info('Prepping data for database')
metrics = N.metrics_to_dict()
bulk_data = []
#
comb_measures = node_measures + segment_measures
com_ang_measures = angular_node_measures + angular_segment_measures
for k, v in metrics.items():
# first check that this is a live node (i.e. within the original city boundary)
if not v['live']:
continue
# start node data list - initialise with node label
node_data = [k]
# pack shortest path data
for measure in comb_measures:
inner_data = []
for d in distances:
inner_data.append(v['centrality'][measure][d])
node_data.append(inner_data)
# pack simplest path data
for ang_measure in com_ang_measures:
inner_ang_data = []
for d in distances:
inner_ang_data.append(v['centrality'][ang_measure][d])
node_data.append(inner_ang_data)
bulk_data.append(node_data)
logger.info('Writing data back to database')
db_con = await asyncpg.connect(**db_config)
# check that the columns exist
# do this separately to control the order in which the columns are added (by theme instead of distance)
for measure in comb_measures:
# prepend with "c_"
c_measure = f'c_{measure}'
await db_con.execute(f'''
ALTER TABLE {nodes_table}
ADD COLUMN IF NOT EXISTS {c_measure} real[];
''')
for ang_measure in com_ang_measures:
c_ang_measure = f'c_{ang_measure}'
await db_con.execute(f'''
ALTER TABLE {nodes_table}
ADD COLUMN IF NOT EXISTS {c_ang_measure} real[];
''')
await db_con.executemany(f'''
UPDATE {nodes_table}
SET
c_node_density = $2,
c_node_farness = $3,
c_node_cycles = $4,
c_node_harmonic = $5,
c_node_beta = $6,
c_node_betweenness = $7,
c_node_betweenness_beta = $8,
c_segment_density = $9,
c_segment_harmonic = $10,
c_segment_beta = $11,
c_segment_betweenness = $12,
c_node_harmonic_angular = $13,
c_node_betweenness_angular = $14,
c_segment_harmonic_hybrid = $15,
c_segment_betweeness_hybrid = $16
WHERE id = $1
''', bulk_data)
await db_con.close()
from cityseer.metrics import networks, layers
from cityseer.tools import mock, graphs, plot
base_path = os.getcwd()
plt.style.use('matplotlibrc')
###
# INTRO PLOT
G = mock.mock_graph()
plot.plot_nX(G, labels=True, node_size=80, path='images/graph.png', dpi=150)
# INTRO EXAMPLE PLOTS
G = graphs.nX_simple_geoms(G)
G = graphs.nX_decompose(G, 20)
N = networks.NetworkLayerFromNX(G, distances=[400, 800])
N.segment_centrality(measures=['segment_harmonic'])
data_dict = mock.mock_data_dict(G, random_seed=25)
D = layers.DataLayerFromDict(data_dict)
D.assign_to_network(N, max_dist=400)
landuse_labels = mock.mock_categorical_data(len(data_dict), random_seed=25)
D.hill_branch_wt_diversity(landuse_labels, qs=[0])
G_metrics = N.to_networkX()
segment_harmonic_vals = []
mixed_uses_vals = []
for node, data in G_metrics.nodes(data=True):
segment_harmonic_vals.append(
data['metrics']['centrality']['segment_harmonic'][800])
mixed_uses_vals.append(
def test_nX_from_graph_maps(primal_graph):
# also see test_networks.test_to_networkX for tests on implementation via Network layer
# check round trip to and from graph maps results in same graph
# explicitly set live params for equality checks
# graph_maps_from_networkX generates these implicitly if missing
for n in primal_graph.nodes():
primal_graph.nodes[n]['live'] = bool(np.random.randint(0, 1))
# test directly from and to graph maps
node_uids, node_data, edge_data, node_edge_map = graphs.graph_maps_from_nX(primal_graph)
G_round_trip = graphs.nX_from_graph_maps(node_uids, node_data, edge_data, node_edge_map)
assert list(G_round_trip.nodes) == list(primal_graph.nodes)
assert list(G_round_trip.edges) == list(primal_graph.edges)
# check with metrics dictionary
N = networks.NetworkLayerFromNX(primal_graph, distances=[500, 1000])
N.node_centrality(measures=['node_harmonic'])
data_dict = mock.mock_data_dict(primal_graph)
landuse_labels = mock.mock_categorical_data(len(data_dict))
D = layers.DataLayerFromDict(data_dict)
D.assign_to_network(N, max_dist=400)
D.compute_landuses(landuse_labels,
mixed_use_keys=['hill', 'shannon'],
accessibility_keys=['a', 'c'],
qs=[0, 1])
metrics_dict = N.metrics_to_dict()
# without backbone
G_round_trip_data = graphs.nX_from_graph_maps(node_uids,
node_data,
edge_data,
node_edge_map,
metrics_dict=metrics_dict)
for uid, metrics in metrics_dict.items():
assert G_round_trip_data.nodes[uid]['metrics'] == metrics
# with backbone
G_round_trip_data = graphs.nX_from_graph_maps(node_uids,
node_data,
edge_data,
node_edge_map,
networkX_multigraph=primal_graph,
metrics_dict=metrics_dict)
for uid, metrics in metrics_dict.items():
assert G_round_trip_data.nodes[uid]['metrics'] == metrics
# test with decomposed
G_decomposed = graphs.nX_decompose(primal_graph, decompose_max=20)
# set live explicitly
for n in G_decomposed.nodes():
G_decomposed.nodes[n]['live'] = bool(np.random.randint(0, 1))
node_uids_d, node_data_d, edge_data_d, node_edge_map_d = graphs.graph_maps_from_nX(G_decomposed)
G_round_trip_d = graphs.nX_from_graph_maps(node_uids_d, node_data_d, edge_data_d, node_edge_map_d)
assert list(G_round_trip_d.nodes) == list(G_decomposed.nodes)
for n, iter_node_data in G_round_trip.nodes(data=True):
assert n in G_decomposed
assert iter_node_data['live'] == G_decomposed.nodes[n]['live']
assert iter_node_data['x'] == G_decomposed.nodes[n]['x']
assert iter_node_data['y'] == G_decomposed.nodes[n]['y']
assert G_round_trip_d.edges == G_decomposed.edges
# error checks for when using backbone graph:
# mismatching numbers of nodes
corrupt_G = primal_graph.copy()
corrupt_G.remove_node(0)
with pytest.raises(ValueError):
graphs.nX_from_graph_maps(node_uids,
node_data,
edge_data,
node_edge_map,
networkX_multigraph=corrupt_G)
# mismatching node uid
with pytest.raises(KeyError):
corrupt_node_uids = list(node_uids)
corrupt_node_uids[0] = 'boo'
graphs.nX_from_graph_maps(corrupt_node_uids,
node_data,
edge_data,
node_edge_map,
networkX_multigraph=primal_graph)
# missing edge
with pytest.raises(KeyError):
corrupt_primal_graph = primal_graph.copy()
corrupt_primal_graph.remove_edge(0, 1)
graphs.nX_from_graph_maps(node_uids,
node_data,
edge_data,
node_edge_map,
networkX_multigraph=corrupt_primal_graph)
async def centrality_dual(db_config, nodes_table, links_table, city_pop_id,
distances):
logger.info(f'Loading graph for city: {city_pop_id} '
f'derived from table: {nodes_table}')
G = await postGIS_to_networkX(db_config, nodes_table, links_table,
city_pop_id)
if len(G) == 0:
return
logger.info('Casting to dual')
G = graphs.nX_to_dual(G) # convert to dual
logger.info(f'Generating node map and edge map')
N = networks.NetworkLayerFromNX(G, distances=distances)
# the round trip graph is needed for the generated lengths, angles, etc.
logger.info('Making round-trip graph')
G_round_trip = N.to_networkX()
db_con = await asyncpg.connect(**db_config)
# ADD DUAL GRAPH'S VERTICES TABLE
logger.info('Preparing dual nodes table')
nodes_table_name_only = nodes_table
if '.' in nodes_table_name_only:
nodes_table_name_only = nodes_table_name_only.split('.')[-1]
await db_con.execute(f'''
-- add dual nodes table
CREATE TABLE IF NOT EXISTS {nodes_table}_dual (
id text PRIMARY KEY,
city_pop_id int,
within bool,
geom geometry(Point, 27700)
);
CREATE INDEX IF NOT EXISTS geom_idx_{nodes_table_name_only}_dual
ON {nodes_table}_dual USING GIST (geom);
CREATE INDEX IF NOT EXISTS city_pop_idx_{nodes_table_name_only}_dual
ON {nodes_table}_dual (city_pop_id);
''')
logger.info('Preparing dual nodes data')
dual_nodes_data = []
for n, d in G_round_trip.nodes(data=True):
dual_nodes_data.append([
n,
city_pop_id,
d['live'], # within
d['x'],
d['y']
])
logger.info('Writing dual nodes to DB')
await db_con.executemany(
f'''
INSERT INTO {nodes_table}_dual (id, city_pop_id, within, geom)
VALUES ($1, $2, $3, ST_SetSRID(ST_MakePoint($4, $5), 27700))
ON CONFLICT DO NOTHING;
''', dual_nodes_data)
logger.info('Preparing dual edges table')
links_table_name_only = links_table
if '.' in links_table_name_only:
links_table_name_only = links_table_name_only.split('.')[-1]
await db_con.execute(f'''
-- add dual links table
CREATE TABLE IF NOT EXISTS {links_table}_dual (
id text PRIMARY KEY,
parent_id text,
city_pop_id int,
node_a text,
node_b text,
distance real,
angle real,
impedance_factor real,
geom geometry(Linestring, 27700)
);
CREATE INDEX IF NOT EXISTS city_pop_idx_{links_table_name_only}_dual
ON {links_table}_dual (city_pop_id);
CREATE INDEX IF NOT EXISTS geom_idx_{links_table_name_only}_dual
ON {links_table}_dual USING GIST (geom);
''')
# prepare the dual edges and nodes tables
logger.info('Preparing data for dual edges table')
dual_edge_data = []
parent_primal_counter = {}
for s, e, d in G_round_trip.edges(data=True):
# number each of the new dual edges sequentially
# based on the original parent primal node
primal_parent = d['parent_primal_node']
if primal_parent not in parent_primal_counter:
parent_primal_counter[primal_parent] = 1
else:
parent_primal_counter[primal_parent] += 1
label = f'{primal_parent}_{parent_primal_counter[primal_parent]}'
# add the data tuple
dual_edge_data.append(
(label, primal_parent, city_pop_id, s, e, d['length'],
d['angle_sum'], d['imp_factor'], d['geom'].wkb_hex))
logger.info('Writing dual edges to DB')
await db_con.executemany(
f'''
INSERT INTO {links_table}_dual (
id,
parent_id,
city_pop_id,
node_a,
node_b,
distance,
angle,
impedance_factor,
geom)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, ST_SetSRID($9::geometry, 27700))
ON CONFLICT DO NOTHING;
''', dual_edge_data)
await db_con.close()
logger.info('Calculating centrality paths and centralities '
'for centrality-path heuristics')
start = time.localtime()
measures = [
'node_density', 'node_harmonic', 'node_beta', 'node_betweenness',
'node_betweenness_beta'
]
N.node_centrality(measures=measures)
time_duration = datetime.timedelta(seconds=time.mktime(time.localtime()) -
time.mktime(start))
logger.info(f'Algo duration: {time_duration}')
logger.info('Calculating centrality paths and centralities '
'for simplest-path heuristics')
start = time.localtime()
angular_measures = ['node_harmonic_angular', 'node_betweenness_angular']
N.node_centrality(measures=angular_measures, angular=True)
time_duration = datetime.timedelta(seconds=time.mktime(time.localtime()) -
time.mktime(start))
logger.info(f'Algo duration: {time_duration}')
db_con = await asyncpg.connect(**db_config)
# check that the columns exist
# do this separately to control the order in which the columns are added (by theme instead of distance)
for measure in measures:
# prepend with "c_"
c_measure = f'c_{measure}'
await db_con.execute(f'''
ALTER TABLE {nodes_table}_dual
ADD COLUMN IF NOT EXISTS {c_measure} real[];
''')
for ang_measure in angular_measures:
c_ang_measure = f'c_{ang_measure}'
await db_con.execute(f'''
ALTER TABLE {nodes_table}_dual
ADD COLUMN IF NOT EXISTS {c_ang_measure} real[];
''')
# Quite slow writing to database so do all distances at once
logger.info('Prepping data for database')
metrics = N.metrics_to_dict()
bulk_data = []
for k, v in metrics.items():
# first check that this is a live node
# (i.e. within the original city boundary)
if not v['live']:
continue
# start node data list - initialise with node label
node_data = [k]
# pack centrality path data
for measure in measures:
inner_data = []
for d in distances:
inner_data.append(v['centrality'][measure][d])
node_data.append(inner_data)
# pack simplest path data
for ang_measure in angular_measures:
inner_ang_data = []
for d in distances:
inner_ang_data.append(v['centrality'][ang_measure][d])
node_data.append(inner_ang_data)
bulk_data.append(node_data)
logger.info('Writing data back to database')
await db_con.executemany(
f'''
UPDATE {nodes_table}_dual
SET
c_node_density = $2,
c_node_harmonic = $3,
c_node_beta = $4,
c_node_betweenness = $5,
c_node_betweenness_beta = $6,
c_node_harmonic_angular = $7,
c_node_betweenness_angular = $8
WHERE id = $1
''', bulk_data)
await db_con.close()
def test_compute_stats(primal_graph):
"""
Test stats component
"""
betas = np.array([0.01, 0.005])
distances = networks.distance_from_beta(betas)
# network layer
N_single = networks.NetworkLayerFromNX(primal_graph, distances=distances)
N_multi = networks.NetworkLayerFromNX(primal_graph, distances=distances)
node_map = N_multi._node_data
edge_map = N_multi._edge_data
node_edge_map = N_multi._node_edge_map
# data layer
data_dict = mock.mock_data_dict(primal_graph)
D_single = layers.DataLayerFromDict(data_dict)
D_multi = layers.DataLayerFromDict(data_dict)
# check single metrics independently against underlying for some use-cases, e.g. hill, non-hill, accessibility...
D_single.assign_to_network(N_single, max_dist=500)
D_multi.assign_to_network(N_multi, max_dist=500)
# generate some mock landuse data
mock_numeric = mock.mock_numerical_data(len(data_dict), num_arrs=2)
# generate stats
D_single.compute_stats(stats_keys='boo', stats_data_arrs=mock_numeric[0])
D_single.compute_stats(stats_keys='baa', stats_data_arrs=mock_numeric[1])
D_multi.compute_stats(stats_keys=['boo', 'baa'],
stats_data_arrs=mock_numeric)
# test against underlying method
data_map = D_single._data
stats_sum, stats_sum_wt, stats_mean, stats_mean_wt, stats_variance, stats_variance_wt, stats_max, stats_min = \
data.aggregate_stats(node_map,
edge_map,
node_edge_map,
data_map,
distances,
betas,
numerical_arrays=mock_numeric)
stats_keys = [
'max', 'min', 'sum', 'sum_weighted', 'mean', 'mean_weighted',
'variance', 'variance_weighted'
]
stats_data = [
stats_max, stats_min, stats_sum, stats_sum_wt, stats_mean,
stats_mean_wt, stats_variance, stats_variance_wt
]
for num_idx, num_label in enumerate(['boo', 'baa']):
for s_key, stats in zip(stats_keys, stats_data):
for d_idx, d_key in enumerate(distances):
# check one-at-a-time computed vs multiply computed
assert np.allclose(
N_single.metrics['stats'][num_label][s_key][d_key],
N_multi.metrics['stats'][num_label][s_key][d_key],
atol=0.001,
rtol=0,
equal_nan=True)
# check one-at-a-time against manual
assert np.allclose(
N_single.metrics['stats'][num_label][s_key][d_key],
stats[num_idx][d_idx],
atol=0.001,
rtol=0,
equal_nan=True)
# check multiply computed against manual
assert np.allclose(
N_multi.metrics['stats'][num_label][s_key][d_key],
stats[num_idx][d_idx],
atol=0.001,
rtol=0,
equal_nan=True)
# check that problematic keys and data arrays are caught
for labels, arrs, err in (
(['a'], mock_numeric, ValueError), # mismatching lengths
(['a', 'b'], None, TypeError), # missing arrays
(['a', 'b'], [], ValueError), # missing arrays
(None, mock_numeric, TypeError), # missing labels
([], mock_numeric, ValueError)): # missing labels
with pytest.raises(err):
D_multi.compute_stats(stats_keys=labels, stats_data_arrs=arrs)
def test_compute_landuses(primal_graph):
betas = np.array([0.01, 0.005])
distances = networks.distance_from_beta(betas)
# network layer
N = networks.NetworkLayerFromNX(primal_graph, distances=distances)
node_map = N._node_data
edge_map = N._edge_data
node_edge_map = N._node_edge_map
# data layer
data_dict = mock.mock_data_dict(primal_graph)
qs = np.array([0, 1, 2])
D = layers.DataLayerFromDict(data_dict)
# check single metrics independently against underlying for some use-cases, e.g. hill, non-hill, accessibility...
D.assign_to_network(N, max_dist=500)
# generate some mock landuse data
landuse_labels = mock.mock_categorical_data(len(data_dict))
landuse_classes, landuse_encodings = layers.encode_categorical(
landuse_labels)
# compute hill mixed uses
D.compute_landuses(landuse_labels,
mixed_use_keys=['hill_branch_wt'],
qs=qs)
# test against underlying method
data_map = D._data
mu_data_hill, mu_data_other, ac_data, ac_data_wt = data.aggregate_landuses(
node_map,
edge_map,
node_edge_map,
data_map,
distances,
betas,
landuse_encodings,
qs=qs,
mixed_use_hill_keys=np.array([1]))
for q_idx, q_key in enumerate(qs):
for d_idx, d_key in enumerate(distances):
assert np.allclose(
N.metrics['mixed_uses']['hill_branch_wt'][q_key][d_key],
mu_data_hill[0][q_idx][d_idx],
atol=0.001,
rtol=0)
# gini simpson
D.compute_landuses(landuse_labels, mixed_use_keys=['gini_simpson'])
# test against underlying method
data_map = D._data
mu_data_hill, mu_data_other, ac_data, ac_data_wt = data.aggregate_landuses(
node_map,
edge_map,
node_edge_map,
data_map,
distances,
betas,
landuse_encodings,
mixed_use_other_keys=np.array([1]))
for d_idx, d_key in enumerate(distances):
assert np.allclose(N.metrics['mixed_uses']['gini_simpson'][d_key],
mu_data_other[0][d_idx],
atol=0.001,
rtol=0)
# accessibilities
D.compute_landuses(landuse_labels, accessibility_keys=['c'])
# test against underlying method
data_map = D._data
mu_data_hill, mu_data_other, ac_data, ac_data_wt = data.aggregate_landuses(
node_map,
edge_map,
node_edge_map,
data_map,
distances,
betas,
landuse_encodings,
accessibility_keys=np.array([landuse_classes.index('c')]))
for d_idx, d_key in enumerate(distances):
assert np.allclose(
N.metrics['accessibility']['non_weighted']['c'][d_key],
ac_data[0][d_idx],
atol=0.001,
rtol=0)
assert np.allclose(N.metrics['accessibility']['weighted']['c'][d_key],
ac_data_wt[0][d_idx],
atol=0.001,
rtol=0)
# also check the number of returned types for a few assortments of metrics
mixed_uses_hill_types = np.array([
'hill', 'hill_branch_wt', 'hill_pairwise_wt', 'hill_pairwise_disparity'
])
mixed_use_other_types = np.array(
['shannon', 'gini_simpson', 'raos_pairwise_disparity'])
ac_codes = np.array(landuse_classes)
# mixed uses hill
mu_hill_random = np.arange(len(mixed_uses_hill_types))
np.random.shuffle(mu_hill_random)
# mixed uses other
mu_other_random = np.arange(len(mixed_use_other_types))
np.random.shuffle(mu_other_random)
# accessibility
ac_random = np.arange(len(landuse_classes))
np.random.shuffle(ac_random)
# mock disparity matrix
mock_disparity_wt_matrix = np.full(
(len(landuse_classes), len(landuse_classes)), 1)
# not necessary to do all labels, first few should do
for mu_h_min in range(3):
mu_h_keys = np.array(mu_hill_random[mu_h_min:])
for mu_o_min in range(3):
mu_o_keys = np.array(mu_other_random[mu_o_min:])
for ac_min in range(3):
ac_keys = np.array(ac_random[ac_min:])
# in the final case, set accessibility to a single code otherwise an error would be raised
if len(mu_h_keys) == 0 and len(mu_o_keys) == 0 and len(
ac_keys) == 0:
ac_keys = np.array([0])
# randomise order of keys and metrics
mu_h_metrics = mixed_uses_hill_types[mu_h_keys]
mu_o_metrics = mixed_use_other_types[mu_o_keys]
ac_metrics = ac_codes[ac_keys]
# prepare network and compute
N_temp = networks.NetworkLayerFromNX(primal_graph,
distances=distances)
D_temp = layers.DataLayerFromDict(data_dict)
D_temp.assign_to_network(N_temp, max_dist=500)
D_temp.compute_landuses(
landuse_labels,
mixed_use_keys=list(mu_h_metrics) + list(mu_o_metrics),
accessibility_keys=ac_metrics,
cl_disparity_wt_matrix=mock_disparity_wt_matrix,
qs=qs)
# test against underlying method
mu_data_hill, mu_data_other, ac_data, ac_data_wt = \
data.aggregate_landuses(node_map,
edge_map,
node_edge_map,
data_map,
distances,
betas,
landuse_encodings,
qs=qs,
mixed_use_hill_keys=mu_h_keys,
mixed_use_other_keys=mu_o_keys,
accessibility_keys=ac_keys,
cl_disparity_wt_matrix=mock_disparity_wt_matrix)
for mu_h_idx, mu_h_met in enumerate(mu_h_metrics):
for q_idx, q_key in enumerate(qs):
for d_idx, d_key in enumerate(distances):
assert np.allclose(
N_temp.metrics['mixed_uses'][mu_h_met][q_key]
[d_key],
mu_data_hill[mu_h_idx][q_idx][d_idx],
atol=0.001,
rtol=0)
for mu_o_idx, mu_o_met in enumerate(mu_o_metrics):
for d_idx, d_key in enumerate(distances):
assert np.allclose(
N_temp.metrics['mixed_uses'][mu_o_met][d_key],
mu_data_other[mu_o_idx][d_idx],
atol=0.001,
rtol=0)
for ac_idx, ac_met in enumerate(ac_metrics):
for d_idx, d_key in enumerate(distances):
assert np.allclose(N_temp.metrics['accessibility']
['non_weighted'][ac_met][d_key],
ac_data[ac_idx][d_idx],
atol=0.001,
rtol=0)
assert np.allclose(N_temp.metrics['accessibility']
['weighted'][ac_met][d_key],
ac_data_wt[ac_idx][d_idx],
atol=0.001,
rtol=0)
# most integrity checks happen in underlying method, though check here for mismatching labels length and typos
with pytest.raises(ValueError):
D.compute_landuses(landuse_labels[-1], mixed_use_keys=['shannon'])
with pytest.raises(ValueError):
D.compute_landuses(landuse_labels, mixed_use_keys=['spelling_typo'])
# don't check accessibility_labels for typos - because only warning is triggered (not all labels will be in all data)
# check that unassigned data layer flags
with pytest.raises(ValueError):
D_new = layers.DataLayerFromDict(data_dict)
D_new.compute_landuses(landuse_labels, mixed_use_keys=['shannon'])
