def test_compute_stats_single():
for G, distances, betas in network_generator():
data_dict = mock.mock_data_dict(G)
numeric_data = mock.mock_numerical_data(len(data_dict), num_arrs=1)
# easy version
N_easy = networks.Network_Layer_From_nX(G, distances)
D_easy = layers.Data_Layer_From_Dict(data_dict)
D_easy.assign_to_network(N_easy, max_dist=500)
D_easy.compute_stats_single('boo', numeric_data[0])
# custom version
N_full = networks.Network_Layer_From_nX(G, distances)
D_full = layers.Data_Layer_From_Dict(data_dict)
D_full.assign_to_network(N_full, max_dist=500)
D_full.compute_aggregated(stats_keys=['boo'],
stats_data_arrs=numeric_data)
# compare
for n_label in ['boo']:
for s_label in [
'max', 'min', 'mean', 'mean_weighted', 'variance',
'variance_weighted'
]:
for dist in distances:
assert np.allclose(
N_easy.metrics['stats'][n_label][s_label][dist],
N_full.metrics['stats'][n_label][s_label][dist],
equal_nan=True,
atol=0.001,
rtol=0)
# check that non-single dimension arrays are caught
with pytest.raises(ValueError):
D_easy.compute_stats_single('boo', numeric_data)
def test_compute_stats_multiple():
for G, distances, betas in network_generator():
data_dict = mock.mock_data_dict(G)
numeric_data = mock.mock_numerical_data(len(data_dict), num_arrs=2)
# easy version
N_easy = networks.Network_Layer_From_nX(G, distances)
D_easy = layers.Data_Layer_From_Dict(data_dict)
D_easy.assign_to_network(N_easy, max_dist=500)
D_easy.compute_stats_multiple(['boo', 'baa'], numeric_data)
# custom version
N_full = networks.Network_Layer_From_nX(G, distances)
D_full = layers.Data_Layer_From_Dict(data_dict)
D_full.assign_to_network(N_full, max_dist=500)
D_full.compute_aggregated(stats_keys=['boo', 'baa'],
stats_data_arrs=numeric_data)
# compare
for n_label in ['boo', 'baa']:
for s_label in [
'max', 'min', 'mean', 'mean_weighted', 'variance',
'variance_weighted'
]:
for dist in distances:
assert np.allclose(
N_easy.metrics['stats'][n_label][s_label][dist],
N_full.metrics['stats'][n_label][s_label][dist],
equal_nan=True,
atol=0.001,
rtol=0)
def test_metrics_to_dict():
G = mock.mock_graph()
G = graphs.nX_simple_geoms(G)
# create a network layer and run some metrics
N = networks.Network_Layer_From_nX(G, distances=[500, 1000])
# check with no metrics
metrics_dict = N.metrics_to_dict()
dict_check(metrics_dict, N)
# check with centrality metrics
N.compute_centrality(measures=['node_harmonic'])
metrics_dict = N.metrics_to_dict()
dict_check(metrics_dict, N)
# check with data metrics
data_dict = mock.mock_data_dict(G)
landuse_labels = mock.mock_categorical_data(len(data_dict))
numerical_data = mock.mock_numerical_data(len(data_dict))
# TODO:
'''
D = layers.Data_Layer_From_Dict(data_dict)
D.assign_to_network(N, max_dist=400)
D.compute_aggregated(landuse_labels,
mixed_use_keys=['hill', 'shannon'],
accessibility_keys=['a', 'c'],
qs=[0, 1],
stats_keys=['boo'],
stats_data_arrs=numerical_data)
'''
metrics_dict = N.metrics_to_dict()
dict_check(metrics_dict, N)
def test_check_data_map():
G = mock.mock_graph()
G = graphs.nX_simple_geoms(G)
N = networks.Network_Layer_From_nX(G, distances=[500])
data_dict = mock.mock_data_dict(G)
data_uids, data_map = layers.data_map_from_dict(data_dict)
# should throw error if not assigned
with pytest.raises(ValueError):
checks.check_data_map(data_map)
# should work if flag set to False
checks.check_data_map(data_map, check_assigned=False)
# assign then check that it runs as intended
data_map = data.assign_to_network(data_map,
N._node_data,
N._edge_data,
N._node_edge_map,
max_dist=400)
checks.check_data_map(data_map)
# catch zero length data arrays
empty_2d_arr = np.full((0, 4), np.nan)
with pytest.raises(ValueError):
checks.check_data_map(empty_2d_arr)
# catch invalid dimensionality
with pytest.raises(ValueError):
checks.check_data_map(data_map[:, :-1])
def mmm_layercake_diversity(_graph,
_layer_spec,
_n_iters=200,
_spans=200,
_bands=4,
_tiers=2,
random_seed=0):
np.random.seed(random_seed)
for k in [
'cap_step', 'dist_threshold', 'pop_threshold', 'spill_rate',
'explore_rate'
]:
if k not in _layer_spec:
raise AttributeError(f'Missing key {k}')
# generate the backbone Network Layer
# include 1 (local only) for directly counting items assigned to current node
# also include np.inf so that all node combinations are considered in spite of wrapping around back
distances = list({1, np.inf, _layer_spec['dist_threshold']})
Netw_Layer = networks.Network_Layer_From_nX(_graph, distances=distances)
# generate population data layer
Pop_Layer = generate_data_layer(_spans, 20, Netw_Layer, _randomised=False)
# population state and map
# for this experiment assignments are not changing
pop_state = np.full(len(Pop_Layer.uids), 1.0) # use floats!
# generate the landuse substrate
# keep to 1 location per node for visualisation sake
# randomisation is immaterial because all will be instanced as 0.0
Landuse_Layer = generate_data_layer(_spans,
1,
Netw_Layer,
_randomised=False)
# calculate neighbourhood density
# move into loop if using dynamic populations
Pop_Layer.compute_stats_single('pop_intensity', pop_state)
pop_intensity = np.copy(
Netw_Layer.metrics['stats']['pop_intensity']['sum'][1])
# iterate
cap_step = _layer_spec['cap_step']
dist_threshold = _layer_spec['dist_threshold']
pop_threshold = _layer_spec['pop_threshold']
spill_rate = _layer_spec['spill_rate']
explore_rate = _layer_spec['explore_rate']
innovation_rate = _layer_spec['innovation_rate']
landuse_state = compute_iter(_n_iters, Netw_Layer._nodes,
Netw_Layer._edges, Landuse_Layer._data,
np.array(Netw_Layer.distances),
np.array(Netw_Layer.betas), _tiers, _bands,
_spans,
Netw_Layer.distances.index(dist_threshold),
spill_rate, pop_threshold, cap_step,
explore_rate, innovation_rate, pop_intensity)
return landuse_state
def test_check_network_maps():
# network maps
G = mock.mock_graph()
G = graphs.nX_simple_geoms(G)
N = networks.Network_Layer_From_nX(G, distances=[500])
# from cityseer.util import plot
# plot.plot_networkX_primal_or_dual(primal=G)
# plot.plot_graph_maps(N.uids, N._node_data, N._edge_data)
# catch zero length node and edge arrays
empty_node_arr = np.full((0, 5), np.nan)
with pytest.raises(ValueError):
checks.check_network_maps(empty_node_arr, N._edge_data,
N._node_edge_map)
empty_edge_arr = np.full((0, 4), np.nan)
with pytest.raises(ValueError):
checks.check_network_maps(N._node_data, empty_edge_arr,
N._node_edge_map)
# check that malformed node and data maps throw errors
with pytest.raises(ValueError):
checks.check_network_maps(N._node_data[:, :-1], N._edge_data,
N._node_edge_map)
with pytest.raises(ValueError):
checks.check_network_maps(N._node_data, N._edge_data[:, :-1],
N._node_edge_map)
# catch problematic edge map values
for x in [np.nan, -1]:
# missing start node
corrupted_edges = N._edge_data.copy()
corrupted_edges[0, 0] = x
with pytest.raises(AssertionError):
checks.check_network_maps(N._node_data, corrupted_edges,
N._node_edge_map)
# missing end node
corrupted_edges = N._edge_data.copy()
corrupted_edges[0, 1] = x
with pytest.raises(KeyError):
checks.check_network_maps(N._node_data, corrupted_edges,
N._node_edge_map)
# invalid length
corrupted_edges = N._edge_data.copy()
corrupted_edges[0, 2] = x
with pytest.raises(ValueError):
checks.check_network_maps(N._node_data, corrupted_edges,
N._node_edge_map)
# invalid angle_sum
corrupted_edges = N._edge_data.copy()
corrupted_edges[0, 3] = x
with pytest.raises(ValueError):
checks.check_network_maps(N._node_data, corrupted_edges,
N._node_edge_map)
# invalid imp_factor
corrupted_edges = N._edge_data.copy()
corrupted_edges[0, 4] = x
with pytest.raises(ValueError):
checks.check_network_maps(N._node_data, corrupted_edges,
N._node_edge_map)
def test_compute_accessibilities():
for G, distances, betas in network_generator():
data_dict = mock.mock_data_dict(G)
landuse_labels = mock.mock_categorical_data(len(data_dict))
# easy version
N_easy = networks.Network_Layer_From_nX(G, distances)
D_easy = layers.Data_Layer_From_Dict(data_dict)
D_easy.assign_to_network(N_easy, max_dist=500)
D_easy.compute_accessibilities(landuse_labels, ['c'])
# custom version
N_full = networks.Network_Layer_From_nX(G, distances)
D_full = layers.Data_Layer_From_Dict(data_dict)
D_full.assign_to_network(N_full, max_dist=500)
D_full.compute_aggregated(landuse_labels, accessibility_keys=['c'])
# compare
for d in distances:
for wt in ['weighted', 'non_weighted']:
assert np.allclose(N_easy.metrics['accessibility'][wt]['c'][d],
N_full.metrics['accessibility'][wt]['c'][d],
atol=0.001,
rtol=0)
def test_hill_diversity():
for G, distances, betas in network_generator():
data_dict = mock.mock_data_dict(G)
landuse_labels = mock.mock_categorical_data(len(data_dict))
# easy version
N_easy = networks.Network_Layer_From_nX(G, distances)
D_easy = layers.Data_Layer_From_Dict(data_dict)
D_easy.assign_to_network(N_easy, max_dist=500)
D_easy.hill_diversity(landuse_labels, qs=[0, 1, 2])
# custom version
N_full = networks.Network_Layer_From_nX(G, distances)
D_full = layers.Data_Layer_From_Dict(data_dict)
D_full.assign_to_network(N_full, max_dist=500)
D_full.compute_aggregated(landuse_labels,
mixed_use_keys=['hill'],
qs=[0, 1, 2])
# compare
for d in distances:
for q in [0, 1, 2]:
assert np.allclose(N_easy.metrics['mixed_uses']['hill'][q][d],
N_full.metrics['mixed_uses']['hill'][q][d],
atol=0.001,
rtol=0)
def test_to_networkX():
# also see test_graphs.test_networkX_from_graph_maps for underlying graph maps version
# check round trip to and from graph maps results in same graph
G = mock.mock_graph()
G = graphs.nX_simple_geoms(G)
# explicitly set live and weight params for equality checks
# graph_maps_from_networkX generates these implicitly if missing
G = graphs.nX_decompose(G, decompose_max=20)
for n in G.nodes():
G.nodes[n]['live'] = bool(np.random.randint(0, 1))
for s, e in G.edges():
G[s][e]['imp_factor'] = np.random.randint(0, 2)
# add random data to check persistence at other end
baa_node = None
for n in G.nodes():
baa_node = n
G.nodes[n]['boo'] = 'baa'
break
boo_edge = None
for s, e in G.edges():
boo_edge = (s, e)
G[s][e]['baa'] = 'boo'
# test with metrics
N = networks.Network_Layer_From_nX(G, distances=[500])
N.compute_centrality(measures=['node_harmonic'])
metrics_dict = N.metrics_to_dict()
G_round_trip = N.to_networkX()
for n, d in G.nodes(data=True):
assert G_round_trip.nodes[n]['x'] == d['x']
assert G_round_trip.nodes[n]['y'] == d['y']
assert G_round_trip.nodes[n]['live'] == d['live']
for s, e, d in G.edges(data=True):
assert G_round_trip[s][e]['geom'] == d['geom']
assert G_round_trip[s][e]['imp_factor'] == d['imp_factor']
# check that metrics came through
for uid, metrics in metrics_dict.items():
assert G_round_trip.nodes[uid]['metrics'] == metrics
# check data persistence
assert G_round_trip.nodes[baa_node]['boo'] == 'baa'
assert G_round_trip[boo_edge[0]][boo_edge[1]]['baa'] == 'boo'
def test_Network_Layer_From_nX():
G = mock.mock_graph()
G = graphs.nX_simple_geoms(G)
node_uids, node_data, edge_data, node_edge_map = graphs.graph_maps_from_nX(
G)
x_arr = node_data[:, 0]
y_arr = node_data[:, 1]
betas = np.array([-0.04, -0.02])
distances = networks.distance_from_beta(betas)
# test Network_Layer_From_NetworkX's class
for d, b in zip([distances, None], [None, betas]):
for angular in [True, False]:
N = networks.Network_Layer_From_nX(G, distances=d, betas=b)
assert np.allclose(N.uids, node_uids, atol=0.001, rtol=0)
assert np.allclose(N._node_data, node_data, atol=0.001, rtol=0)
assert np.allclose(N._edge_data, edge_data, atol=0.001, rtol=0)
assert np.allclose(
N.distances, distances, atol=0.001,
rtol=0) # inferred automatically when only betas provided
assert np.allclose(
N.betas, betas, atol=0.001,
rtol=0) # inferred automatically when only distances provided
assert N.min_threshold_wt == checks.def_min_thresh_wt
assert np.allclose(N.x_arr, x_arr, atol=0.001, rtol=0)
assert np.allclose(N.y_arr, y_arr, atol=0.001, rtol=0)
assert np.allclose(N.live, node_data[:, 2], atol=0.001, rtol=0)
assert np.allclose(N.edge_lengths,
edge_data[:, 2],
atol=0.001,
rtol=0)
assert np.allclose(N.edge_angles,
edge_data[:, 3],
atol=0.001,
rtol=0)
assert np.allclose(N.edge_impedance_factor,
edge_data[:, 4],
atol=0.001,
rtol=0)
assert np.allclose(N.edge_in_bearing,
edge_data[:, 5],
atol=0.001,
rtol=0)
assert np.allclose(N.edge_out_bearing,
edge_data[:, 6],
atol=0.001,
rtol=0)
# check alternate min_threshold_wt gets passed through successfully
alt_min = 0.02
alt_distances = networks.distance_from_beta(betas,
min_threshold_wt=alt_min)
N = networks.Network_Layer_From_nX(G,
betas=betas,
min_threshold_wt=alt_min)
assert np.allclose(N.distances, alt_distances, atol=0.001, rtol=0)
# check for malformed signatures
with pytest.raises(TypeError):
networks.Network_Layer_From_nX('boo', distances=distances)
with pytest.raises(ValueError):
networks.Network_Layer_From_nX(G) # no betas or distances
with pytest.raises(ValueError):
networks.Network_Layer_From_nX(G, distances=None, betas=None)
with pytest.raises(ValueError):
networks.Network_Layer_From_nX(G, distances=[])
with pytest.raises(ValueError):
networks.Network_Layer_From_nX(G, betas=[])
def test_nX_from_graph_maps():
# also see test_networks.test_to_networkX for tests on implementation via Network layer
# check round trip to and from graph maps results in same graph
G = mock.mock_graph()
G = graphs.nX_simple_geoms(G)
# explicitly set live params for equality checks
# graph_maps_from_networkX generates these implicitly if missing
for n in G.nodes():
G.nodes[n]['live'] = bool(np.random.randint(0, 1))
# test directly from and to graph maps
node_uids, node_data, edge_data, node_edge_map = graphs.graph_maps_from_nX(G)
G_round_trip = graphs.nX_from_graph_maps(node_uids, node_data, edge_data, node_edge_map)
assert list(G_round_trip.nodes) == list(G.nodes)
assert list(G_round_trip.edges) == list(G.edges)
# check with metrics dictionary
N = networks.Network_Layer_From_nX(G, distances=[500, 1000])
N.compute_centrality(measures=['node_harmonic'])
data_dict = mock.mock_data_dict(G)
landuse_labels = mock.mock_categorical_data(len(data_dict))
D = layers.Data_Layer_From_Dict(data_dict)
D.assign_to_network(N, max_dist=400)
D.compute_aggregated(landuse_labels,
mixed_use_keys=['hill', 'shannon'],
accessibility_keys=['a', 'c'],
qs=[0, 1])
metrics_dict = N.metrics_to_dict()
# without backbone
G_round_trip_data = graphs.nX_from_graph_maps(node_uids,
node_data,
edge_data,
node_edge_map,
metrics_dict=metrics_dict)
for uid, metrics in metrics_dict.items():
assert G_round_trip_data.nodes[uid]['metrics'] == metrics
# with backbone
G_round_trip_data = graphs.nX_from_graph_maps(node_uids,
node_data,
edge_data,
node_edge_map,
networkX_graph=G,
metrics_dict=metrics_dict)
for uid, metrics in metrics_dict.items():
assert G_round_trip_data.nodes[uid]['metrics'] == metrics
# test with decomposed
G_decomposed = graphs.nX_decompose(G, decompose_max=20)
# set live explicitly
for n in G_decomposed.nodes():
G_decomposed.nodes[n]['live'] = bool(np.random.randint(0, 1))
node_uids_d, node_data_d, edge_data_d, node_edge_map_d = graphs.graph_maps_from_nX(G_decomposed)
G_round_trip_d = graphs.nX_from_graph_maps(node_uids_d, node_data_d, edge_data_d, node_edge_map_d)
assert list(G_round_trip_d.nodes) == list(G_decomposed.nodes)
for n, node_data in G_round_trip.nodes(data=True):
assert n in G_decomposed
assert node_data['live'] == G_decomposed.nodes[n]['live']
assert node_data['x'] == G_decomposed.nodes[n]['x']
assert node_data['y'] == G_decomposed.nodes[n]['y']
assert G_round_trip_d.edges == G_decomposed.edges
# error checks for when using backbone graph:
# mismatching numbers of nodes
corrupt_G = G.copy()
corrupt_G.remove_node(0)
with pytest.raises(ValueError):
graphs.nX_from_graph_maps(node_uids, node_data, edge_data, node_edge_map, networkX_graph=corrupt_G)
# mismatching node uid
with pytest.raises(ValueError):
corrupt_node_uids = list(node_uids)
corrupt_node_uids[0] = 'boo'
graphs.nX_from_graph_maps(corrupt_node_uids, node_data, edge_data, node_edge_map, networkX_graph=G)
def mmml_phd(_graph, _iters: int, distances: list, avg_dwell_dens: float,
models: dict, column_order: list, cap_step: float,
cap_jitter: float, max_cap: float, new_threshold: float,
random_seed: int):
'''
landuse states is based on the number of nodes: kept simple for visualisation
don't confuse with the data layer, which is POI based
for the sake of the simulation, all POI's assume the same x, y locations as assigned nodes
column order must be filtered to exclude the currently targeted layer
the tension between existing stores and latent competition (for given capacity) is important to dynamics
the non-linearity between activation and deactivation is also important
'''
_spans = len(_graph)
# set random seed for reproducibility and comparison across scenarios
np.random.seed(random_seed)
# build network layer
Netw_Layer = networks.Network_Layer_From_nX(_graph, distances=distances)
# calculate centralities - static
Netw_Layer.compute_centrality(['node_harmonic_angular'], angular=True)
Netw_Layer.compute_centrality(['node_betweenness_beta'], angular=False)
# generate flat data layer - network assignment happens internally - note randomised=False
Flat_Layer = generate_data_layer(_spans, 1, Netw_Layer, _randomised=False)
# calculate dwelling density - static
dwellings = np.full(_spans, avg_dwell_dens)
Flat_Layer.compute_stats_single('population_density', dwellings)
# the dwellings map is static in this case, so may as well set up front
dwellings_map = np.full((_iters, _spans), float(avg_dwell_dens))
# prepare the arrays for timeline maps that keep track of computations
# one per layer / model
landuse_maps = []
capacitance_maps = []
for _ in models:
landuse_maps.append(np.full((_iters, _spans), 0.0))
capacitance_maps.append(np.full((_iters, _spans), 0.0))
# per layer
n_layers = len(models)
landuse_states = np.full((n_layers, _spans), 0.0)
capacitances = np.full((n_layers, _spans), 0.0)
measured_accessibility = np.full((n_layers, _spans), 0.0)
predicted_accessibility = np.full((n_layers, _spans), 0.0)
# iterate iters
for n in tqdm(range(_iters)):
# build a landuse map reflecting the current state
# shape is sum of all landuses by x, y, nearest, next nearest
data_uids = []
lu_data_map = np.full((int(landuse_states.sum()), 4), np.nan)
landuse_labels = []
# iterate each model's layer
current_lu_counter = 0
for model_idx, model_key in enumerate(models.keys()):
# get the node indices of each active landuse location (for given model's layer)
current_active = np.where(landuse_states[model_idx] != 0)[0]
# iterate the active node locations and add to the current landuses data structure
for active_idx in current_active:
# create one instance per landuse
for active_n in range(
int(landuse_states[model_idx][active_idx])):
# get the x and y
x, y = (Netw_Layer.x_arr[active_idx],
Netw_Layer.y_arr[active_idx])
# add to uids
data_uids.append(current_lu_counter)
# add to the data structure: nearest node will be the current node
lu_data_map[current_lu_counter] = [
x, y, active_idx, np.nan
]
current_lu_counter += 1
# add the landuse code associated with the model
landuse_labels.append(model_key)
# the current landuses data structure can now be used for landuse accessibilities
# use empty placeholders if no landuse datapoints
# first iter needs to create keys
if not lu_data_map.shape[0]:
for key in models.keys():
if not key in Netw_Layer.metrics['accessibility']['weighted']:
Netw_Layer.metrics['accessibility']['weighted'][key] = {}
for dist in distances:
if not dist in Netw_Layer.metrics['accessibility'][
'weighted'][key]:
Netw_Layer.metrics['accessibility']['weighted'][key][
dist] = {}
Netw_Layer.metrics['accessibility']['weighted'][key][
dist] = np.full(_spans, 0.0)
else:
# generate a datalayer
DL = layers.Data_Layer(data_uids, lu_data_map)
# manually assign the network layer
DL._Network = Netw_Layer
# compute accessibilities for label of each datapoint against list of target keys
DL.compute_accessibilities(landuse_labels, list(models.keys()))
# for each model, build the input data array fed to the model, then predict
for model_idx, (model_key, model_vals) in enumerate(models.items()):
# number of columns is columns * distances - target columns * distances
col_num = len(column_order) * len(distances) - len(distances)
# placeholder array
X_arr = np.full((_spans, col_num), np.nan)
# iterate the columns, then distances, and add to input data array
col_num_idx = 0
for col in column_order:
# target columns
if col == model_key:
continue
elif col == 'c_node_harmonic_angular_{dist}':
c = Netw_Layer.metrics['centrality'][
'node_harmonic_angular']
elif col == 'c_node_betweenness_beta_{dist}':
c = Netw_Layer.metrics['centrality'][
'node_betweenness_beta']
elif col == 'cens_dwellings_{dist}':
c = Netw_Layer.metrics['stats']['population_density'][
'sum_weighted']
else:
c = Netw_Layer.metrics['accessibility']['weighted'][col]
for dist in distances:
X_arr[:, col_num_idx] = c[dist]
col_num_idx += 1
assert col_num_idx == col_num
# predict y_hat
X_trans = model_vals['transformer'].transform(X_arr)
predicted_accessibility[model_idx] = model_vals['model'].predict(
X_trans).flatten()
# extract the target column at the target distance for comparison to predicted
for model_idx, (model_key, model_vals) in enumerate(models.items()):
d = model_vals['target_dist']
measured_accessibility[model_idx] = \
Netw_Layer.metrics['accessibility']['weighted'][model_key][d]
# adjust capacitances and landuse states
for i in range(len(models)):
# calculate potentials
potentials = predicted_accessibility[i] - measured_accessibility[i]
# currently occupied landuse locations for given layer
occupied_loc = landuse_states[i] == 1
# squash landuses - do this inside loop to capture previous on / off state changes
squashed_landuses = landuse_states.sum(axis=0)
"""
EXISTING LOCATIONS
"""
# existing locations - OK as long as potentials are greater than measured
exist_pot = np.copy(potentials)
# normalise to smoothen and avoid batch-wise blocks
exist_pot /= np.abs(
potentials).max() # intentionally using potentials
# scale to capacitance step
exist_pot *= cap_step
# only applies to current landuse locations
capacitances[i][occupied_loc] += exist_pot[occupied_loc]
"""
POTENTIAL LOCATIONS
"""
future_pot = np.copy(potentials)
# future potentials - must exceed new_threshold to trigger new location
future_pot -= new_threshold
# normalise to smoothen and avoid batch-wise blocks
future_pot /= np.abs(
potentials).max() # intentionally using potentials
# scale to capacitance step
future_pot *= cap_step
# only applies to available locations
capacitances[i][~occupied_loc] += future_pot[~occupied_loc]
"""
LANDUSE STATES
"""
# add some stochasticity - scale to capacitance step
capacitances[i] += np.random.randn(_spans) * cap_step * cap_jitter
# clip capacitances
capacitances[i] = np.clip(capacitances[i], 0, max_cap)
# turn off per current capacitance for current landuse
off_idx = np.logical_and(capacitances[i] <= 0,
landuse_states[i] == 1)
landuse_states[i][off_idx] = 0
# turn on if no landuses currently at this index
on_idx = np.logical_and(capacitances[i] >= 1,
squashed_landuses == 0)
landuse_states[i][on_idx] = 1
# record landuse and capacitance states
capacitance_maps[i][n] = capacitances[i]
landuse_maps[i][n] = landuse_states[i]
return dwellings_map, landuse_maps, capacitance_maps
def test_compute_aggregated_A():
G = mock.mock_graph()
G = graphs.nX_simple_geoms(G)
betas = np.array([-0.01, -0.005])
distances = networks.distance_from_beta(betas)
# network layer
N = networks.Network_Layer_From_nX(G, distances)
node_map = N._node_data
edge_map = N._edge_data
node_edge_map = N._node_edge_map
# data layer
data_dict = mock.mock_data_dict(G)
qs = np.array([0, 1, 2])
D = layers.Data_Layer_From_Dict(data_dict)
# check single metrics independently against underlying for some use-cases, e.g. hill, non-hill, accessibility...
D.assign_to_network(N, max_dist=500)
# generate some mock landuse data
landuse_labels = mock.mock_categorical_data(len(data_dict))
landuse_classes, landuse_encodings = layers.encode_categorical(
landuse_labels)
# compute hill mixed uses
D.compute_aggregated(landuse_labels,
mixed_use_keys=['hill_branch_wt'],
qs=qs)
# test against underlying method
data_map = D._data
mu_data_hill, mu_data_other, ac_data, ac_data_wt, \
stats_sum, stats_sum_wt, stats_mean, stats_mean_wt, stats_variance, stats_variance_wt, stats_max, stats_min = \
data.local_aggregator(node_map,
edge_map,
node_edge_map,
data_map,
distances,
betas,
landuse_encodings,
qs=qs,
mixed_use_hill_keys=np.array([1]))
for q_idx, q_key in enumerate(qs):
for d_idx, d_key in enumerate(distances):
assert np.allclose(
N.metrics['mixed_uses']['hill_branch_wt'][q_key][d_key],
mu_data_hill[0][q_idx][d_idx],
atol=0.001,
rtol=0)
# gini simpson
D.compute_aggregated(landuse_labels, mixed_use_keys=['gini_simpson'])
# test against underlying method
data_map = D._data
mu_data_hill, mu_data_other, ac_data, ac_data_wt, \
stats_sum, stats_sum_wt, stats_mean, stats_mean_wt, stats_variance, stats_variance_wt, stats_max, stats_min = \
data.local_aggregator(node_map,
edge_map,
node_edge_map,
data_map,
distances,
betas,
landuse_encodings,
mixed_use_other_keys=np.array([1]))
for d_idx, d_key in enumerate(distances):
assert np.allclose(N.metrics['mixed_uses']['gini_simpson'][d_key],
mu_data_other[0][d_idx],
atol=0.001,
rtol=0)
# accessibilities
D.compute_aggregated(landuse_labels, accessibility_keys=['c'])
# test against underlying method
data_map = D._data
mu_data_hill, mu_data_other, ac_data, ac_data_wt, \
stats_sum, stats_sum_wt, stats_mean, stats_mean_wt, stats_variance, stats_variance_wt, stats_max, stats_min = \
data.local_aggregator(node_map,
edge_map,
node_edge_map,
data_map,
distances,
betas,
landuse_encodings,
accessibility_keys=np.array([landuse_classes.index('c')]))
for d_idx, d_key in enumerate(distances):
assert np.allclose(
N.metrics['accessibility']['non_weighted']['c'][d_key],
ac_data[0][d_idx],
atol=0.001,
rtol=0)
assert np.allclose(N.metrics['accessibility']['weighted']['c'][d_key],
ac_data_wt[0][d_idx],
atol=0.001,
rtol=0)
# also check the number of returned types for a few assortments of metrics
mixed_uses_hill_types = np.array([
'hill', 'hill_branch_wt', 'hill_pairwise_wt', 'hill_pairwise_disparity'
])
mixed_use_other_types = np.array(
['shannon', 'gini_simpson', 'raos_pairwise_disparity'])
ac_codes = np.array(landuse_classes)
mu_hill_random = np.arange(len(mixed_uses_hill_types))
np.random.shuffle(mu_hill_random)
mu_other_random = np.arange(len(mixed_use_other_types))
np.random.shuffle(mu_other_random)
ac_random = np.arange(len(landuse_classes))
np.random.shuffle(ac_random)
# mock disparity matrix
mock_disparity_wt_matrix = np.full(
(len(landuse_classes), len(landuse_classes)), 1)
# not necessary to do all labels, first few should do
for mu_h_min in range(3):
mu_h_keys = np.array(mu_hill_random[mu_h_min:])
for mu_o_min in range(3):
mu_o_keys = np.array(mu_other_random[mu_o_min:])
for ac_min in range(3):
ac_keys = np.array(ac_random[ac_min:])
# in the final case, set accessibility to a single code otherwise an error would be raised
if len(mu_h_keys) == 0 and len(mu_o_keys) == 0 and len(
ac_keys) == 0:
ac_keys = np.array([0])
# randomise order of keys and metrics
mu_h_metrics = mixed_uses_hill_types[mu_h_keys]
mu_o_metrics = mixed_use_other_types[mu_o_keys]
ac_metrics = ac_codes[ac_keys]
N_temp = networks.Network_Layer_From_nX(G, distances)
D_temp = layers.Data_Layer_From_Dict(data_dict)
D_temp.assign_to_network(N_temp, max_dist=500)
D_temp.compute_aggregated(
landuse_labels,
mixed_use_keys=list(mu_h_metrics) + list(mu_o_metrics),
accessibility_keys=ac_metrics,
cl_disparity_wt_matrix=mock_disparity_wt_matrix,
qs=qs)
# test against underlying method
mu_data_hill, mu_data_other, ac_data, ac_data_wt, stats_sum, stats_sum_wt, \
stats_mean, stats_mean_wt, stats_variance, stats_variance_wt, stats_max, stats_min = \
data.local_aggregator(node_map,
edge_map,
node_edge_map,
data_map,
distances,
betas,
landuse_encodings,
qs=qs,
mixed_use_hill_keys=mu_h_keys,
mixed_use_other_keys=mu_o_keys,
accessibility_keys=ac_keys,
cl_disparity_wt_matrix=mock_disparity_wt_matrix)
for mu_h_idx, mu_h_met in enumerate(mu_h_metrics):
for q_idx, q_key in enumerate(qs):
for d_idx, d_key in enumerate(distances):
assert np.allclose(
N_temp.metrics['mixed_uses'][mu_h_met][q_key]
[d_key],
mu_data_hill[mu_h_idx][q_idx][d_idx],
atol=0.001,
rtol=0)
for mu_o_idx, mu_o_met in enumerate(mu_o_metrics):
for d_idx, d_key in enumerate(distances):
assert np.allclose(
N_temp.metrics['mixed_uses'][mu_o_met][d_key],
mu_data_other[mu_o_idx][d_idx],
atol=0.001,
rtol=0)
for ac_idx, ac_met in enumerate(ac_metrics):
for d_idx, d_key in enumerate(distances):
assert np.allclose(N_temp.metrics['accessibility']
['non_weighted'][ac_met][d_key],
ac_data[ac_idx][d_idx],
atol=0.001,
rtol=0)
assert np.allclose(N_temp.metrics['accessibility']
['weighted'][ac_met][d_key],
ac_data_wt[ac_idx][d_idx],
atol=0.001,
rtol=0)
# most integrity checks happen in underlying method, though check here for mismatching labels length and typos
with pytest.raises(ValueError):
D.compute_aggregated(landuse_labels[-1], mixed_use_keys=['shannon'])
with pytest.raises(ValueError):
D.compute_aggregated(landuse_labels, mixed_use_keys=['spelling_typo'])
# don't check accessibility_labels for typos - because only warning is triggered (not all labels will be in all data)
# check that unassigned data layer flags
with pytest.raises(ValueError):
D_new = layers.Data_Layer_From_Dict(data_dict)
D_new.compute_aggregated(landuse_labels, mixed_use_keys=['shannon'])
def mmm_layercake_phd(_graph,
_iters=200,
echo_rate=0.2,
echo_distance=400,
competition_factor=1.1,
death_rate=0.01,
flow_jitter=0.2,
cap_jitter=0.1,
_layer_specs=(),
random_seed=0):
if isinstance(_layer_specs, dict):
_layer_specs = (_layer_specs,)
if not len(_layer_specs):
raise AttributeError('''
No layer specs provided: e.g. ({
cap_step=0.5,
dist_threshold=800,
pop_threshold=800
})
''')
for l_s in _layer_specs:
for k in ['cap_step', 'dist_threshold', 'pop_threshold']:
if k not in l_s:
raise AttributeError(f'Missing key {k}')
_spans = len(_graph)
# generate the backbone Network Layer
# include 1 (local only) for directly counting items assigned to current node
# also include np.inf so that all node combinations are considered
# in spite of wrapping around back
distances = [l_s['dist_threshold'] for l_s in _layer_specs]
distances = list(set(distances))
# add echo distance
distances.append(echo_distance)
distances = list(set(distances))
# build network layer
Netw_Layer = networks.Network_Layer_From_nX(_graph, distances=distances)
# generate population data layer - assignment happens internally - note randomised=False
Pop_Layer = generate_data_layer(_spans, 1, Netw_Layer, _randomised=False)
# population state and timeline map
# for this experiment assignments are not changing
pop_state = np.full(len(Pop_Layer.uids), 1.0) # use floats!
pop_map = np.full((_iters, _spans), 0.0)
# generate the landuse substrate
# keep to 1 location per node for visualisation sake
Landuse_Layer = generate_data_layer(_spans, 1, Netw_Layer, _randomised=False)
# prepare the arrays for timeline maps that keep track of computations - one per layer
landuse_maps = []
capacitance_maps = []
for _ in _layer_specs:
landuse_maps.append(np.full((_iters, _spans), 0.0))
capacitance_maps.append(np.full((_iters, _spans), 0.0))
# flow map is based on squashed flows - only one copy necessary
flow_map = np.full((_iters, _spans), 0.0)
n_layers = len(_layer_specs)
# per layer
landuse_states = np.full((n_layers, len(Landuse_Layer.uids)), 0.0)
capacitances = np.full((n_layers, _spans), 0.0)
assigned_trips_actual = np.full((n_layers, _spans), 0.0)
# assigned_trips_potential = np.full((n_layers, _spans), 0.0)
assigned_trips_echos = np.full(_spans, 0.0) # echos is squashed
netw_flow_actual = np.full((n_layers, _spans), 0.0)
# netw_flow_potential = np.full((n_layers, _spans), 0.0)
netw_flow_echos = np.full(_spans, 0.0) # echos is squashed
agged_trips_actual = np.full((n_layers, _spans), 0.0)
agged_flows_actual = np.full((n_layers, _spans), 0.0)
# agged_trips_potential = np.full((n_layers, _spans), 0.0)
# agged_flows_potential = np.full((n_layers, _spans), 0.0)
# set random seed for reproducibility and comparison across scenarios
np.random.seed(random_seed)
# set seed landuse location (otherwise no initial flows)
for i in range(n_layers):
rdm_idx = np.random.choice(_spans, 1)
landuse_states[i][rdm_idx] = 1
# iterate
for n in tqdm(range(_iters)):
pop_intensity = pop_state
# record current state - actually doesn't change for this experiment...
pop_map[n] = pop_intensity
# apportion flow - once per layer
for i, l_s in enumerate(_layer_specs):
### ACTUAL
# compute singly constrained realised flows
# NOTE: do not use a capacitance weighted version:
# singly-constrained already effectively takes competition vis-a-vis flow
# potentials into account!!!
Landuse_Layer.model_singly_constrained('trips',
Pop_Layer._data,
Landuse_Layer._data,
pop_intensity,
landuse_states[i])
assigned_trips_actual[i] = \
Netw_Layer.metrics['models']['trips'][l_s['dist_threshold']]['assigned_trips']
netw_flow_actual[i] = Netw_Layer.metrics['models']['trips'][l_s['dist_threshold']][
'network_flows']
# aggregate PREVIOUS iterations echos to current trips BEFORE computing new echos
agged_trips_actual[i] = assigned_trips_actual[i] + assigned_trips_echos
agged_flows_actual[i] = netw_flow_actual[i] + netw_flow_echos
# squashing
squashed_trips = agged_trips_actual.sum(axis=0)
squashed_flows = agged_flows_actual.sum(axis=0)
squashed_landuses = landuse_states.sum(axis=0)
flow_map[n] = squashed_flows # record flow state
### ECHOS
# compute singly constrained echos (spillovers) - combined for all layers
echos = squashed_trips * echo_rate
Landuse_Layer.model_singly_constrained('echos',
Landuse_Layer._data, # landuse to landuse
Landuse_Layer._data, # landuse to landuse
echos,
squashed_landuses)
assigned_trips_echos = Netw_Layer.metrics['models']['echos'][echo_distance][
'assigned_trips']
netw_flow_echos = Netw_Layer.metrics['models']['echos'][echo_distance]['network_flows']
# compute
# the tension between existing stores and latent competition (for given capacity) is
# important to dynamics
# the non-linearity between activation and deactivation is also important
for i, l_s in enumerate(_layer_specs):
# update squashed landuses inside loop to capture changes
squashed_landuses = landuse_states.sum(axis=0)
'''
FITNESS - this is based on actual trips and assessed for this layer only
'''
# the fitness function:
# trips to specific location + conversion of adjacent flows
up_locations = np.logical_and(agged_trips_actual[i] >= l_s['pop_threshold'],
landuse_states[i] == 1)
capacitances[i][up_locations] += l_s['cap_step']
# decreases otherwise
down_locations = np.logical_and(agged_trips_actual[i] < l_s['pop_threshold'],
landuse_states[i] == 1)
capacitances[i][down_locations] -= l_s['cap_step']
'''
POTENTIALS - this is based on potentials
'''
# identify n potential: total population / threshold for landuse
potential = pop_state.sum() / l_s['pop_threshold']
# multiply by competition factor and cast to int
potential = int(potential * competition_factor)
# identify latent demand by subtracting actual number of current landuses
actual = landuse_states[i].sum()
latent = potential - actual
latent = int(np.clip(latent, 0, np.abs(latent)))
# check for available locations
available_locations = squashed_landuses == 0
# identify n highest (jittered) flows at unoccupied locations
jitter = np.random.randn(_spans) * squashed_flows.max() * flow_jitter
sorted_flows_idx = np.argsort(squashed_flows + jitter, kind='mergesort')[::-1]
# sort available locations accordingly and filter
sorted_locations = available_locations[sorted_flows_idx]
filtered_flows_idx = sorted_flows_idx[sorted_locations]
# snip per latent demand
filtered_flows_idx = filtered_flows_idx[:latent]
capacitances[i][filtered_flows_idx] += l_s['cap_step']
'''
CAPACITANCE - non-linearity
'''
# add some stochasticity - scale to capacitance step
capacitances[i] += np.random.randn(_spans) * l_s['cap_step'] * cap_jitter
# clip capacitances to range
capacitances[i] = np.clip(capacitances[i], 0, 1)
# turn off and on per current capacitance
off_idx = np.intersect1d(np.where(capacitances[i] == 0),
np.where(landuse_states[i] == 1)) # use layer
landuse_states[i][off_idx] = 0
# cull based on death rate probability
active_idx = np.where(landuse_states[i] == 1)[0]
cull = np.random.choice([0, 1], active_idx.shape[0],
p=[death_rate, 1 - death_rate])
cull_idx = active_idx[cull == 0]
landuse_states[i][cull_idx] = 0
capacitances[i][cull_idx] = 0
# turn on per current capacitance
on_idx = np.intersect1d(np.where(capacitances[i] == 1),
np.where(squashed_landuses == 0)) # use squashed
landuse_states[i][on_idx] = 1
# record landuse and capacitance states
capacitance_maps[i][n] = capacitances[i]
landuse_maps[i][n] = landuse_states[i]
return pop_map, landuse_maps, capacitance_maps, flow_map
def test_compute_aggregated_B():
'''
Test stats component
'''
G = mock.mock_graph()
G = graphs.nX_simple_geoms(G)
betas = np.array([-0.01, -0.005])
distances = networks.distance_from_beta(betas)
# network layer
N = networks.Network_Layer_From_nX(G, distances)
node_map = N._node_data
edge_map = N._edge_data
node_edge_map = N._node_edge_map
# data layer
data_dict = mock.mock_data_dict(G)
qs = np.array([0, 1, 2])
D = layers.Data_Layer_From_Dict(data_dict)
# check single metrics independently against underlying for some use-cases, e.g. hill, non-hill, accessibility...
D.assign_to_network(N, max_dist=500)
# generate some mock landuse data
mock_numeric = mock.mock_numerical_data(len(data_dict), num_arrs=2)
# generate stats
D.compute_aggregated(stats_keys=['boo', 'baa'],
stats_data_arrs=mock_numeric)
# test against underlying method
data_map = D._data
mu_data_hill, mu_data_other, ac_data, ac_data_wt, \
stats_sum, stats_sum_wt, stats_mean, stats_mean_wt, stats_variance, stats_variance_wt, stats_max, stats_min = \
data.local_aggregator(node_map,
edge_map,
node_edge_map,
data_map,
distances,
betas,
numerical_arrays=mock_numeric)
stats_keys = [
'max', 'min', 'sum', 'sum_weighted', 'mean', 'mean_weighted',
'variance', 'variance_weighted'
]
stats_data = [
stats_max, stats_min, stats_sum, stats_sum_wt, stats_mean,
stats_mean_wt, stats_variance, stats_variance_wt
]
for num_idx, num_label in enumerate(['boo', 'baa']):
for s_key, stats in zip(stats_keys, stats_data):
for d_idx, d_key in enumerate(distances):
assert np.allclose(N.metrics['stats'][num_label][s_key][d_key],
stats[num_idx][d_idx],
atol=0.001,
rtol=0)
# check that mismatching label and array lengths are caught
for labels, arrs in (
(['a'], mock_numeric), # mismatching lengths
(['a', 'b'], None), # missing arrays
(None, mock_numeric)): # missing labels
with pytest.raises(ValueError):
D.compute_aggregated(stats_keys=labels, stats_data_arrs=arrs)
def test_compute_centrality():
'''
Underlying method also tested via test_networks.test_network_centralities
'''
G = mock.mock_graph()
G = graphs.nX_simple_geoms(G)
betas = np.array([-0.01, -0.005])
distances = networks.distance_from_beta(betas)
# generate data structures
N = networks.Network_Layer_From_nX(G, distances)
node_data = N._node_data
edge_data = N._edge_data
node_edge_map = N._node_edge_map
# check measures against underlying method
N = networks.Network_Layer_From_nX(G, distances)
N.compute_centrality(measures=['node_density'])
# test against underlying method
measures_data = centrality.local_centrality(
node_data,
edge_data,
node_edge_map,
distances,
betas,
measure_keys=('node_density', ))
for d_idx, d_key in enumerate(distances):
assert np.allclose(N.metrics['centrality']['node_density'][d_key],
measures_data[0][d_idx])
# also check the number of returned types for a few assortments of metrics
measures = [
'node_density', 'node_farness', 'node_cycles', 'node_harmonic',
'segment_density', 'node_betweenness', 'segment_betweenness'
]
np.random.shuffle(measures) # in place
# not necessary to do all labels, first few should do
for min_idx in range(3):
measure_keys = np.array(measures[min_idx:])
N = networks.Network_Layer_From_nX(G, distances)
N.compute_centrality(measures=measures)
# test against underlying method
measures_data = centrality.local_centrality(
node_data,
edge_data,
node_edge_map,
distances,
betas,
measure_keys=tuple(measure_keys))
for m_idx, measure_name in enumerate(measure_keys):
for d_idx, d_key in enumerate(distances):
assert np.allclose(
N.metrics['centrality'][measure_name][d_key],
measures_data[m_idx][d_idx],
atol=0.001,
rtol=0)
# check that angular gets passed through
N_ang = networks.Network_Layer_From_nX(G, distances=[2000])
N_ang.compute_centrality(measures=['node_harmonic_angular'], angular=True)
N = networks.Network_Layer_From_nX(G, distances=[2000])
N.compute_centrality(measures=['node_harmonic'], angular=False)
assert not np.allclose(
N_ang.metrics['centrality']['node_harmonic_angular'][2000],
N.metrics['centrality']['node_harmonic'][2000],
atol=0.001,
rtol=0)
assert not np.allclose(
N_ang.metrics['centrality']['node_harmonic_angular'][2000],
N.metrics['centrality']['node_harmonic'][2000],
atol=0.001,
rtol=0)
# check that typos, duplicates, and mixed angular / non-angular are caught
with pytest.raises(ValueError):
N.compute_centrality(measures=['spelling_typo'])
with pytest.raises(ValueError):
N.compute_centrality(measures=['node_density', 'node_density'])
with pytest.raises(ValueError):
N.compute_centrality(
measures=['harmonic_angle', 'node_harmonic_angular'])
def mmm_layercake_a(_graph, _iters=200, _layer_specs=(), random_seed=0):
if isinstance(_layer_specs, dict):
_layer_specs = (_layer_specs, )
if not len(_layer_specs):
raise AttributeError('''
No layer specs provided: e.g. ({
cap_step=0.5,
dist_threshold=800,
pop_threshold=800,
spill_rate=1.1
})
''')
for l_s in _layer_specs:
for k in ['cap_step', 'dist_threshold', 'pop_threshold', 'spill_rate']:
if k not in l_s:
raise AttributeError(f'Missing key {k}')
_spans = len(_graph)
# generate the backbone Network Layer
# include 1 (local only) for directly counting items assigned to current node
# also include np.inf so that all node combinations are considered in spite of wrapping around back
distances = [l_s['dist_threshold'] for l_s in _layer_specs]
distances = list(set(distances))
Netw_Layer = networks.Network_Layer_From_nX(_graph, distances=distances)
# generate population data layer
Pop_Layer = generate_data_layer(_spans, 1, Netw_Layer, _randomised=False)
# population state and map
# for this experiment assignments are not changing
pop_state = np.full(len(Pop_Layer.uids), 20.0) # use floats!
# for plotting density onto to network nodes
pop_map = np.full((_iters, _spans), 0.0)
# generate the landuse substrate
# keep to 1 location per node for visualisation sake
# randomisation is immaterial because all will be instanced as 0.0
Landuse_Layer = generate_data_layer(_spans,
1,
Netw_Layer,
_randomised=False)
landuse_maps = []
capacitance_maps = []
spillover_maps = []
for _ in _layer_specs:
landuse_maps.append(np.full((_iters, _spans), 0.0))
capacitance_maps.append(np.full((_iters, _spans), 0.0))
spillover_maps.append(np.full((_iters, _spans), 0.0))
n_layers = len(_layer_specs)
# per layer
landuse_states = np.full((n_layers, len(Landuse_Layer.uids)), 0.0)
capacitances = np.full((n_layers, _spans), 0.0)
assigned_trips_actual = np.full((n_layers, _spans), 0.0)
constrained_spillovers = np.full((n_layers, _spans), 0.0)
netw_flows_actual = np.full((n_layers, _spans), 0.0)
# left and right spatial gradients
right_gradient = np.zeros_like(pop_state)
left_gradient = np.zeros_like(pop_state)
# get max and min
ma = 0
mi = np.inf
for n, d in _graph.nodes(data=True):
x = d['x']
if x > ma:
ma = x
if x < mi:
mi = x
# set gradient strength
for i, (n, d) in enumerate(_graph.nodes(data=True)):
x = d['x']
right_gradient[i] = (x - mi) / (ma - mi)
left_gradient[i] = 1 - right_gradient[i]
# for keeping track of fifths for different gradients etc.
inc_5 = int(_iters / 5)
# set random seed for reproducibility and comparison across scenarios
np.random.seed(random_seed)
# iterate
for n in tqdm(range(_iters)):
# calculate neighbourhood density
if n < inc_5:
pop_intensity = pop_state
elif n < inc_5 * 2:
pop_intensity = right_gradient * pop_state
elif n < inc_5 * 3:
pop_intensity = left_gradient * pop_state
elif n < inc_5 * 4:
pop_intensity = pop_state * 2
else:
pop_intensity = pop_state / 2
# record current state - actually doesn't change for this experiment...
pop_map[n] = pop_intensity
# identify locations for landuse development
# enforce single landuse per location - identify free parcels
squashed_landuse_states = np.sum(landuse_states, axis=0)
# resets
assigned_trips_actual.fill(0)
constrained_spillovers.fill(0)
for i, l_s in enumerate(_layer_specs):
# record current landuse state
# strength = landuse_states[i] * capacitances[i]
landuse_maps[i][n] = landuse_states[i]
# compute singly constrained realised flows
Landuse_Layer.model_singly_constrained('assigned_flows',
Pop_Layer._data,
Landuse_Layer._data,
pop_intensity,
landuse_states[i])
assigned_trips_actual[i] = Netw_Layer.metrics['models'][
'assigned_flows'][l_s['dist_threshold']]['assigned_trips']
flows = Netw_Layer.metrics['models']['assigned_flows'][
l_s['dist_threshold']]['network_flows']
# reciprocate spillovers
spills = assigned_trips_actual[i] * l_s['spill_rate']
Landuse_Layer.model_singly_constrained('assigned_spillovers',
Landuse_Layer._data,
Landuse_Layer._data, spills,
squashed_landuse_states)
constrained_spillovers[i] = Netw_Layer.metrics['models'][
'assigned_spillovers'][l_s['dist_threshold']]['assigned_trips']
spillovers = Netw_Layer.metrics['models']['assigned_spillovers'][
l_s['dist_threshold']]['assigned_trips']
# TODO:
# important: distributing flows this way causes different behaviour to below manner
# below manner immediately seeds to the lagging edge of landuses - causing folds or pulses
# on the other hand this manner mainly seeds on the leading edge nearer the actual flows
# i.e. back-fill situations more likely to head in other direction
# netw_flows_actual[i] = flows + spillovers
# spillover_maps[i][n] = spillovers
squashed_flows = np.sum(assigned_trips_actual, axis=0)
squashed_spillovers = np.sum(constrained_spillovers, axis=0)
for i, l_s in enumerate(_layer_specs):
# distribute flows & spillovers - does not strictly take competition into account... but easier than furness process
# TODO - spatial version: assymetrical betweenness or some or other gravity wouldn't take k competition into account...?
Landuse_Layer.compute_stats_single('flow_intensity',
squashed_flows)
flows = Netw_Layer.metrics['stats']['flow_intensity'][
'mean_weighted'][l_s['dist_threshold']]
Landuse_Layer.compute_stats_single('spillover_intensity',
squashed_spillovers)
spillovers = Netw_Layer.metrics['stats']['spillover_intensity'][
'mean_weighted'][l_s['dist_threshold']]
# TODO: see above
netw_flows_actual[i] = flows + spillovers
spillover_maps[i][n] = spillovers
# compute caps
for i, l_s in enumerate(_layer_specs):
# squash inside the loop to capture changes from previous iter
squashed_landuse_states = np.sum(landuse_states, axis=0)
# deduce flows and update capacitances
flows = np.copy(netw_flows_actual[i])
flows -= l_s['pop_threshold']
flows /= l_s['pop_threshold']
flows *= l_s['cap_step']
flows = np.clip(flows, -l_s['cap_step'], l_s['cap_step'])
capacitances[i] += flows
# only seed per seed_rate
t = netw_flows_actual[i]
potential = int(np.nansum(t) / l_s['pop_threshold'])
existing = np.nansum(t > l_s['pop_threshold'])
new = potential - existing
if new <= 0:
rd_idx = np.random.randint(0, _spans)
capacitances[i][rd_idx] = 1
else:
# prepare jitter - only scale jitter if not all == 0, e.g. first iter, otherwise use plain jitter
# jitter = np.random.random(_spans)
# jitter_scale = np.nanmax(flows) - np.nanmin(flows)
# if jitter_scale:
# jitter *= jitter_scale # * l_s['explore_rate']
# add jitter to flow
# jittered = flows + jitter
# sort by highest jittered flow
seed_idx = np.argsort(flows, kind='mergesort')[::-1]
# can't use np.intersect1d because it will sort indices
seed_idx = seed_idx[np.in1d(
seed_idx, np.where(squashed_landuse_states == 0))]
seed_idx = seed_idx[:new]
# normalise seeds and then add to capacitances
# normalise seeds and then add to capacitances for continuous changes
# for single length arrays, normalised min will have max = 0 - use nanmax or treat separately
if len(seed_idx) == 1:
capacitances[i][seed_idx] += 1
elif len(seed_idx) != 0:
seed_vals = flows[seed_idx]
seed_vals -= np.nanmin(seed_vals)
seed_vals /= np.nanmax(seed_vals)
# seed_vals *= l_s['cap_step']
capacitances[i][seed_idx] += seed_vals
# constrain capacitance
capacitances[i] = np.clip(capacitances[i], 0, 1)
# deactivate dead landuses and activate new
off_idx = np.intersect1d(np.where(capacitances[i] <= 0),
np.where(squashed_landuse_states == 1))
on_idx = np.intersect1d(np.where(capacitances[i] >= 1),
np.where(squashed_landuse_states == 0))
landuse_states[i][off_idx] = 0
landuse_states[i][on_idx] = 1
# record capacitance state
capacitance_maps[i][n] = capacitances[i]
return pop_map, landuse_maps, capacitance_maps, spillover_maps
plt.style.use('./matplotlibrc')
base_path = path.dirname(__file__)
#
#
# INTRO PLOT
G = mock.mock_graph()
plot.plot_nX(G, path='graph.png', labels=True, dpi=150)
# INTRO EXAMPLE PLOTS
G = graphs.nX_simple_geoms(G)
G = graphs.nX_decompose(G, 20)
N = networks.Network_Layer_From_nX(G, distances=[400, 800])
N.compute_centrality(measures=['segment_harmonic'])
data_dict = mock.mock_data_dict(G, random_seed=25)
D = layers.Data_Layer_From_Dict(data_dict)
D.assign_to_network(N, max_dist=400)
landuse_labels = mock.mock_categorical_data(len(data_dict), random_seed=25)
D.hill_branch_wt_diversity(landuse_labels, qs=[0])
G_metrics = N.to_networkX()
segment_harmonic_vals = []
mixed_uses_vals = []
for node, data in G_metrics.nodes(data=True):
segment_harmonic_vals.append(
data['metrics']['centrality']['segment_harmonic'][800])
mixed_uses_vals.append(
def mmm_layercake_b(_graph,
_iters=200,
_layer_specs=(),
seed=False,
random_seed=0):
if isinstance(_layer_specs, dict):
_layer_specs = (_layer_specs, )
if not len(_layer_specs):
raise AttributeError('''
No layer specs provided: e.g. ({
cap_step=0.5,
dist_threshold=800,
pop_threshold=800,
explore_rate=0.5
})
''')
for l_s in _layer_specs:
for k in [
'cap_step', 'dist_threshold', 'pop_threshold', 'explore_rate'
]:
if k not in l_s:
raise AttributeError(f'Missing key {k}')
_spans = len(_graph)
# generate the backbone Network Layer
# include 1 (local only) for directly counting items assigned to current node
# also include np.inf so that all node combinations are considered in spite of wrapping around back
distances = [l_s['dist_threshold'] for l_s in _layer_specs]
distances = list(set(distances))
Netw_Layer = networks.Network_Layer_From_nX(_graph, distances=distances)
# generate population data layer - assignment happens internally - note randomised=False
Pop_Layer = generate_data_layer(_spans, 1, Netw_Layer, _randomised=False)
# population state and map
# for this experiment assignments are not changing
pop_state = np.full(len(Pop_Layer.uids), 20.0) # use floats!
# for plotting density onto to network nodes
pop_map = np.full((_iters, _spans), 0.0)
# generate the landuse substrate
# keep to 1 location per node for visualisation sake
Landuse_Layer = generate_data_layer(_spans,
1,
Netw_Layer,
_randomised=False)
landuse_maps = []
capacitance_maps = []
for _ in _layer_specs:
landuse_maps.append(np.full((_iters, _spans), 0.0))
capacitance_maps.append(np.full((_iters, _spans), 0.0))
n_layers = len(_layer_specs)
landuse_states = np.full((n_layers, len(Landuse_Layer.uids)),
0.0) # per layer
capacitances = np.full((n_layers, _spans), 0.0)
assigned_trips_actual = np.full((n_layers, _spans), 0.0)
assigned_trips_potential = np.full((n_layers, _spans), 0.0)
netw_flow_actual = np.full((n_layers, _spans), 0.0)
netw_flow_potential = np.full((n_layers, _spans), 0.0)
# left and right gradients
gradient = np.arange(_spans)
gradient = gradient / np.max(gradient)
right_gradient = np.full(_spans, 0.0)
mid = int(_spans / 2)
right_gradient[mid:] = gradient[:mid]
right_gradient[:mid] = gradient[mid:]
left_gradient = np.flip(right_gradient)
# for keeping track of fifths for different gradients etc.
inc_5 = int(_iters / 5)
# set random seed for reproducibility and comparison across scenarios
np.random.seed(random_seed)
# seed
if seed:
for n in range(n_layers):
rand_idx = np.random.randint(0, _spans)
landuse_states[n][rand_idx] = 1
# iterate
for n in tqdm(range(_iters)):
# calculate neighbourhood density
pop_intensity = np.copy(pop_state)
if n < inc_5:
pass
elif n < inc_5 * 2:
pop_intensity *= left_gradient
elif n < inc_5 * 3:
pop_intensity *= right_gradient
elif n < inc_5 * 4:
pop_intensity *= 2
else:
pop_intensity /= 2
# record current state - actually doesn't change for this experiment...
pop_map[n] = pop_intensity
# apportion flow - once per layer
assigned_trips_actual.fill(0)
netw_flow_actual.fill(0)
assigned_trips_potential.fill(0)
netw_flow_potential.fill(0)
flat_lu = np.full(_spans, 1.0)
for i, l_s in enumerate(_layer_specs):
# record current landuse state
landuse_maps[i][n] = landuse_states[i]
# compute singly constrained realised flows
Landuse_Layer.model_singly_constrained('assigned_flows',
Pop_Layer._data,
Landuse_Layer._data,
pop_intensity,
landuse_states[i])
assigned_trips_actual[i] += Netw_Layer.metrics['models'][
'assigned_flows'][l_s['dist_threshold']]['assigned_trips']
netw_flow_actual[i] += Netw_Layer.metrics['models'][
'assigned_flows'][l_s['dist_threshold']]['network_flows']
# compute potential flows (for exploration)
# this is an even surface and doesn't need smoothing
Landuse_Layer.model_singly_constrained('potential_flows',
Pop_Layer._data,
Landuse_Layer._data,
pop_intensity, flat_lu)
assigned_trips_potential[i] += Netw_Layer.metrics['models'][
'potential_flows'][l_s['dist_threshold']]['assigned_trips']
netw_flow_potential[i] += Netw_Layer.metrics['models'][
'potential_flows'][l_s['dist_threshold']]['network_flows']
# compute caps
squashed_netw_flow_actual = np.sum(netw_flow_actual, axis=0)
squashed_netw_flow_potential = np.sum(netw_flow_potential, axis=0)
for i, l_s in enumerate(_layer_specs):
# deduce flows and update capacitances
flows = np.copy(squashed_netw_flow_actual)
flows -= l_s['pop_threshold']
flows /= l_s['pop_threshold']
flows *= l_s['cap_step']
flows = np.clip(flows, -l_s['cap_step'], l_s['cap_step'])
capacitances[i] += flows
# constrain capacitance
capacitances[i] = np.clip(capacitances[i], 0, 1)
# deactivate dead landuses and activate new
# TODO: the non-linearity of the activation / deactivation seems significant...?
off_idx = np.intersect1d(np.where(capacitances[i] <= 0),
np.where(landuse_states[i] == 1))
landuse_states[i][off_idx] = 0
on_idx = np.intersect1d(np.where(capacitances[i] >= 1),
np.where(landuse_states[i] == 0))
landuse_states[i][on_idx] = 1
# record capacitance state
capacitance_maps[i][n] = capacitances[i]
# only seed per seed_rate
potential = np.nansum(
assigned_trips_potential[i]) / l_s['pop_threshold']
existing = np.nansum(
assigned_trips_actual[i]) > l_s['pop_threshold']
# VS: existing = np.nansum(landuse_states[i] == 1)
new = int(potential - existing)
if new > 0:
blended = squashed_netw_flow_actual * (
1 - l_s['explore_rate']
) + squashed_netw_flow_potential * l_s['explore_rate']
jitter = np.random.randn(
_spans) * l_s['cap_step'] - l_s['cap_step'] / 2
blended += jitter
# sort by highest jittered flow
seed_idx = np.argsort(blended, kind='mergesort')[::-1]
# can't use np.intersect1d because it will sort indices
seed_idx = seed_idx[np.in1d(
seed_idx,
np.where(landuse_states[i] == 0)[0])]
seed_idx = seed_idx[:new]
# normalise seeds and then add to capacitances for continuous changes
# for single length arrays, normalised min will have max = 0 - use nanmax or treat separately
if len(seed_idx) == 1:
capacitances[i][seed_idx] += 1
elif len(seed_idx) != 0:
seed_vals = blended[seed_idx]
seed_vals -= np.nanmin(seed_vals)
seed_vals /= np.nanmax(seed_vals)
# seed_vals *= l_s['cap_step']
capacitances[i][seed_idx] += seed_vals
return pop_map, landuse_maps, capacitance_maps
def mmm_layercake_c(_graph, _iters=200, _layer_specs=(), random_seed=0):
if isinstance(_layer_specs, dict):
_layer_specs = (_layer_specs, )
if not len(_layer_specs):
raise AttributeError('''
No layer specs provided: e.g. ({
cap_step=0.5,
dist_threshold=800,
pop_threshold=800,
explore_rate=0.5,
comp_rate=0.5
})
''')
for l_s in _layer_specs:
for k in [
'cap_step', 'dist_threshold', 'pop_threshold', 'explore_rate',
'comp_rate'
]:
if k not in l_s:
raise AttributeError(f'Missing key {k}')
_spans = len(_graph)
# generate the backbone Network Layer
# include 1 (local only) for directly counting items assigned to current node
# also include np.inf so that all node combinations are considered in spite of wrapping around back
distances = [l_s['dist_threshold'] for l_s in _layer_specs]
distances = list(set(distances))
Netw_Layer = networks.Network_Layer_From_nX(_graph, distances=distances)
# generate population data layer - assignment happens internally - note randomised=False
Pop_Layer = generate_data_layer(_spans, 1, Netw_Layer, _randomised=False)
# population state and map
# for this experiment assignments are not changing
pop_state = np.full(len(Pop_Layer.uids), 20.0) # use floats!
# for plotting density onto to network nodes
pop_map = np.full((_iters, _spans), 0.0)
# generate the landuse substrate
# keep to 1 location per node for visualisation sake
Landuse_Layer = generate_data_layer(_spans,
1,
Netw_Layer,
_randomised=False)
landuse_maps = []
capacitance_maps = []
flow_maps = []
for _ in _layer_specs:
landuse_maps.append(np.full((_iters, _spans), 0.0))
capacitance_maps.append(np.full((_iters, _spans), 0.0))
flow_maps.append(np.full((_iters, _spans), 0.0))
n_layers = len(_layer_specs)
# per layer
landuse_states = np.full((n_layers, len(Landuse_Layer.uids)), 0.0)
capacitances = np.full((n_layers, _spans), 0.0)
assigned_trips_actual = np.full((n_layers, _spans), 0.0)
assigned_trips_potential = np.full((n_layers, _spans), 0.0)
netw_flow_actual = np.full((n_layers, _spans), 0.0)
netw_flow_potential = np.full((n_layers, _spans), 0.0)
# left and right spatial gradients
right_gradient = np.zeros_like(pop_state)
left_gradient = np.zeros_like(pop_state)
# get max and min
ma = 0
mi = np.inf
for n, d in _graph.nodes(data=True):
x = d['x']
if x > ma:
ma = x
if x < mi:
mi = x
# set gradient strength
for i, (n, d) in enumerate(_graph.nodes(data=True)):
x = d['x']
right_gradient[i] = (x - mi) / (ma - mi)
left_gradient[i] = 1 - right_gradient[i]
# for keeping track of fifths for different gradients etc.
inc_5 = int(_iters / 5)
# set random seed for reproducibility and comparison across scenarios
np.random.seed(random_seed)
# iterate
for n in tqdm(range(_iters)):
# calculate neighbourhood density
if n < inc_5:
pop_intensity = pop_state
elif n < inc_5 * 2:
pop_intensity = right_gradient * pop_state
elif n < inc_5 * 3:
pop_intensity = left_gradient * pop_state
elif n < inc_5 * 4:
pop_intensity = pop_state * 2
else:
pop_intensity = pop_state / 2
# record current state - actually doesn't change for this experiment...
pop_map[n] = pop_intensity
# apportion flow - once per layer
assigned_trips_actual.fill(0)
netw_flow_actual.fill(0)
assigned_trips_potential.fill(0)
netw_flow_potential.fill(0)
flat_lu = np.full(_spans, 1.0)
for i, l_s in enumerate(_layer_specs):
# record current landuse state
landuse_maps[i][n] = landuse_states[i]
# compute singly constrained realised flows
Landuse_Layer.model_singly_constrained('assigned_flows',
Pop_Layer._data,
Landuse_Layer._data,
pop_intensity,
landuse_states[i])
assigned_trips_actual[i] += Netw_Layer.metrics['models'][
'assigned_flows'][l_s['dist_threshold']]['assigned_trips']
netw_flow_actual[i] += Netw_Layer.metrics['models'][
'assigned_flows'][l_s['dist_threshold']]['network_flows']
# record flow state
flow_maps[i][n] = netw_flow_actual[i]
# compute potential flows (for exploration)
# this is an even surface and doesn't need smoothing
Landuse_Layer.model_singly_constrained('potential_flows',
Pop_Layer._data,
Landuse_Layer._data,
pop_intensity, flat_lu)
assigned_trips_potential[i] += Netw_Layer.metrics['models'][
'potential_flows'][l_s['dist_threshold']]['assigned_trips']
netw_flow_potential[i] += Netw_Layer.metrics['models'][
'potential_flows'][l_s['dist_threshold']]['network_flows']
# compute
# the tension between existing stores and latent competition (for given capacity) is important to dynamics
# the non-linearity between activation and deactivation is also important
squashed_netw_flow_actual = np.sum(netw_flow_actual, axis=0)
squashed_netw_flow_potential = np.sum(netw_flow_potential, axis=0)
for i, l_s in enumerate(_layer_specs):
# assign health to existing locations based on actual constrained flows - using continuous form
# don't need to filter by active because constrained flows only available for active locations
health = np.copy(assigned_trips_actual[i])
health -= l_s['pop_threshold']
health /= l_s['pop_threshold']
health *= l_s['cap_step']
health = np.clip(health, -l_s['cap_step'], l_s['cap_step'])
landuse_states[i] += health
landuse_states[i] = np.clip(landuse_states[i], 0, 1)
# capacitances are fluid - they constantly decay and need constant energy inputs to persist or grow
capacitances[i] /= 2
# determine latent competition - start by measuring actual and potential landuse intensity
potential = np.nansum(assigned_trips_potential[i])
actual = np.nansum(assigned_trips_actual[i])
# high potential represents full competition for flows, new entrants are trying to capture from existing
high_potential = potential / l_s['pop_threshold'] - np.nansum(
actual > l_s['pop_threshold'])
# low potential represents weak competition for flows, new entrants only trying to capture untapped potential
low_potential = (potential - actual) / l_s['pop_threshold']
# apply competition rate
new = int(low_potential * (1 - l_s['comp_rate']) +
high_potential * l_s['comp_rate'])
if new > 0:
# deduce flows, this does not need to be exact, as long as proportional representation of flows
blended = squashed_netw_flow_actual * (
1 - l_s['explore_rate']
) + squashed_netw_flow_potential * l_s['explore_rate']
jitter = np.random.randn(
_spans) * l_s['cap_step'] - l_s['cap_step'] / 2
blended += jitter
# sort by highest jittered flow
seed_idx = np.argsort(blended, kind='mergesort')[::-1]
# can't use np.intersect1d because it will sort indices
# trying to steal flows from existing buildings, so can't populate currently existing landuse activations
seed_idx = seed_idx[np.in1d(
seed_idx,
np.where(landuse_states[i] == 0)[0])]
# snip off at new
seed_idx = seed_idx[:new]
# normalise seeds and then add to capacitances for continuous changes
# for single length arrays, normalised min will have max = 0 - use nanmax or treat separately
if len(seed_idx) == 1:
capacitances[i][seed_idx] += 1
elif len(seed_idx) != 0:
seed_vals = blended[seed_idx]
seed_vals -= np.nanmin(seed_vals)
seed_vals /= np.nanmax(seed_vals)
# seed_vals *= l_s['cap_step']
capacitances[i][seed_idx] += seed_vals
on_idx = np.intersect1d(np.where(capacitances[i] >= 1),
np.where(landuse_states[i] == 0))
landuse_states[i][on_idx] = 1
# record capacitance state
capacitance_maps[i][n] = capacitances[i]
return pop_map, landuse_maps, capacitance_maps, flow_maps
B) calculate density (normalise)
C) calculate density weighted centrality
D) randomly select two population units and relocate from lower to higher (enforce min and max constraint)
'''
iters = 400
spans = 200
spine_nx = generate_graph(_spans=spans)
spine_nx_stepped = generate_graph(_spans=spans, _stepped=True)
util_funcs.plt_setup()
fig, axes = plt.subplots(1, 2, figsize=(12, 20))
for ax_n, graph in enumerate([spine_nx, spine_nx_stepped]):
# generate the Network Layer
Netw_Layer = networks.Network_Layer_From_nX(graph,
distances=[400, 800, np.inf])
# generate population layer
Pop_Layer = generate_data_layer(_spans=spans,
_intensity=20,
_Network_Layer=Netw_Layer)
# population state and map
pop_state = np.full(len(Pop_Layer.uids), 1.0) # use floats!
pop_map = np.full((iters, spans), 0.0)
# iterate
for n in tqdm(range(iters)):
# POPULATION
# record current state
pop_map[n] = set_current_num(Pop_Layer._data, Netw_Layer._nodes)
# calculate the effective density
# each population point is a single unit
# the state technically remains the same, it is the x, y and assignments that change!