def test_invalid_beta_throws(self):
distances = dok_matrix(hand_net_dists)
if issparse(distances):
hand_net_incidence_local = dok_matrix(hand_net_incidence)
else:
hand_net_incidence_local = hand_net_incidence
data_list = [distances]
network_struct = ModelDataStruct(data_list,
hand_net_incidence_local,
data_array_names_debug=("distances",
"u_turn"))
beta_vec = np.array([-5])
model = RecursiveLogitModelPrediction(network_struct,
initial_beta=beta_vec,
mu=1)
try:
model.generate_observations(origin_indices=[0, 1, 2, 7],
dest_indices=[1, 6, 3],
num_obs_per_pair=4,
iter_cap=15,
rng_seed=1)
except ValueError as e:
print(str(e))
def _tiny_modified_common_data_checks(travel_times_mat, left, u_turn,
t_time_incidence, incidence_mat,
obs_record):
data_list = [travel_times_mat, left, u_turn, t_time_incidence]
network_data_struct = ModelDataStruct(data_list, incidence_mat)
# network_data_struct.add_second_travel_time_for_testing()
optimiser = optimisers.LineSearchOptimiser(
optimisers.OptimHessianType.BFGS, max_iter=4)
RecursiveLogitModelEstimation.zeros_error_override = False
# hack
model = RecursiveLogitModelEstimation(network_data_struct,
optimiser,
observations_record=obs_record)
log_like_out, grad_out = model.get_log_likelihood()
eps = 1e-6
print(optimiser.get_iteration_log(
model.optim_function_state)) # Note this is currently required to
# set the gradient so that compute relative gradient works, really bad
# model.hessian = np.identity(network_data_struct.n_dims)
# Pre @42f564e9, we are manipulating behaviour slightly so this test still works
# should just replace with Sioux falls or similar
ll, line_search_step, grad_norm, rel_grad_norm = (2.079441541679836,
0.0,
0.7071067811865476,
0.24044917348149386)
assert np.abs(log_like_out - ll) < eps
assert np.abs(model.optimiser.step - line_search_step) < eps
assert np.abs(linalg.norm(grad_out) - grad_norm) < eps
assert np.abs(model.optimiser.compute_relative_gradient_non_static() -
rel_grad_norm) < eps
targets = [(1.699556, 0.7071068, 0.3803406, 0.1582422),
(1.495032, 0.8230394, 0.1457128, 0.06891792),
(1.440549, 0.5111388, 0.07468365, 0.03665915)]
for t in targets:
ll, line_search_step, grad_norm, rel_grad_norm = t
out_flag, hessian, log = optimiser.iterate_step(
model.optim_function_state, verbose=False)
log_like_out, grad_out = model.get_log_likelihood()
print(
f"({log_like_out:.7}, "
f"{linalg.norm(model.optimiser.step):.7}, {linalg.norm(grad_out):.7}, "
f"{model.optimiser.compute_relative_gradient_non_static():.7})"
)
assert np.abs(log_like_out - ll) < eps
assert np.abs(
linalg.norm(model.optimiser.step) - line_search_step) < eps
assert np.abs(linalg.norm(grad_out) - grad_norm) < eps
assert np.abs(
model.optimiser.compute_relative_gradient_non_static() -
rel_grad_norm) < eps
RecursiveLogitModelEstimation.zeros_error_override = None # reset
def test_compare_optim_methods(self):
subfolder = "ExampleTinyModifiedObs" # big data from classical v2
folder = join("Datasets", subfolder)
obs_mat, attrs = load_standard_path_format_csv(folder,
delim=" ",
angles_included=True)
import awkward1 as ak
obs_mat = obs_mat.toarray()
obs_record = ak.from_numpy(obs_mat)
incidence_mat, travel_times_mat, angle_cts_mat = attrs
left, _, _, u_turn = AngleProcessor.get_turn_categorical_matrices(
angle_cts_mat, incidence_mat)
data_list = [travel_times_mat, left, u_turn]
network_data_struct = ModelDataStruct(data_list, incidence_mat)
# network_data_struct.add_second_travel_time_for_testing()
optimiser = optimisers.LineSearchOptimiser(
optimisers.OptimHessianType.BFGS, max_iter=4)
RecursiveLogitModelEstimation.zeros_error_override = False
model = RecursiveLogitModelEstimation(network_data_struct,
optimiser,
observations_record=obs_record,
initial_beta=-15)
m1_ll_out, m1_grad_out = model.get_log_likelihood()
optimiser2 = optimisers.ScipyOptimiser(method='newton-cg')
model2 = RecursiveLogitModelEstimation(network_data_struct,
optimiser2,
observations_record=obs_record,
initial_beta=-15)
m2_ll_out, m2_grad_out = model2.get_log_likelihood()
assert np.allclose(m2_ll_out, m1_ll_out)
assert np.allclose(m2_grad_out, m1_grad_out)
beta1 = model.solve_for_optimal_beta()
beta2 = model2.solve_for_optimal_beta(verbose=True)
m1_ll_out, m1_grad_out = model.get_log_likelihood()
m2_ll_out, m2_grad_out = model2.get_log_likelihood()
print(m1_ll_out, m2_ll_out)
print(m1_grad_out, m2_grad_out)
assert np.allclose(beta1, beta2, 0.34657)
RecursiveLogitModelEstimation.zeros_error_override = None
def struct_bigger():
distances = np.array([[0, 5, 0, 4, 0, 0, 0, 0, 0, 0],
[0, 0, 6, 0, 0, 0, 0, 0, 0, 6],
[0, 6, 0, 5, 0, 0, 0, 0, 0, 0],
[4, 0, 0, 0, 5, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 6, 6, 0, 0, 0],
[5, 0, 0, 0, 6, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 6, 6, 0],
[0, 0, 0, 0, 0, 6, 6, 0, 0, 0],
[0, 0, 6, 0, 0, 0, 0, 0, 0, 6],
[0, 0, 0, 0, 0, 0, 0, 6, 6, 0]])
distances = dok_matrix(distances)
incidence_mat = (distances > 0).astype(int)
network_struct = ModelDataStruct([distances],
incidence_mat,
data_array_names_debug=("distances", ))
return network_struct
def _basic_consistencey_checks(distances):
data_list = [distances]
if issparse(distances):
hand_net_incidence_local = dok_matrix(hand_net_incidence)
else:
hand_net_incidence_local = hand_net_incidence
network_struct = ModelDataStruct(data_list,
hand_net_incidence_local,
data_array_names_debug=("distances",
"u_turn"))
beta_vec = np.array([-1])
model = RecursiveLogitModelPrediction(network_struct,
initial_beta=beta_vec,
mu=1)
obs = model.generate_observations(origin_indices=[0, 1, 2, 7],
dest_indices=[1, 6, 3],
num_obs_per_pair=4,
iter_cap=15,
rng_seed=1)
expected = TestSimulation._get_basic_consistency_expected(
ALLOW_POSITIVE_VALUE_FUNCTIONS)
assert obs == expected
def _first_example_common_data_checks(travel_times_mat, incidence_mat,
obs_mat):
data_list = [travel_times_mat, travel_times_mat]
network_data_struct = ModelDataStruct(data_list, incidence_mat)
optimiser = optimisers.LineSearchOptimiser(
optimisers.OptimHessianType.BFGS, max_iter=4)
model = RecursiveLogitModelEstimation(network_data_struct,
optimiser,
observations_record=obs_mat)
log_like_out, grad_out = model.get_log_likelihood()
eps = 1e-6
assert np.abs(log_like_out - 2.7725887222397816) < eps
assert np.abs(linalg.norm(grad_out) - 0) < eps
# model.hessian = np.identity(network_data_struct.n_dims)
out_flag, hessian, log = optimiser.iterate_step(
model.optim_function_state, verbose=False)
assert out_flag is True
assert (hessian == np.identity(2)).all()
assert optimiser.n_func_evals == 1
def consistency_test(network_file,
orig_indices,
dest_indices,
obs_per_pair,
beta0,
test_range=None):
if test_range is None:
test_range = np.arange(-0.1, -2.1, -0.1)
# network_file = "EMA_net.tntp"
data_list, data_list_names = load_tntp_node_formulation(
network_file,
columns_to_extract=[
"length",
],
)
distances = data_list[0]
incidence_mat = (distances > 0).astype(int)
network_struct = ModelDataStruct(data_list,
incidence_mat,
data_array_names_debug=("distances",
"u_turn"))
beta_vec = np.array([-0.1])
model = RecursiveLogitModelPrediction(network_struct,
initial_beta=beta_vec,
mu=1)
print("Linear system size", model.get_exponential_utility_matrix().shape)
print(
f"Generating {obs_per_pair * len(orig_indices) * len(dest_indices)} obs total per "
f"beta sim val")
def get_data(beta_vec, seed=None):
beta_vec_generate = np.array([beta_vec])
model = RecursiveLogitModelPrediction(network_struct,
initial_beta=beta_vec_generate,
mu=1)
obs = model.generate_observations(
origin_indices=orig_indices,
dest_indices=dest_indices,
num_obs_per_pair=obs_per_pair,
iter_cap=2000,
rng_seed=seed,
)
return obs
optimiser = optimisers.ScipyOptimiser(method='l-bfgs-b') # bfgs, l-bfgs-b
import time
a = time.time()
expected = []
actual = []
for n, beta_gen in enumerate(test_range, start=1):
expected.append(beta_gen)
try:
obs = get_data(beta_gen, seed=None)
except ValueError as e:
print(f"beta = {beta_gen} failed, {e}")
actual.append(0.0)
continue
# print(obs)
beta0 = -5
model = RecursiveLogitModelEstimation(network_struct,
observations_record=obs,
initial_beta=beta0,
mu=1,
optimiser=optimiser)
beta = model.solve_for_optimal_beta(verbose=False)
actual.append(float(beta))
print("beta_expected", beta_gen, "beta actual", beta, "\nOBS:")
# text_list = wrapper.wrap(str(obs))
# print("\n".join(text_list))
b = time.time()
print("elapsed =", b - a, "s")
return np.array(expected), np.array(actual)
# [3, 2.5, 2],
# ])
distances = np.array(
[[4, 3.5, 4.5, 3, 3, 0, 0, 0],
[3.5, 3, 4, 0, 2.5, 3, 3, 0],
[4.5, 4, 5, 0, 0, 0, 4, 3.5],
[3, 0, 0, 2, 2, 2.5, 0, 2],
[3, 2.5, 0, 2, 2, 2.5, 2.5, 0],
[0, 3, 0, 2.5, 2.5, 3, 3, 2.5],
[0, 3, 4, 0, 2.5, 3, 3, 2.5],
[0, 0, 3.5, 2, 0, 2.5, 2.5, 2]])
incidence_mat = (distances > 0).astype(int)
data_list = [distances]
network_struct = ModelDataStruct(data_list, incidence_mat,
data_array_names_debug=("distances"))
beta_vec = np.array([-16])
model = RecursiveLogitModelPrediction(network_struct,
initial_beta=beta_vec, mu=1)
# obs_indices = [i for i in range(8)]
# obs = model.generate_observations(origin_indices=obs_indices,
# dest_indices=obs_indices,
# num_obs_per_pair=1, iter_cap=2000, rng_seed=1,
# )
obs_indices = [1, 2, 3, 4, 5, 6, 7, 8]
obs = model.generate_observations(origin_indices=obs_indices,
dest_indices=[7, 3],
num_obs_per_pair=20, iter_cap=2000, rng_seed=1,
)
# subfolder ="ExampleTutorial"# "ExampleTutorial" from classical logicv2
# subfolder = "ExampleTiny"
subfolder = "ExampleTinyModifiedObs"
print(os.getcwd())
folder = os.path.join("Datasets", subfolder)
obs_mat, attrs = load_standard_path_format_csv(folder,
delim=" ",
angles_included=True)
incidence_mat, travel_times_mat, angle_cts_mat = attrs
left, _, _, u_turn = AngleProcessor.get_turn_categorical_matrices(
angle_cts_mat, incidence_mat)
# incidence matrix which only has nonzero travel times - rather than what is specified in file
t_time_incidence = (travel_times_mat > 0).astype('int').todok()
data_list = [travel_times_mat, left, u_turn, t_time_incidence]
network_data_struct = ModelDataStruct(data_list, incidence_mat)
#
# network_data_struct, obs_mat = ModelDataStruct.from_directory(
# folder, add_angles=True, angle_type='comparison', delim=" ")
#
# print(network_data_struct.data_array)
#
# print(network_data_struct.data_array)
time_io_end = time.time()
optimiser = LineSearchOptimiser(OptimHessianType.BFGS, max_iter=4)
model = RecursiveLogitModelEstimation(network_data_struct,
optimiser,
observations_record=obs_mat)
import numpy as np
from recursiveRouteChoice import RecursiveLogitModelPrediction, ModelDataStruct
# DATA
# A trivial network
distances = np.array([[0, 5, 0, 4], [0, 0, 6, 0], [0, 6, 0, 5], [4, 0, 0, 0]])
incidence_mat = (distances > 0).astype(int)
network_attribute_list = [distances]
network_struct = ModelDataStruct(network_attribute_list,
incidence_mat,
data_array_names_debug=("distances", ))
model = RecursiveLogitModelPrediction(network_struct,
initial_beta=[-0.4],
mu=1)
obs_indices = [0, 3]
dest_indices = [1, 2]
obs_per_pair = 15
print(
f"Generating {obs_per_pair * len(obs_indices) * len(dest_indices)} obs total"
)
obs = model.generate_observations(origin_indices=obs_indices,
dest_indices=dest_indices,
num_obs_per_pair=obs_per_pair,
iter_cap=2000,
rng_seed=1)
print(obs)