def renumber(self, new_id: int):
"""Renumbers the node in the network
Raises a warning if another node already exists with this node_id
Args:
*new_id* (:obj:`int`): New node_id
"""
new_id = int(new_id)
if new_id == self.node_id:
warn("This is already the node number")
return
conn = database_connection()
curr = conn.cursor()
curr.execute("BEGIN;")
curr.execute("Update Nodes set node_id=? where node_id=?",
[new_id, self.node_id])
curr.execute("Update Links set a_node=? where a_node=?",
[new_id, self.node_id])
curr.execute("Update Links set b_node=? where b_node=?",
[new_id, self.node_id])
curr.execute("COMMIT;")
conn.close()
logger.info(f"Node {self.node_id} was renumbered to {new_id}")
self.__dict__["node_id"] = new_id
self.__original__["node_id"] = new_id
def open(self, project_path: str) -> None:
"""
Loads project from disk
Args:
*project_path* (:obj:`str`): Full path to the project data folder. If the project inside does
not exist, it will fail.
"""
if self.__other_project_still_open():
raise Exception("You already have a project open. Close that project before opening another one")
file_name = os.path.join(project_path, "project_database.sqlite")
if not os.path.isfile(file_name):
raise FileNotFoundError("Model does not exist. Check your path and try again")
self.project_base_path = project_path
self.path_to_file = file_name
self.source = self.path_to_file
os.environ[ENVIRON_VAR] = self.project_base_path
self.conn = database_connection()
self.__load_objects()
self.__set_logging_path()
logger.info(f"Opened project on {self.project_base_path}")
self.logger = logger
clean()
def data_structures(self):
osmi = []
logger.info("Consolidating geo elements")
self.__emit_all(["text", "Consolidating geo elements"])
self.__emit_all(["maxValue", len(self.osm_items)])
for i, x in enumerate(self.osm_items):
osmi.append(x["elements"])
self.__emit_all(["Value", i])
self.osm_items = sum(osmi, [])
logger.info("Separating nodes and links")
self.__emit_all(["text", "Separating nodes and links"])
self.__emit_all(["maxValue", len(self.osm_items)])
alinks = []
n = []
for i, x in enumerate(self.osm_items):
if x["type"] == "way":
alinks.append(x)
elif x["type"] == "node":
n.append(x)
self.__emit_all(["Value", i])
self.osm_items = None
logger.info("Setting data structures for nodes")
self.__emit_all(["text", "Setting data structures for nodes"])
self.__emit_all(["maxValue", len(n)])
for i, node in enumerate(n):
nid = node.pop("id")
_ = node.pop("type")
self.nodes[nid] = node
self.__emit_all(["Value", i])
del n
logger.info("Setting data structures for links")
self.__emit_all(["text", "Setting data structures for links"])
self.__emit_all(["maxValue", len(alinks)])
all_nodes = []
for i, link in enumerate(alinks):
osm_id = link.pop("id")
_ = link.pop("type")
all_nodes.extend(link["nodes"])
self.links[osm_id] = link
self.__emit_all(["Value", i])
del alinks
logger.info("Finalizing data structures")
self.__emit_all(["text", "Finalizing data structures"])
node_count = self.unique_count(np.array(all_nodes))
return node_count
def save(self) -> None:
"""Saves any field descriptions which my have been changed to the database"""
qry = 'update attributes_documentation set description="{}" where attribute="{}" and name_table="{}"'
for key, val in self._original_values.items():
new_val = self.__dict__[key]
if new_val != val:
self.__run_query_commit(qry.format(new_val, key, self._table))
logger.info(
f"Metadata for field {key} on table {self._table} was updated to {new_val}"
)
def add(self, mode: Mode) -> None:
""" We add a mode to the project"""
self.__update_list_of_modes()
if mode.mode_id in self.__all_modes:
raise ValueError("Mode already exists in the model")
self.curr.execute(
"insert into 'modes'(mode_id, mode_name) Values(?,?)",
[mode.mode_id, mode.mode_name])
self.conn.commit()
logger.info(
f'mode {mode.mode_name}({mode.mode_id}) was added to the project')
mode.save()
self.__update_list_of_modes()
def close(self) -> None:
"""Safely closes the project"""
if ENVIRON_VAR in os.environ:
self.conn.commit()
clean()
self.conn.close()
for obj in [self.parameters, self.network]:
del obj
del os.environ[ENVIRON_VAR]
del self.network.link_types
del self.network.modes
logger.info(f"Closed project on {self.project_base_path}")
else:
warnings.warn("There is no Aequilibrae project open that you could close")
def run_queries_from_sql_file(conn: Connection, qry_file: str) -> None:
curr = conn.cursor()
with open(qry_file, "r") as sql_file:
query_list = sql_file.read()
# Running one query/command at a time helps debugging in the case a particular command fails
for cmd in query_list.split("--#"):
try:
curr.execute(cmd)
except Exception as e:
msg = f"Error running SQL command: {e.args}"
logger.error(msg)
logger.info(cmd)
raise e
def __add_trigger_from_file(self, qry_file: str):
curr = self.conn.cursor()
sql_file = open(qry_file, "r")
query_list = sql_file.read()
sql_file.close()
# Run one query/command at a time
for cmd in query_list.split("#"):
try:
curr.execute(cmd)
except Exception as e:
msg = f"Error creating trigger: {e.args}"
logger.error(msg)
logger.info(cmd)
self.conn.commit()
def new(self, zone_id: int) -> Zone:
"""Creates a new zone
Returns:
*zone* (:obj:`Zone`): A new zone object populated only with zone_id (but not saved in the model yet)
"""
if zone_id in self.__items:
raise Exception(f"Zone ID {zone_id} already exists")
data = {key: None for key in self.__fields}
data["zone_id"] = zone_id
logger.info(f"Zone with id {zone_id} was created")
return self.__create_return_zone(data)
def data_structures(self):
logger.info("Separating nodes and links")
self.__emit_all(["text", "Separating nodes and links"])
self.__emit_all(["maxValue", len(self.osm_items)])
alinks = []
n = []
tot_items = len(self.osm_items)
# When downloading data for entire countries, memory consumption can be quite intensive
# So we get rid of everything we don't need
for i in range(tot_items, 0, -1):
item = self.osm_items.pop(-1)
if item['type'] == "way":
alinks.append(item)
elif item['type'] == "node":
n.append(item)
self.__emit_all(["Value", tot_items - i])
gc.collect()
logger.info("Setting data structures for nodes")
self.__emit_all(["text", "Setting data structures for nodes"])
self.__emit_all(["maxValue", len(n)])
for i, node in enumerate(n):
nid = node.pop("id")
_ = node.pop("type")
self.nodes[nid] = node
self.__emit_all(["Value", i])
del n
logger.info("Setting data structures for links")
self.__emit_all(["text", "Setting data structures for links"])
self.__emit_all(["maxValue", len(alinks)])
all_nodes = []
for i, link in enumerate(alinks):
osm_id = link.pop("id")
_ = link.pop("type")
all_nodes.extend(link["nodes"])
self.links[osm_id] = link
self.__emit_all(["Value", i])
del alinks
logger.info("Finalizing data structures")
self.__emit_all(["text", "Finalizing data structures"])
node_count = self.unique_count(np.array(all_nodes))
return node_count
def new(self, project_path: str) -> None:
"""Creates a new project
Args:
*project_path* (:obj:`str`): Full path to the project data folder. If folder exists, it will fail
"""
if self.__other_project_still_open():
raise Exception("You already have a project open. Close that project before creating a new one")
self.project_base_path = project_path
self.path_to_file = os.path.join(self.project_base_path, "project_database.sqlite")
self.source = self.path_to_file
if os.path.isdir(project_path):
raise FileNotFoundError("Location already exists. Choose a different name or remove the existing directory")
os.environ[ENVIRON_VAR] = self.project_base_path
self.__create_empty_project()
self.__load_objects()
self.about.create()
self.__set_logging_path()
self.logger = logger
logger.info(f"Created project on {self.project_base_path}")
def import_nodes(self, nodes_to_add, node_ids):
table = "nodes"
fields = self.get_node_fields()
field_names = ",".join(fields)
field_names = ",".join(
['"{}"'.format(x) for x in field_names.split(",")])
logger.info("Adding network nodes")
self.__emit_all(["text", "Adding network nodes"])
self.__emit_all(["maxValue", len(nodes_to_add)])
vars = {}
for counter, osm_id in enumerate(nodes_to_add):
self.__emit_all(["Value", counter])
vars["node_id"] = node_ids[osm_id]
vars["osm_id"] = osm_id
vars["is_centroid"] = 0
geometry = "POINT({} {})".format(self.nodes[osm_id]["lon"],
self.nodes[osm_id]["lat"])
attributes = [vars.get(x) for x in fields]
attributes = ", ".join([str(x) for x in attributes])
sql = self.insert_qry.format(table, field_names, attributes,
geometry)
sql = sql.replace("None", "null")
try:
self.curr.execute(sql)
except Exception as e:
data = list(vars.values())
logger.error("error when inserting NODE {}. Error {}".format(
data, e.args))
logger.error(sql)
self.conn.commit()
self.curr.close()
self.__emit_all(["finished_threaded_procedure", 0])
def transfer_layer_features(self, table, layer, layer_fields):
self.emit_messages(message=f"Transferring features from {table} layer",
value=0,
max_val=layer.featureCount())
curr = self.project.conn.cursor()
field_titles = ", ".join(list(layer_fields.keys()))
all_modes = set()
for j, f in enumerate(layer.getFeatures()):
self.emit_messages(value=j)
attrs = []
for k, val in layer_fields.items():
if val < 0:
attrs.append("NULL")
else:
attr_val = self.convert_data(f.attributes()[val])
if not str(attr_val).isnumeric():
attrs.append(f'"{attr_val}"')
else:
attrs.append(attr_val)
attrs = ", ".join(attrs)
geom = f.geometry().asWkt().upper()
crs = str(layer.crs().authid().split(":")[1])
sql = f"INSERT INTO {table} ({field_titles} , geometry) VALUES ({attrs} , GeomFromText('{geom}', {crs}))"
if table == 'links':
all_modes.update(list(f.attributes()[layer_fields['modes']]))
try:
curr.execute(sql)
except:
logger.info(f'Failed inserting link {f.id()}')
logger.info(sql)
if f.id():
msg = f"feature with id {f.id()} could not be added to layer {table}"
else:
msg = f"feature with no node id present. It could not be added to layer {table}"
self.report.append(msg)
# We check if all modes exist
a = self.project.network.modes()
for x in all_modes:
if x not in a:
par = [
f'"automatic_{x}"', f'"{x}"',
'"Mode automatically added during project creation from layers"'
]
curr.execute(
f'INSERT INTO "modes" (mode_name, mode_id, description) VALUES({",".join(par)})'
)
logger.info(f'New mode inserted during project creation {x}')
self.project.conn.commit()
curr.close()
def __add_mode_triggers(self) -> None:
logger.info("Adding mode table triggers")
pth = os.path.dirname(os.path.realpath(__file__))
qry_file = os.path.join(pth, "database_triggers", "modes_table_triggers.sql")
self.__add_trigger_from_file(qry_file)
def execute(self):
for c in self.traffic_classes:
c.graph.set_graph(self.time_field)
logger.info(f"{self.algorithm} Assignment STATS")
logger.info("Iteration, RelativeGap, stepsize")
for self.iter in range(1, self.max_iter + 1):
self.iteration_issue = []
if pyqt:
self.equilibration.emit(["rgap", self.rgap])
self.equilibration.emit(["iterations", self.iter])
aon_flows = []
for c in self.traffic_classes:
aon = allOrNothing(c.matrix, c.graph, c._aon_results)
if pyqt:
aon.assignment.connect(self.signal_handler)
aon.execute()
aon_flows.append(c._aon_results.total_link_loads * c.pce)
self.aon_total_flow = np.sum(aon_flows, axis=0)
flows = []
if self.iter == 1:
for c in self.traffic_classes:
copy_two_dimensions(c.results.link_loads,
c._aon_results.link_loads, self.cores)
copy_one_dimension(c.results.total_link_loads,
c._aon_results.total_link_loads,
self.cores)
if c.results.num_skims > 0:
copy_three_dimensions(c.results.skims.matrix_view,
c._aon_results.skims.matrix_view,
self.cores)
flows.append(c.results.total_link_loads * c.pce)
else:
self.__calculate_step_direction()
self.calculate_stepsize()
for c in self.traffic_classes:
stp_dir = self.step_direction[c.mode]
cls_res = c.results
linear_combination(cls_res.link_loads, stp_dir.link_loads,
cls_res.link_loads, self.stepsize,
self.cores)
if cls_res.num_skims > 0:
linear_combination_skims(
cls_res.skims.matrix_view,
stp_dir.skims.matrix_view,
cls_res.skims.matrix_view,
self.stepsize,
self.cores,
)
cls_res.total_flows()
flows.append(cls_res.total_link_loads * c.pce)
self.fw_total_flow = np.sum(flows, axis=0)
# Check convergence
# This needs to be done with the current costs, and not the future ones
converged = False
if self.iter > 1:
converged = self.check_convergence()
self.convergence_report["iteration"].append(self.iter)
self.convergence_report["rgap"].append(self.rgap)
self.convergence_report["warnings"].append("; ".join(
self.iteration_issue))
self.convergence_report["alpha"].append(self.stepsize)
if self.algorithm == "bfw":
self.convergence_report["beta0"].append(self.betas[0])
self.convergence_report["beta1"].append(self.betas[1])
self.convergence_report["beta2"].append(self.betas[2])
logger.info(f"{self.iter},{self.rgap},{self.stepsize}")
if converged:
if self.steps_below >= self.steps_below_needed_to_terminate:
break
else:
self.steps_below += 1
self.vdf.apply_vdf(
self.congested_time,
self.fw_total_flow,
self.capacity,
self.free_flow_tt,
*self.vdf_parameters,
self.cores,
)
for c in self.traffic_classes:
aggregate_link_costs(self.congested_time, c.graph.compact_cost,
c.results.crosswalk)
if self.time_field in c.graph.skim_fields:
idx = c.graph.skim_fields.index(self.time_field)
c.graph.skims[:, idx] = self.congested_time[:]
c._aon_results.reset()
if self.rgap > self.rgap_target:
logger.error(f"Desired RGap of {self.rgap_target} was NOT reached")
logger.info(
f"{self.algorithm} Assignment finished. {self.iter} iterations and {self.rgap} final gap"
)
if pyqt:
self.equilibration.emit(["rgap", self.rgap])
self.equilibration.emit(["iterations", self.iter])
self.equilibration.emit(["finished_threaded_procedure"])
def execute(self):
# We build the fixed cost field
for c in self.traffic_classes:
if c.fixed_cost_field:
# divide fixed cost by volume-dependent prefactor (vot) such that we don't have to do it for
# each occurence in the objective funtion. TODO: Need to think about cost skims here, we do
# not want this there I think
c.fixed_cost[c.graph.graph.__supernet_id__] = (
c.graph.graph[c.fixed_cost_field].values[:] *
c.fc_multiplier / c.vot)
c.fixed_cost[np.isnan(c.fixed_cost)] = 0
# TODO: Review how to eliminate this. It looks unnecessary
# Just need to create some arrays for cost
for c in self.traffic_classes:
c.graph.set_graph(self.time_field)
logger.info(f"{self.algorithm} Assignment STATS")
logger.info("Iteration, RelativeGap, stepsize")
for self.iter in range(1, self.max_iter + 1):
self.iteration_issue = []
if pyqt:
self.equilibration.emit(["rgap", self.rgap])
self.equilibration.emit(["iterations", self.iter])
aon_flows = []
self.__maybe_create_path_file_directories()
for c in self.traffic_classes: # type: TrafficClass
# cost = c.fixed_cost / c.vot + self.congested_time # now only once
cost = c.fixed_cost + self.congested_time
aggregate_link_costs(cost, c.graph.compact_cost,
c.results.crosswalk)
aon = allOrNothing(c.matrix, c.graph, c._aon_results)
if pyqt:
aon.assignment.connect(self.signal_handler)
aon.execute()
c._aon_results.link_loads *= c.pce
c._aon_results.total_flows()
aon_flows.append(c._aon_results.total_link_loads)
self.aon_total_flow = np.sum(aon_flows, axis=0)
flows = []
if self.iter == 1:
for c in self.traffic_classes:
copy_two_dimensions(c.results.link_loads,
c._aon_results.link_loads, self.cores)
c.results.total_flows()
if c.results.num_skims > 0:
copy_three_dimensions(c.results.skims.matrix_view,
c._aon_results.skims.matrix_view,
self.cores)
flows.append(c.results.total_link_loads)
if self.algorithm == "all-or-nothing":
break
else:
self.__calculate_step_direction()
self.calculate_stepsize()
for c in self.traffic_classes:
stp_dir = self.step_direction[c.__id__]
cls_res = c.results
linear_combination(cls_res.link_loads, stp_dir.link_loads,
cls_res.link_loads, self.stepsize,
self.cores)
if cls_res.num_skims > 0:
linear_combination_skims(
cls_res.skims.matrix_view,
stp_dir.skims.matrix_view,
cls_res.skims.matrix_view,
self.stepsize,
self.cores,
)
cls_res.total_flows()
flows.append(cls_res.total_link_loads)
self.fw_total_flow = np.sum(flows, axis=0)
# Check convergence
# This needs to be done with the current costs, and not the future ones
converged = self.check_convergence() if self.iter > 1 else False
self.vdf.apply_vdf(
self.congested_time,
self.fw_total_flow,
self.capacity,
self.free_flow_tt,
*self.vdf_parameters,
self.cores,
)
self.convergence_report["iteration"].append(self.iter)
self.convergence_report["rgap"].append(self.rgap)
self.convergence_report["warnings"].append("; ".join(
self.iteration_issue))
self.convergence_report["alpha"].append(self.stepsize)
if self.algorithm in ["cfw", "bfw"]:
self.convergence_report["beta0"].append(self.betas[0])
self.convergence_report["beta1"].append(self.betas[1])
self.convergence_report["beta2"].append(self.betas[2])
for c in self.traffic_classes:
c._aon_results.reset()
if self.time_field not in c.graph.skim_fields:
continue
idx = c.graph.skim_fields.index(self.time_field)
c.graph.skims[:, idx] = self.congested_time[:]
logger.info(f"{self.iter},{self.rgap},{self.stepsize}")
if converged:
self.steps_below += 1
if self.steps_below >= self.steps_below_needed_to_terminate:
break
else:
self.steps_below = 0
for c in self.traffic_classes:
c.results.link_loads /= c.pce
c.results.total_flows()
# TODO (Jan 18/4/21): Do we want to blob store path files (by iteration, class, origin, destination) in sqlite?
# or do we just use one big hdf5 file?
if (self.rgap > self.rgap_target) and (self.algorithm !=
"all-or-nothing"):
logger.error(f"Desired RGap of {self.rgap_target} was NOT reached")
logger.info(
f"{self.algorithm} Assignment finished. {self.iter} iterations and {self.rgap} final gap"
)
if pyqt:
self.equilibration.emit(["rgap", self.rgap])
self.equilibration.emit(["iterations", self.iter])
self.equilibration.emit(["finished_threaded_procedure"])
def create_from_osm(
self,
west: float = None,
south: float = None,
east: float = None,
north: float = None,
place_name: str = None,
modes=["car", "transit", "bicycle", "walk"],
) -> None:
"""
Downloads the network from Open-Street Maps
Args:
*west* (:obj:`float`, Optional): West most coordinate of the download bounding box
*south* (:obj:`float`, Optional): South most coordinate of the download bounding box
*east* (:obj:`float`, Optional): East most coordinate of the download bounding box
*place_name* (:obj:`str`, Optional): If not downloading with East-West-North-South boundingbox, this is
required
*modes* (:obj:`list`, Optional): List of all modes to be downloaded. Defaults to the modes in the parameter
file
p = Project()
p.new(nm)
::
from aequilibrae import Project, Parameters
p = Project()
p.new('path/to/project')
# We now choose a different overpass endpoint (say a deployment in your local network)
par = Parameters()
par.parameters['osm']['overpass_endpoint'] = "http://192.168.1.234:5678/api"
# Because we have our own server, we can set a bigger area for download (in M2)
par.parameters['osm']['max_query_area_size'] = 10000000000
# And have no pause between successive queries
par.parameters['osm']['sleeptime'] = 0
# Save the parameters to disk
par.write_back()
# And do the import
p.network.create_from_osm(place_name=my_beautiful_hometown)
p.close()
"""
if self.count_links() > 0:
raise FileExistsError(
"You can only import an OSM network into a brand new model file"
)
curr = self.conn.cursor()
curr.execute("""ALTER TABLE links ADD COLUMN osm_id integer""")
curr.execute("""ALTER TABLE nodes ADD COLUMN osm_id integer""")
self.conn.commit()
if isinstance(modes, (tuple, list)):
modes = list(modes)
elif isinstance(modes, str):
modes = [modes]
else:
raise ValueError(
"'modes' needs to be string or list/tuple of string")
if place_name is None:
if min(east, west) < -180 or max(east, west) > 180 or min(
north, south) < -90 or max(north, south) > 90:
raise ValueError("Coordinates out of bounds")
bbox = [west, south, east, north]
else:
bbox, report = placegetter(place_name)
west, south, east, north = bbox
if bbox is None:
msg = f'We could not find a reference for place name "{place_name}"'
warn(msg)
logger.warning(msg)
return
for i in report:
if "PLACE FOUND" in i:
logger.info(i)
# Need to compute the size of the bounding box to not exceed it too much
height = haversine((east + west) / 2, south, (east + west) / 2, north)
width = haversine(east, (north + south) / 2, west, (north + south) / 2)
area = height * width
par = Parameters().parameters["osm"]
max_query_area_size = par["max_query_area_size"]
if area < max_query_area_size:
polygons = [bbox]
else:
polygons = []
parts = math.ceil(area / max_query_area_size)
horizontal = math.ceil(math.sqrt(parts))
vertical = math.ceil(parts / horizontal)
dx = (east - west) / horizontal
dy = (north - south) / vertical
for i in range(horizontal):
xmin = max(-180, west + i * dx)
xmax = min(180, west + (i + 1) * dx)
for j in range(vertical):
ymin = max(-90, south + j * dy)
ymax = min(90, south + (j + 1) * dy)
box = [xmin, ymin, xmax, ymax]
polygons.append(box)
logger.info("Downloading data")
self.downloader = OSMDownloader(polygons, modes)
self.downloader.doWork()
logger.info("Building Network")
self.builder = OSMBuilder(self.downloader.json, self.source)
self.builder.doWork()
logger.info("Network built successfully")
def create_from_osm(
self,
west: float = None,
south: float = None,
east: float = None,
north: float = None,
place_name: str = None,
modes=["car", "transit", "bicycle", "walk"],
spatial_index=False,
) -> None:
if self._check_if_exists():
raise FileExistsError("You can only import an OSM network into a brand new model file")
self.create_empty_tables()
curr = self.conn.cursor()
curr.execute("""ALTER TABLE links ADD COLUMN osm_id integer""")
curr.execute("""ALTER TABLE nodes ADD COLUMN osm_id integer""")
self.conn.commit()
if isinstance(modes, (tuple, list)):
modes = list(modes)
elif isinstance(modes, str):
modes = [modes]
else:
raise ValueError("'modes' needs to be string or list/tuple of string")
if place_name is None:
if min(east, west) < -180 or max(east, west) > 180 or min(north, south) < -90 or max(north, south) > 90:
raise ValueError("Coordinates out of bounds")
bbox = [west, south, east, north]
else:
bbox, report = placegetter(place_name)
west, south, east, north = bbox
if bbox is None:
msg = f'We could not find a reference for place name "{place_name}"'
warn(msg)
logger.warn(msg)
return
for i in report:
if "PLACE FOUND" in i:
logger.info(i)
# Need to compute the size of the bounding box to not exceed it too much
height = haversine((east + west) / 2, south, (east + west) / 2, north)
width = haversine(east, (north + south) / 2, west, (north + south) / 2)
area = height * width
if area < max_query_area_size:
polygons = [bbox]
else:
polygons = []
parts = math.ceil(area / max_query_area_size)
horizontal = math.ceil(math.sqrt(parts))
vertical = math.ceil(parts / horizontal)
dx = east - west
dy = north - south
for i in range(horizontal):
xmin = max(-180, west + i * dx)
xmax = min(180, west + (i + 1) * dx)
for j in range(vertical):
ymin = max(-90, south + j * dy)
ymax = min(90, south + (j + 1) * dy)
box = [xmin, ymin, xmax, ymax]
polygons.append(box)
logger.info("Downloading data")
self.downloader = OSMDownloader(polygons, modes)
self.downloader.doWork()
logger.info("Building Network")
self.builder = OSMBuilder(self.downloader.json, self.conn)
self.builder.doWork()
if spatial_index:
logger.info("Adding spatial indices")
self.add_spatial_index()
self.add_triggers()
logger.info("Network built successfully")
def importing_links(self, node_count):
node_ids = {}
vars = {}
vars["link_id"] = 1
table = "links"
fields = self.get_link_fields()
self.__update_table_structure()
field_names = ",".join(fields)
logger.info("Adding network links")
self.__emit_all(["text", "Adding network links"])
L = len(list(self.links.keys()))
self.__emit_all(["maxValue", L])
counter = 0
mode_codes, not_found_tags = self.modes_per_link_type()
owf, twf = self.field_osm_source()
for osm_id, link in self.links.items():
self.__emit_all(["Value", counter])
counter += 1
if counter % 1000 == 0:
logger.info(
f'Inserting segments from {counter:,} out of {L:,} OSM link objects'
)
vars["osm_id"] = osm_id
vars['link_type'] = 'default'
linknodes = link["nodes"]
linktags = link["tags"]
indices = np.searchsorted(node_count[:, 0], linknodes)
nodedegree = node_count[indices, 1]
# Makes sure that beginning and end are end nodes for a link
nodedegree[0] = 2
nodedegree[-1] = 2
intersections = np.where(nodedegree > 1)[0]
segments = intersections.shape[0] - 1
# Attributes that are common to all individual links/segments
vars["direction"] = (linktags.get("oneway") == "yes") * 1
for k, v in owf.items():
vars[k] = linktags.get(v)
for k, v in twf.items():
val = linktags.get(v["osm_source"])
if vars["direction"] == 0:
for d1, d2 in [("ab", "forward"), ("ba", "backward")]:
vars[f"{k}_{d1}"] = self.__get_link_property(
d2, val, linktags, v)
elif vars["direction"] == -1:
vars[f"{k}_ba"] = linktags.get(
f"{v['osm_source']}:{'backward'}", val)
elif vars["direction"] == 1:
vars[f"{k}_ab"] = linktags.get(
f"{v['osm_source']}:{'forward'}", val)
vars["modes"] = mode_codes.get(linktags.get("highway"),
not_found_tags)
vars['link_type'] = self.__link_type_quick_reference.get(
vars['link_type'].lower(),
self.__repair_link_type(vars['link_type']))
if len(vars["modes"]) > 0:
for i in range(segments):
attributes = self.__build_link_data(
vars, intersections, i, linknodes, node_ids, fields)
sql = self.insert_qry.format(
table, field_names,
','.join(['?'] * (len(attributes) - 1)))
try:
self.curr.execute(sql, attributes)
self.curr.execute(
'Select a_node, b_node from links where link_id=?',
[vars["link_id"]])
a, b = self.curr.fetchone()
self.curr.executemany(
'update nodes set osm_id=? where node_id=?',
[[linknodes[intersections[i]], a],
[linknodes[intersections[i + 1]], b]])
except Exception as e:
data = list(vars.values())
logger.error(
"error when inserting link {}. Error {}".format(
data, e.args))
logger.error(sql)
vars["link_id"] += 1
self.conn.commit()
self.__emit_all(
["text", f"{counter:,} of {L:,} super links added"])
self.links[osm_id] = []
self.conn.commit()
self.curr.close()
# %%
proj_matrices.list()
# %% md
## Future traffic assignment
# %%
from aequilibrae.paths import TrafficAssignment, TrafficClass
from aequilibrae import logger
# %%
logger.info("\n\n\n TRAFFIC ASSIGNMENT FOR FUTURE YEAR")
# %%
demand = proj_matrices.get_matrix("demand_power_modeled")
# let's see what is the core we ended up getting. It should be 'gravity'
demand.names
# %%
# Let's use the IPF matrix
demand.computational_view("gravity")
assig = TrafficAssignment()
def overpass_request(self,
data,
pause_duration=None,
timeout=180,
error_pause_duration=None):
"""
Send a request to the Overpass API via HTTP POST and return the JSON
response.
Parameters
----------
data : dict or OrderedDict
key-value pairs of parameters to post to the API
pause_duration : int
how long to pause in seconds before requests, if None, will query API
status endpoint to find when next slot is available
timeout : int
the timeout interval for the requests library
error_pause_duration : int
how long to pause in seconds before re-trying requests if error
Returns
-------
dict
"""
# define the Overpass API URL, then construct a GET-style URL as a string to
url = overpass_endpoint.rstrip("/") + "/interpreter"
if pause_duration is None:
time.sleep(5)
start_time = time.time()
self.report.append(
f'Posting to {url} with timeout={timeout}, "{data}"')
response = requests.post(url,
data=data,
timeout=timeout,
headers=http_headers)
# get the response size and the domain, log result
size_kb = len(response.content) / 1000.0
domain = re.findall(r"(?s)//(.*?)/", url)[0]
msg = "Downloaded {:,.1f}KB from {} in {:,.2f} seconds".format(
size_kb, domain,
time.time() - start_time)
self.report.append(msg)
logger.info(msg)
try:
response_json = response.json()
if "remark" in response_json:
msg = f'Server remark: "{response_json["remark"]}"'
self.report.append(msg)
logger.info(msg)
except Exception:
# 429 is 'too many requests' and 504 is 'gateway timeout' from server
# overload - handle these errors by recursively calling
# overpass_request until we get a valid response
if response.status_code in [429, 504]:
# pause for error_pause_duration seconds before re-trying request
if error_pause_duration is None:
error_pause_duration = 5
msg = "Server at {} returned status code {} and no JSON data. Re-trying request in {:.2f} seconds.".format(
domain, response.status_code, error_pause_duration)
self.report.append(msg)
logger.info(msg)
time.sleep(error_pause_duration)
response_json = self.overpass_request(
data=data, pause_duration=pause_duration, timeout=timeout)
# else, this was an unhandled status_code, throw an exception
else:
self.report.append(
f"Server at {domain} returned status code {response.status_code} and no JSON data"
)
raise Exception(
f"Server returned no JSON data.\n{response} {response.reason}\n{response.text}"
)
return response_json
def execute(self):
for c in self.traffic_classes:
c.graph.set_graph(self.time_field)
logger.info("{} Assignment STATS".format(self.algorithm))
logger.info("Iteration, RelativeGap, stepsize")
for self.iter in range(1, self.max_iter + 1):
flows = []
aon_flows = []
for c in self.traffic_classes:
aon = allOrNothing(c.matrix, c.graph, c._aon_results)
aon.execute()
c._aon_results.total_flows()
aon_flows.append(c._aon_results.total_link_loads * c.pce)
self.aon_total_flow = np.sum(aon_flows, axis=0)
if self.iter == 1:
for c in self.traffic_classes:
copy_two_dimensions(c.results.link_loads, c._aon_results.link_loads, self.cores)
c.results.total_flows()
copy_one_dimension(c.results.total_link_loads, c._aon_results.total_link_loads, self.cores)
if c.results.num_skims > 0:
copy_three_dimensions(c.results.skims.matrix_view, c._aon_results.skims.matrix_view, self.cores)
flows.append(c.results.total_link_loads * c.pce)
else:
self.__calculate_step_direction()
self.calculate_stepsize()
for c in self.traffic_classes:
stp_dir = self.step_direction[c.mode]
cls_res = c.results
linear_combination(cls_res.link_loads, stp_dir.link_loads, cls_res.link_loads, self.stepsize,
self.cores)
# TODO: We need to compute the step direction for skims as well.
# It is probably a matter of transforming the step_direction values from numpy arrays to
# full AssignmentResults() ones, and cleaning the stuff we don't need
if cls_res.num_skims > 0:
linear_combination_skims(cls_res.skims.matrix_view,
stp_dir.skims.matrix_view,
cls_res.skims.matrix_view,
self.stepsize,
self.cores)
cls_res.total_flows()
flows.append(cls_res.total_link_loads * c.pce)
self.fw_total_flow = np.sum(flows, axis=0)
# Check convergence
# This needs ot be done with the current costs, and not the future ones
if self.iter > 1:
if self.check_convergence():
if self.steps_below >= self.steps_below_needed_to_terminate:
break
else:
self.steps_below += 1
self.vdf.apply_vdf(
self.congested_time, self.fw_total_flow, self.capacity, self.free_flow_tt, *self.vdf_parameters
)
for c in self.traffic_classes:
c.graph.cost = self.congested_time
c._aon_results.reset()
logger.info("{},{},{}".format(self.iter, self.rgap, self.stepsize))
if self.rgap > self.rgap_target:
logger.error("Desired RGap of {} was NOT reached".format(self.rgap_target))
logger.info(
"{} Assignment finished. {} iterations and {} final gap".format(self.algorithm, self.iter, self.rgap)
)
def execute(self):
for c in self.traffic_classes:
c.graph.set_graph(self.time_field)
logger.info("{} Assignment STATS".format(self.algorithm))
logger.info("Iteration, RelativeGap, stepsize")
for self.iter in range(1, self.max_iter + 1):
if pyqt:
self.equilibration.emit(['rgap', self.rgap])
self.equilibration.emit(['iterations', self.iter])
flows = []
aon_flows = []
for c in self.traffic_classes:
aon = allOrNothing(c.matrix, c.graph, c._aon_results)
if pyqt:
aon.assignment.connect(self.signal_handler)
aon.execute()
c._aon_results.total_flows()
aon_flows.append(c._aon_results.total_link_loads * c.pce)
self.aon_total_flow = np.sum(aon_flows, axis=0)
if self.iter == 1:
for c in self.traffic_classes:
copy_two_dimensions(c.results.link_loads,
c._aon_results.link_loads, self.cores)
c.results.total_flows()
copy_one_dimension(c.results.total_link_loads,
c._aon_results.total_link_loads,
self.cores)
if c.results.num_skims > 0:
copy_three_dimensions(c.results.skims.matrix_view,
c._aon_results.skims.matrix_view,
self.cores)
flows.append(c.results.total_link_loads * c.pce)
else:
self.__calculate_step_direction()
self.calculate_stepsize()
for c in self.traffic_classes:
stp_dir = self.step_direction[c.mode]
cls_res = c.results
linear_combination(cls_res.link_loads, stp_dir.link_loads,
cls_res.link_loads, self.stepsize,
self.cores)
if cls_res.num_skims > 0:
linear_combination_skims(cls_res.skims.matrix_view,
stp_dir.skims.matrix_view,
cls_res.skims.matrix_view,
self.stepsize, self.cores)
cls_res.total_flows()
flows.append(cls_res.total_link_loads * c.pce)
self.fw_total_flow = np.sum(flows, axis=0)
# Check convergence
# This needs ot be done with the current costs, and not the future ones
if self.iter > 1:
if self.check_convergence():
if self.steps_below >= self.steps_below_needed_to_terminate:
break
else:
self.steps_below += 1
self.vdf.apply_vdf(self.congested_time, self.fw_total_flow,
self.capacity, self.free_flow_tt,
*self.vdf_parameters)
for c in self.traffic_classes:
c.graph.cost = self.congested_time
if self.time_field in c.graph.skim_fields:
idx = c.graph.skim_fields.index(self.time_field)
c.graph.skims[:, idx] = self.congested_time[:]
c._aon_results.reset()
logger.info("{},{},{}".format(self.iter, self.rgap, self.stepsize))
if self.rgap > self.rgap_target:
logger.error("Desired RGap of {} was NOT reached".format(
self.rgap_target))
logger.info(
f"{self.algorithm} Assignment finished. {self.iter} iterations and {self.rgap} final gap"
)
if pyqt:
self.equilibration.emit(['rgap', self.rgap])
self.equilibration.emit(['iterations', self.iter])
self.equilibration.emit(['finished_threaded_procedure'])
def importing_links(self, node_count):
node_ids = {}
vars = {}
vars["link_id"] = 1
table = "links"
fields = self.get_link_fields()
field_names = ",".join(fields)
fn = ",".join(['"{}"'.format(x) for x in field_names.split(",")])
logger.info("Adding network links")
self.__emit_all(["text", "Adding network links"])
L = len(list(self.links.keys()))
self.__emit_all(["maxValue", L])
nodes_to_add = set()
counter = 0
mode_codes, not_found_tags = self.modes_per_link_type()
for osm_id, link in self.links.items():
self.__emit_all(["Value", counter])
counter += 1
vars["osm_id"] = osm_id
linknodes = link["nodes"]
linktags = link["tags"]
indices = np.searchsorted(node_count[:, 0], linknodes)
nodedegree = node_count[indices, 1]
# Makes sure that beginning and end are end nodes for a link
nodedegree[0] = 2
nodedegree[-1] = 2
intersections = np.where(nodedegree > 1)[0]
segments = intersections.shape[0] - 1
owf, twf = self.field_osm_source()
# Attributes that are common to all individual links/segments
vars["direction"] = (linktags.get("oneway") == "yes") * 1
for k, v in owf.items():
attr_value = linktags.get(v)
if isinstance(attr_value, str):
attr_value = attr_value.replace('"', "'")
attr_value = '"{}"'.format(attr_value)
vars[k] = attr_value
for k, v in twf.items():
val = linktags.get(v["osm_source"])
for d1, d2 in [("ab", "forward"), ("ba", "backward")]:
vars["{}_{}".format(k, d1)] = self.__get_link_property(
d2, val, linktags, v)
vars["modes"] = mode_codes.get(linktags.get("highway"),
not_found_tags)
if len(vars["modes"]) > 0:
for i in range(segments):
geometry, attributes = self.__build_link_data(
vars, intersections, i, linknodes, node_ids, fields)
sql = self.insert_qry.format(table, fn, attributes,
geometry)
sql = sql.replace("None", "null")
try:
self.curr.execute(sql)
nodes_to_add.update([
linknodes[intersections[i]],
linknodes[intersections[i + 1]]
])
except Exception as e:
data = list(vars.values())
logger.error(
"error when inserting link {}. Error {}".format(
data, e.args))
logger.error(sql)
vars["link_id"] += 1
self.__emit_all(
["text", f"{counter:,} of {L:,} super links added"])
return nodes_to_add, node_ids
