def get_osm_filter(self, modes: list) -> str:
"""
loosely adapted from http://www.github.com/gboeing/osmnx
"""
p = Parameters().parameters["network"]["osm"]
all_tags = p["all_link_types"]
p = p["modes"]
all_modes = list(p.keys())
tags_to_keep = []
for m in modes:
if m not in all_modes:
raise ValueError(f"Mode {m} not listed in the parameters file")
tags_to_keep += p[m]["link_types"]
tags_to_keep = list(set(tags_to_keep))
# Default to remove
service = '["service"!~"parking|parking_aisle|driveway|private|emergency_access"]'
access = '["access"!~"private"]'
filtered = [x for x in all_tags if x not in tags_to_keep]
filtered = "|".join(filtered)
filter = f'["area"!~"yes"]["highway"!~"{filtered}"]{service}{access}'
return filter
def __init__(self):
self.compact_link_loads = np.array([]) # Results for assignment on simplified graph
self.compact_total_link_loads = np.array([]) # Results for all user classes summed on simplified graph
self.link_loads = np.array([]) # The actual results for assignment
self.total_link_loads = np.array([]) # The result of the assignment for all user classes summed
self.crosswalk = np.array([]) # crosswalk between compact graph link IDs and actual link IDs
self.skims = AequilibraeMatrix() # The array of skims
self.no_path = None # The list os paths
self.num_skims = 0 # number of skims that will be computed. Depends on the setting of the graph provided
p = Parameters().parameters["system"]["cpus"]
if not isinstance(p, int):
p = 0
self.set_cores(p)
self.classes = {"number": 1, "names": ["flow"]}
self.nodes = -1
self.zones = -1
self.links = -1
self.compact_links = -1
self.compact_nodes = -1
self.__graph_id__ = None
self.__float_type = None
self.__integer_type = None
self.lids = None
self.direcs = None
def modes_per_link_type(self):
p = Parameters()
modes = p.parameters["network"]["osm"]["modes"]
cursor = self.conn.cursor()
cursor.execute("SELECT mode_name, mode_id from modes")
mode_codes = cursor.fetchall()
mode_codes = {p[0]: p[1] for p in mode_codes}
type_list = {}
notfound = ""
for mode, val in modes.items():
all_types = val["link_types"]
md = mode_codes[mode]
for tp in all_types:
type_list[tp] = "{}{}".format(type_list.get(tp, ""), md)
if val["unknown_tags"]:
notfound += md
type_list = {
k: '"{}"'.format("".join(set(v)))
for k, v in type_list.items()
}
return type_list, '"{}"'.format(notfound)
def setUp(self) -> None:
self.file = os.path.join(gettempdir(),
"aequilibrae_project_test.sqlite")
self.project = Project(self.file, True)
self.file2 = os.path.join(gettempdir(),
"aequilibrae_project_test2.sqlite")
self.conn = sqlite3.connect(self.file2)
self.conn.enable_load_extension(True)
plat = platform.platform()
pth = os.getcwd()
if "WINDOWS" in plat.upper():
par = Parameters()
spatialite_path = par.parameters["system"]["spatialite_path"]
if os.path.isfile(
os.path.join(spatialite_path, "mod_spatialite.dll")):
os.chdir(spatialite_path)
try:
self.conn.load_extension("mod_spatialite")
except Exception as e:
warn(
f"AequilibraE might not work as intended without spatialite. {e.args}"
)
os.chdir(pth)
self.network = Network(self)
def test_create_empty_tables(self):
self.network.create_empty_tables()
p = Parameters().parameters["network"]
curr = self.conn.cursor()
curr.execute("""PRAGMA table_info(links);""")
fields = curr.fetchall()
fields = [x[1] for x in fields]
oneway = reduce(lambda a, b: dict(a, **b),
p["links"]["fields"]["one-way"])
owf = list(oneway.keys())
twoway = reduce(lambda a, b: dict(a, **b),
p["links"]["fields"]["two-way"])
twf = []
for k in list(twoway.keys()):
twf.extend([f"{k}_ab", f"{k}_ba"])
for f in owf + twf:
if f not in fields:
self.fail(f"Field {f} not added to links table")
curr = self.conn.cursor()
curr.execute("""PRAGMA table_info(nodes);""")
nfields = curr.fetchall()
nfields = [x[1] for x in nfields]
flds = reduce(lambda a, b: dict(a, **b), p["nodes"]["fields"])
flds = list(flds.keys())
for f in flds:
if f not in nfields:
self.fail(f"Field {f} not added to nodes table")
def create_empty_tables(self) -> None:
"""Creates empty network tables for future filling"""
curr = self.conn.cursor()
# Create the links table
p = Parameters()
fields = p.parameters["network"]["links"]["fields"]
sql = """CREATE TABLE 'links' (
ogc_fid INTEGER PRIMARY KEY,
link_id INTEGER UNIQUE,
a_node INTEGER,
b_node INTEGER,
direction INTEGER NOT NULL DEFAULT 0,
distance NUMERIC,
modes TEXT NOT NULL,
link_type TEXT NOT NULL DEFAULT 'link type not defined'
{});"""
flds = fields["one-way"]
# returns first key in the dictionary
def fkey(f):
return list(f.keys())[0]
owlf = ["{} {}".format(fkey(f), f[fkey(f)]["type"]) for f in flds if fkey(f).lower() not in self.req_link_flds]
flds = fields["two-way"]
twlf = []
for f in flds:
nm = fkey(f)
tp = f[nm]["type"]
twlf.extend([f"{nm}_ab {tp}", f"{nm}_ba {tp}"])
link_fields = owlf + twlf
if link_fields:
sql = sql.format("," + ",".join(link_fields))
else:
sql = sql.format("")
curr.execute(sql)
sql = """CREATE TABLE 'nodes' (ogc_fid INTEGER PRIMARY KEY,
node_id INTEGER UNIQUE NOT NULL,
is_centroid INTEGER NOT NULL DEFAULT 0 {});"""
flds = p.parameters["network"]["nodes"]["fields"]
ndflds = [f"{fkey(f)} {f[fkey(f)]['type']}" for f in flds if fkey(f).lower() not in self.req_node_flds]
if ndflds:
sql = sql.format("," + ",".join(ndflds))
else:
sql = sql.format("")
curr.execute(sql)
curr.execute("""SELECT AddGeometryColumn( 'links', 'geometry', 4326, 'LINESTRING', 'XY' )""")
curr.execute("""SELECT AddGeometryColumn( 'nodes', 'geometry', 4326, 'POINT', 'XY' )""")
self.conn.commit()
def get_link_fields():
p = Parameters()
fields = p.parameters["network"]["links"]["fields"]
owf = [list(x.keys())[0] for x in fields["one-way"]]
twf1 = ["{}_ab".format(list(x.keys())[0]) for x in fields["two-way"]]
twf2 = ["{}_ba".format(list(x.keys())[0]) for x in fields["two-way"]]
return owf + twf1 + twf2 + ["osm_id"]
def __init__(self, polygons, modes):
WorkerThread.__init__(self, None)
self.polygons = polygons
self.filter = self.get_osm_filter(modes)
self.report = []
self.json = []
par = Parameters().parameters['osm']
self.overpass_endpoint = par['overpass_endpoint']
self.timeout = par['timeout']
self.sleeptime = par['sleeptime']
def __create_empty_project(self):
# We create the project folder and create the base file
os.mkdir(self.project_base_path)
shutil.copyfile(spatialite_database, self.path_to_file)
self.conn = database_connection()
# Write parameters to the project folder
p = Parameters()
p.parameters["system"]["logging_directory"] = self.project_base_path
p.write_back()
_ = StartsLogging()
# Create actual tables
cursor = self.conn.cursor()
cursor.execute("PRAGMA foreign_keys = ON;")
self.conn.commit()
initialize_tables(self.conn)
def spatialite_connection(conn):
conn.enable_load_extension(True)
par = Parameters()
spatialite_path = par.parameters["system"]["spatialite_path"]
if spatialite_path not in os.environ['PATH']:
os.environ['PATH'] = spatialite_path + ';' + os.environ['PATH']
try:
conn.load_extension("mod_spatialite")
except Exception as e:
logger.warning(f"AequilibraE might not work as intended without spatialite. {e.args}")
return conn
def get_spatialite_connection(self):
self.conn.enable_load_extension(True)
plat = platform.platform()
pth = os.getcwd()
if "WINDOWS" in plat.upper():
par = Parameters()
spatialite_path = par.parameters["system"]["spatialite_path"]
if os.path.isfile(os.path.join(spatialite_path, "mod_spatialite.dll")):
os.chdir(spatialite_path)
try:
self.conn.load_extension("mod_spatialite")
except Exception as e:
warn(f"AequilibraE might not work as intended without spatialite. {e.args}")
os.chdir(pth)
def get_link_field_type(field_name):
p = Parameters()
fields = p.parameters["network"]["links"]["fields"]
if field_name[-3:].lower() in ['_ab', '_ba']:
field_name = field_name[:-3]
for tp in fields["two-way"]:
if field_name in tp:
return tp[field_name]['type']
else:
for tp in fields["one-way"]:
if field_name in tp:
return tp[field_name]['type']
def __init__(self, path_to_file: str, new_project=False):
self.path_to_file = path_to_file
self.parameters = Parameters().parameters
if not os.path.isfile(path_to_file):
if not new_project:
raise FileNotFoundError(
"Model does not exist. Check your path or use the new_project=True flag to create a new project"
)
else:
self.__create_empty_project()
else:
self.conn = sqlite3.connect(self.path_to_file)
self.source = self.path_to_file
self.get_spatialite_connection()
self.network = Network(self)
def __init__(self, link_id=None):
p = Parameters()
fields = p.parameters["network"]["links"]["fields"]
one_way_fields = [list(x.keys())[0] for x in fields["one-way"]]
one_way_fields = {x: None for x in one_way_fields}
twf = [list(x.keys())[0] for x in fields["two-way"]]
two_way_fields = {f"{x}_ab": None for x in twf}
two_way_fields.update({f"{x}_ba": None for x in twf})
one_way_fields.update(two_way_fields)
self.__dict__.update(one_way_fields)
if link_id is not None:
self.link_id = link_id
self._populate()
def field_osm_source():
p = Parameters()
fields = p.parameters["network"]["links"]["fields"]
owf = {
list(x.keys())[0]: x[list(x.keys())[0]]["osm_source"]
for x in fields["one-way"] if "osm_source" in x[list(x.keys())[0]]
}
twf = {}
for x in fields["two-way"]:
if "osm_source" in x[list(x.keys())[0]]:
twf[list(x.keys())[0]] = {
"osm_source": x[list(x.keys())[0]]["osm_source"],
"osm_behaviour": x[list(x.keys())[0]]["osm_behaviour"],
}
return owf, twf
def new(self, file_name: str) -> None:
"""Creates a new project
Args:
*file_name* (:obj:`str`): Full path to the project data file. If file exists, it will fail
"""
self.path_to_file = file_name
self.source = self.path_to_file
self.parameters = Parameters().parameters
if os.path.isfile(file_name):
raise FileNotFoundError(
"File already exist. Choose a different name or remove the existing file"
)
self.__create_empty_project()
self.conn = spatialite_connection(self.conn)
self.network = Network(self)
def __set_logging_path(self):
p = Parameters()
par = p.parameters
if p.parameters is None:
par = p._default
do_log = par["system"]["logging"]
for handler in logger.handlers:
if handler.name == "aequilibrae":
logger.removeHandler(handler)
if do_log:
formatter = logging.Formatter("%(asctime)s;%(name)s;%(levelname)s ; %(message)s")
log_file = os.path.join(self.project_base_path, "aequilibrae.log")
if not os.path.isfile(log_file):
a = open(log_file, "w")
a.close()
ch = logging.FileHandler(log_file)
ch.name = "aequilibrae"
ch.setFormatter(formatter)
ch.setLevel(logging.DEBUG)
logger.addHandler(ch)
def __init__(self):
self.link_loads = None # type: np.array # The actual results for assignment
self.total_link_loads = None # type: np.array # The result of the assignment for all user classes summed
self.skims = None # The array of skims
self.no_path = None # The list os paths
self.num_skims = None # number of skims that will be computed. Depends on the setting of the graph provided
p = Parameters().parameters['system']['cpus']
if not isinstance(p, int):
p = 0
self.set_cores(p)
self.classes = {"number": 1, "names": ["flow"]}
self.critical_links = {
"save": False,
"queries": {},
"results": False
} # Queries are a dictionary
self.link_extraction = {
"save": False,
"queries": {},
"output": None
} # Queries are a dictionary
self.path_file = {"save": False, "results": None}
self.nodes = -1
self.zones = -1
self.links = -1
self.__graph_id__ = None
self.__float_type = None
self.__integer_type = None
self.lids = None
self.direcs = None
def create_from_osm(
self,
west: float = None,
south: float = None,
east: float = None,
north: float = None,
place_name: str = None,
modes=["car", "transit", "bicycle", "walk"],
) -> None:
"""
Downloads the network from Open-Street Maps
Args:
*west* (:obj:`float`, Optional): West most coordinate of the download bounding box
*south* (:obj:`float`, Optional): South most coordinate of the download bounding box
*east* (:obj:`float`, Optional): East most coordinate of the download bounding box
*place_name* (:obj:`str`, Optional): If not downloading with East-West-North-South boundingbox, this is
required
*modes* (:obj:`list`, Optional): List of all modes to be downloaded. Defaults to the modes in the parameter
file
p = Project()
p.new(nm)
::
from aequilibrae import Project, Parameters
p = Project()
p.new('path/to/project')
# We now choose a different overpass endpoint (say a deployment in your local network)
par = Parameters()
par.parameters['osm']['overpass_endpoint'] = "http://192.168.1.234:5678/api"
# Because we have our own server, we can set a bigger area for download (in M2)
par.parameters['osm']['max_query_area_size'] = 10000000000
# And have no pause between successive queries
par.parameters['osm']['sleeptime'] = 0
# Save the parameters to disk
par.write_back()
# And do the import
p.network.create_from_osm(place_name=my_beautiful_hometown)
p.close()
"""
if self.count_links() > 0:
raise FileExistsError(
"You can only import an OSM network into a brand new model file"
)
curr = self.conn.cursor()
curr.execute("""ALTER TABLE links ADD COLUMN osm_id integer""")
curr.execute("""ALTER TABLE nodes ADD COLUMN osm_id integer""")
self.conn.commit()
if isinstance(modes, (tuple, list)):
modes = list(modes)
elif isinstance(modes, str):
modes = [modes]
else:
raise ValueError(
"'modes' needs to be string or list/tuple of string")
if place_name is None:
if min(east, west) < -180 or max(east, west) > 180 or min(
north, south) < -90 or max(north, south) > 90:
raise ValueError("Coordinates out of bounds")
bbox = [west, south, east, north]
else:
bbox, report = placegetter(place_name)
west, south, east, north = bbox
if bbox is None:
msg = f'We could not find a reference for place name "{place_name}"'
warn(msg)
logger.warning(msg)
return
for i in report:
if "PLACE FOUND" in i:
logger.info(i)
# Need to compute the size of the bounding box to not exceed it too much
height = haversine((east + west) / 2, south, (east + west) / 2, north)
width = haversine(east, (north + south) / 2, west, (north + south) / 2)
area = height * width
par = Parameters().parameters["osm"]
max_query_area_size = par["max_query_area_size"]
if area < max_query_area_size:
polygons = [bbox]
else:
polygons = []
parts = math.ceil(area / max_query_area_size)
horizontal = math.ceil(math.sqrt(parts))
vertical = math.ceil(parts / horizontal)
dx = (east - west) / horizontal
dy = (north - south) / vertical
for i in range(horizontal):
xmin = max(-180, west + i * dx)
xmax = min(180, west + (i + 1) * dx)
for j in range(vertical):
ymin = max(-90, south + j * dy)
ymax = min(90, south + (j + 1) * dy)
box = [xmin, ymin, xmax, ymax]
polygons.append(box)
logger.info("Downloading data")
self.downloader = OSMDownloader(polygons, modes)
self.downloader.doWork()
logger.info("Building Network")
self.builder = OSMBuilder(self.downloader.json, self.source)
self.builder.doWork()
logger.info("Network built successfully")
def placegetter(place: str) -> Tuple[Union[None, List[float]], list]:
"""
Send a request to the Nominatim API via HTTP GET and return a geometry polygon
for the region we are querying
Parameters
----------
place : str
Name of the place we want to download a network for
Adapted from http://www.github.com/gboeing/osmnx
"""
par = Parameters().parameters['osm']
nominatim_endpoint = par['nominatim_endpoint']
max_attempts = par['max_attempts']
params = {"q": place, "format": "json"}
report = []
pause_duration = 1
timeout = 30
error_pause_duration = 180
# prepare the Nominatim API URL
url = nominatim_endpoint.rstrip("/") + "/search"
prepared_url = requests.Request("GET", url, params=params).prepare().url
# Pause, then request it
report.append(
"Pausing {:,.2f} seconds before making API GET request".format(
pause_duration))
time.sleep(pause_duration)
start_time = time.time()
report.append(f"Requesting {prepared_url} with timeout={timeout}")
response = requests.get(url,
params=params,
timeout=timeout,
headers=http_headers)
# get the response size and the domain, log result
size_kb = len(response.content) / 1000.0
domain = re.findall(r"(?s)//(.*?)/", url)[0]
report.append("Downloaded {:,.1f}KB from {} in {:,.2f} seconds".format(
size_kb, domain,
time.time() - start_time))
bbox = None
for attempts in range(max_attempts):
report.append(f"Attempt: {attempts}")
if response.status_code != 200:
report.append(
"Server at {} returned status code {} and no JSON data. Re-trying request in {:.2f} seconds."
.format(domain, response.status_code, error_pause_duration))
if response.status_code in [429, 504]:
# SEND MESSAGE
time.sleep(error_pause_duration)
continue
elif response.status_code == 200:
response_json = response.json()
report.append("COMPLETE QUERY RESPONSE FOR PLACE:")
report.append(str(response_json))
if len(response_json):
bbox = [float(x) for x in response_json[0]["boundingbox"]]
bbox = [bbox[2], bbox[0], bbox[3], bbox[1]]
report.append(
f"PLACE FOUND:{response_json[0]['display_name']}")
return (bbox, report)
else:
bbox = None
if attempts == max_attempts - 1 and bbox is None:
report.append(
"Reached maximum download attempts. Please wait a few minutes and try again"
)
else:
report.append("We got an error for place query.")
return (bbox, report)
def parameters(self) -> dict:
return Parameters().parameters
import requests
from aequilibrae.parameters import Parameters
par = Parameters().parameters['osm']
accept_language = par['accept_language']
memory = 0
user_agent = "AequilibraE (https://github.com/aequilibrae/aequilibrae-GUI)"
referer = "AequilibraE (https://github.com/aequilibrae/aequilibrae-GUI)"
http_headers = requests.utils.default_headers()
http_headers.update({
"User-Agent": user_agent,
"referer": referer,
"Accept-Language": accept_language,
"format": "json"
})
def get_node_fields():
p = Parameters()
fields = p.parameters["network"]["nodes"]["fields"]
fields = [list(x.keys())[0] for x in fields]
return fields + ["osm_id"]
def __init__(self):
self.path_to_file: str = None
self.source: str = None
self.parameters = Parameters().parameters
self.conn: sqlite3.Connection = None
self.network: Network = None
