def get_osm_filter(self, modes: list) -> str: """ loosely adapted from http://www.github.com/gboeing/osmnx """ p = Parameters().parameters["network"]["osm"] all_tags = p["all_link_types"] p = p["modes"] all_modes = list(p.keys()) tags_to_keep = [] for m in modes: if m not in all_modes: raise ValueError(f"Mode {m} not listed in the parameters file") tags_to_keep += p[m]["link_types"] tags_to_keep = list(set(tags_to_keep)) # Default to remove service = '["service"!~"parking|parking_aisle|driveway|private|emergency_access"]' access = '["access"!~"private"]' filtered = [x for x in all_tags if x not in tags_to_keep] filtered = "|".join(filtered) filter = f'["area"!~"yes"]["highway"!~"{filtered}"]{service}{access}' return filter
def __init__(self): self.compact_link_loads = np.array([]) # Results for assignment on simplified graph self.compact_total_link_loads = np.array([]) # Results for all user classes summed on simplified graph self.link_loads = np.array([]) # The actual results for assignment self.total_link_loads = np.array([]) # The result of the assignment for all user classes summed self.crosswalk = np.array([]) # crosswalk between compact graph link IDs and actual link IDs self.skims = AequilibraeMatrix() # The array of skims self.no_path = None # The list os paths self.num_skims = 0 # number of skims that will be computed. Depends on the setting of the graph provided p = Parameters().parameters["system"]["cpus"] if not isinstance(p, int): p = 0 self.set_cores(p) self.classes = {"number": 1, "names": ["flow"]} self.nodes = -1 self.zones = -1 self.links = -1 self.compact_links = -1 self.compact_nodes = -1 self.__graph_id__ = None self.__float_type = None self.__integer_type = None self.lids = None self.direcs = None
def modes_per_link_type(self): p = Parameters() modes = p.parameters["network"]["osm"]["modes"] cursor = self.conn.cursor() cursor.execute("SELECT mode_name, mode_id from modes") mode_codes = cursor.fetchall() mode_codes = {p[0]: p[1] for p in mode_codes} type_list = {} notfound = "" for mode, val in modes.items(): all_types = val["link_types"] md = mode_codes[mode] for tp in all_types: type_list[tp] = "{}{}".format(type_list.get(tp, ""), md) if val["unknown_tags"]: notfound += md type_list = { k: '"{}"'.format("".join(set(v))) for k, v in type_list.items() } return type_list, '"{}"'.format(notfound)
def setUp(self) -> None: self.file = os.path.join(gettempdir(), "aequilibrae_project_test.sqlite") self.project = Project(self.file, True) self.file2 = os.path.join(gettempdir(), "aequilibrae_project_test2.sqlite") self.conn = sqlite3.connect(self.file2) self.conn.enable_load_extension(True) plat = platform.platform() pth = os.getcwd() if "WINDOWS" in plat.upper(): par = Parameters() spatialite_path = par.parameters["system"]["spatialite_path"] if os.path.isfile( os.path.join(spatialite_path, "mod_spatialite.dll")): os.chdir(spatialite_path) try: self.conn.load_extension("mod_spatialite") except Exception as e: warn( f"AequilibraE might not work as intended without spatialite. {e.args}" ) os.chdir(pth) self.network = Network(self)
def test_create_empty_tables(self): self.network.create_empty_tables() p = Parameters().parameters["network"] curr = self.conn.cursor() curr.execute("""PRAGMA table_info(links);""") fields = curr.fetchall() fields = [x[1] for x in fields] oneway = reduce(lambda a, b: dict(a, **b), p["links"]["fields"]["one-way"]) owf = list(oneway.keys()) twoway = reduce(lambda a, b: dict(a, **b), p["links"]["fields"]["two-way"]) twf = [] for k in list(twoway.keys()): twf.extend([f"{k}_ab", f"{k}_ba"]) for f in owf + twf: if f not in fields: self.fail(f"Field {f} not added to links table") curr = self.conn.cursor() curr.execute("""PRAGMA table_info(nodes);""") nfields = curr.fetchall() nfields = [x[1] for x in nfields] flds = reduce(lambda a, b: dict(a, **b), p["nodes"]["fields"]) flds = list(flds.keys()) for f in flds: if f not in nfields: self.fail(f"Field {f} not added to nodes table")
def create_empty_tables(self) -> None: """Creates empty network tables for future filling""" curr = self.conn.cursor() # Create the links table p = Parameters() fields = p.parameters["network"]["links"]["fields"] sql = """CREATE TABLE 'links' ( ogc_fid INTEGER PRIMARY KEY, link_id INTEGER UNIQUE, a_node INTEGER, b_node INTEGER, direction INTEGER NOT NULL DEFAULT 0, distance NUMERIC, modes TEXT NOT NULL, link_type TEXT NOT NULL DEFAULT 'link type not defined' {});""" flds = fields["one-way"] # returns first key in the dictionary def fkey(f): return list(f.keys())[0] owlf = ["{} {}".format(fkey(f), f[fkey(f)]["type"]) for f in flds if fkey(f).lower() not in self.req_link_flds] flds = fields["two-way"] twlf = [] for f in flds: nm = fkey(f) tp = f[nm]["type"] twlf.extend([f"{nm}_ab {tp}", f"{nm}_ba {tp}"]) link_fields = owlf + twlf if link_fields: sql = sql.format("," + ",".join(link_fields)) else: sql = sql.format("") curr.execute(sql) sql = """CREATE TABLE 'nodes' (ogc_fid INTEGER PRIMARY KEY, node_id INTEGER UNIQUE NOT NULL, is_centroid INTEGER NOT NULL DEFAULT 0 {});""" flds = p.parameters["network"]["nodes"]["fields"] ndflds = [f"{fkey(f)} {f[fkey(f)]['type']}" for f in flds if fkey(f).lower() not in self.req_node_flds] if ndflds: sql = sql.format("," + ",".join(ndflds)) else: sql = sql.format("") curr.execute(sql) curr.execute("""SELECT AddGeometryColumn( 'links', 'geometry', 4326, 'LINESTRING', 'XY' )""") curr.execute("""SELECT AddGeometryColumn( 'nodes', 'geometry', 4326, 'POINT', 'XY' )""") self.conn.commit()
def get_link_fields(): p = Parameters() fields = p.parameters["network"]["links"]["fields"] owf = [list(x.keys())[0] for x in fields["one-way"]] twf1 = ["{}_ab".format(list(x.keys())[0]) for x in fields["two-way"]] twf2 = ["{}_ba".format(list(x.keys())[0]) for x in fields["two-way"]] return owf + twf1 + twf2 + ["osm_id"]
def __init__(self, polygons, modes): WorkerThread.__init__(self, None) self.polygons = polygons self.filter = self.get_osm_filter(modes) self.report = [] self.json = [] par = Parameters().parameters['osm'] self.overpass_endpoint = par['overpass_endpoint'] self.timeout = par['timeout'] self.sleeptime = par['sleeptime']
def __create_empty_project(self): # We create the project folder and create the base file os.mkdir(self.project_base_path) shutil.copyfile(spatialite_database, self.path_to_file) self.conn = database_connection() # Write parameters to the project folder p = Parameters() p.parameters["system"]["logging_directory"] = self.project_base_path p.write_back() _ = StartsLogging() # Create actual tables cursor = self.conn.cursor() cursor.execute("PRAGMA foreign_keys = ON;") self.conn.commit() initialize_tables(self.conn)
def spatialite_connection(conn): conn.enable_load_extension(True) par = Parameters() spatialite_path = par.parameters["system"]["spatialite_path"] if spatialite_path not in os.environ['PATH']: os.environ['PATH'] = spatialite_path + ';' + os.environ['PATH'] try: conn.load_extension("mod_spatialite") except Exception as e: logger.warning(f"AequilibraE might not work as intended without spatialite. {e.args}") return conn
def get_spatialite_connection(self): self.conn.enable_load_extension(True) plat = platform.platform() pth = os.getcwd() if "WINDOWS" in plat.upper(): par = Parameters() spatialite_path = par.parameters["system"]["spatialite_path"] if os.path.isfile(os.path.join(spatialite_path, "mod_spatialite.dll")): os.chdir(spatialite_path) try: self.conn.load_extension("mod_spatialite") except Exception as e: warn(f"AequilibraE might not work as intended without spatialite. {e.args}") os.chdir(pth)
def get_link_field_type(field_name): p = Parameters() fields = p.parameters["network"]["links"]["fields"] if field_name[-3:].lower() in ['_ab', '_ba']: field_name = field_name[:-3] for tp in fields["two-way"]: if field_name in tp: return tp[field_name]['type'] else: for tp in fields["one-way"]: if field_name in tp: return tp[field_name]['type']
def __init__(self, path_to_file: str, new_project=False): self.path_to_file = path_to_file self.parameters = Parameters().parameters if not os.path.isfile(path_to_file): if not new_project: raise FileNotFoundError( "Model does not exist. Check your path or use the new_project=True flag to create a new project" ) else: self.__create_empty_project() else: self.conn = sqlite3.connect(self.path_to_file) self.source = self.path_to_file self.get_spatialite_connection() self.network = Network(self)
def __init__(self, link_id=None): p = Parameters() fields = p.parameters["network"]["links"]["fields"] one_way_fields = [list(x.keys())[0] for x in fields["one-way"]] one_way_fields = {x: None for x in one_way_fields} twf = [list(x.keys())[0] for x in fields["two-way"]] two_way_fields = {f"{x}_ab": None for x in twf} two_way_fields.update({f"{x}_ba": None for x in twf}) one_way_fields.update(two_way_fields) self.__dict__.update(one_way_fields) if link_id is not None: self.link_id = link_id self._populate()
def field_osm_source(): p = Parameters() fields = p.parameters["network"]["links"]["fields"] owf = { list(x.keys())[0]: x[list(x.keys())[0]]["osm_source"] for x in fields["one-way"] if "osm_source" in x[list(x.keys())[0]] } twf = {} for x in fields["two-way"]: if "osm_source" in x[list(x.keys())[0]]: twf[list(x.keys())[0]] = { "osm_source": x[list(x.keys())[0]]["osm_source"], "osm_behaviour": x[list(x.keys())[0]]["osm_behaviour"], } return owf, twf
def new(self, file_name: str) -> None: """Creates a new project Args: *file_name* (:obj:`str`): Full path to the project data file. If file exists, it will fail """ self.path_to_file = file_name self.source = self.path_to_file self.parameters = Parameters().parameters if os.path.isfile(file_name): raise FileNotFoundError( "File already exist. Choose a different name or remove the existing file" ) self.__create_empty_project() self.conn = spatialite_connection(self.conn) self.network = Network(self)
def __set_logging_path(self): p = Parameters() par = p.parameters if p.parameters is None: par = p._default do_log = par["system"]["logging"] for handler in logger.handlers: if handler.name == "aequilibrae": logger.removeHandler(handler) if do_log: formatter = logging.Formatter("%(asctime)s;%(name)s;%(levelname)s ; %(message)s") log_file = os.path.join(self.project_base_path, "aequilibrae.log") if not os.path.isfile(log_file): a = open(log_file, "w") a.close() ch = logging.FileHandler(log_file) ch.name = "aequilibrae" ch.setFormatter(formatter) ch.setLevel(logging.DEBUG) logger.addHandler(ch)
def __init__(self): self.link_loads = None # type: np.array # The actual results for assignment self.total_link_loads = None # type: np.array # The result of the assignment for all user classes summed self.skims = None # The array of skims self.no_path = None # The list os paths self.num_skims = None # number of skims that will be computed. Depends on the setting of the graph provided p = Parameters().parameters['system']['cpus'] if not isinstance(p, int): p = 0 self.set_cores(p) self.classes = {"number": 1, "names": ["flow"]} self.critical_links = { "save": False, "queries": {}, "results": False } # Queries are a dictionary self.link_extraction = { "save": False, "queries": {}, "output": None } # Queries are a dictionary self.path_file = {"save": False, "results": None} self.nodes = -1 self.zones = -1 self.links = -1 self.__graph_id__ = None self.__float_type = None self.__integer_type = None self.lids = None self.direcs = None
def create_from_osm( self, west: float = None, south: float = None, east: float = None, north: float = None, place_name: str = None, modes=["car", "transit", "bicycle", "walk"], ) -> None: """ Downloads the network from Open-Street Maps Args: *west* (:obj:`float`, Optional): West most coordinate of the download bounding box *south* (:obj:`float`, Optional): South most coordinate of the download bounding box *east* (:obj:`float`, Optional): East most coordinate of the download bounding box *place_name* (:obj:`str`, Optional): If not downloading with East-West-North-South boundingbox, this is required *modes* (:obj:`list`, Optional): List of all modes to be downloaded. Defaults to the modes in the parameter file p = Project() p.new(nm) :: from aequilibrae import Project, Parameters p = Project() p.new('path/to/project') # We now choose a different overpass endpoint (say a deployment in your local network) par = Parameters() par.parameters['osm']['overpass_endpoint'] = "http://192.168.1.234:5678/api" # Because we have our own server, we can set a bigger area for download (in M2) par.parameters['osm']['max_query_area_size'] = 10000000000 # And have no pause between successive queries par.parameters['osm']['sleeptime'] = 0 # Save the parameters to disk par.write_back() # And do the import p.network.create_from_osm(place_name=my_beautiful_hometown) p.close() """ if self.count_links() > 0: raise FileExistsError( "You can only import an OSM network into a brand new model file" ) curr = self.conn.cursor() curr.execute("""ALTER TABLE links ADD COLUMN osm_id integer""") curr.execute("""ALTER TABLE nodes ADD COLUMN osm_id integer""") self.conn.commit() if isinstance(modes, (tuple, list)): modes = list(modes) elif isinstance(modes, str): modes = [modes] else: raise ValueError( "'modes' needs to be string or list/tuple of string") if place_name is None: if min(east, west) < -180 or max(east, west) > 180 or min( north, south) < -90 or max(north, south) > 90: raise ValueError("Coordinates out of bounds") bbox = [west, south, east, north] else: bbox, report = placegetter(place_name) west, south, east, north = bbox if bbox is None: msg = f'We could not find a reference for place name "{place_name}"' warn(msg) logger.warning(msg) return for i in report: if "PLACE FOUND" in i: logger.info(i) # Need to compute the size of the bounding box to not exceed it too much height = haversine((east + west) / 2, south, (east + west) / 2, north) width = haversine(east, (north + south) / 2, west, (north + south) / 2) area = height * width par = Parameters().parameters["osm"] max_query_area_size = par["max_query_area_size"] if area < max_query_area_size: polygons = [bbox] else: polygons = [] parts = math.ceil(area / max_query_area_size) horizontal = math.ceil(math.sqrt(parts)) vertical = math.ceil(parts / horizontal) dx = (east - west) / horizontal dy = (north - south) / vertical for i in range(horizontal): xmin = max(-180, west + i * dx) xmax = min(180, west + (i + 1) * dx) for j in range(vertical): ymin = max(-90, south + j * dy) ymax = min(90, south + (j + 1) * dy) box = [xmin, ymin, xmax, ymax] polygons.append(box) logger.info("Downloading data") self.downloader = OSMDownloader(polygons, modes) self.downloader.doWork() logger.info("Building Network") self.builder = OSMBuilder(self.downloader.json, self.source) self.builder.doWork() logger.info("Network built successfully")
def placegetter(place: str) -> Tuple[Union[None, List[float]], list]: """ Send a request to the Nominatim API via HTTP GET and return a geometry polygon for the region we are querying Parameters ---------- place : str Name of the place we want to download a network for Adapted from http://www.github.com/gboeing/osmnx """ par = Parameters().parameters['osm'] nominatim_endpoint = par['nominatim_endpoint'] max_attempts = par['max_attempts'] params = {"q": place, "format": "json"} report = [] pause_duration = 1 timeout = 30 error_pause_duration = 180 # prepare the Nominatim API URL url = nominatim_endpoint.rstrip("/") + "/search" prepared_url = requests.Request("GET", url, params=params).prepare().url # Pause, then request it report.append( "Pausing {:,.2f} seconds before making API GET request".format( pause_duration)) time.sleep(pause_duration) start_time = time.time() report.append(f"Requesting {prepared_url} with timeout={timeout}") response = requests.get(url, params=params, timeout=timeout, headers=http_headers) # get the response size and the domain, log result size_kb = len(response.content) / 1000.0 domain = re.findall(r"(?s)//(.*?)/", url)[0] report.append("Downloaded {:,.1f}KB from {} in {:,.2f} seconds".format( size_kb, domain, time.time() - start_time)) bbox = None for attempts in range(max_attempts): report.append(f"Attempt: {attempts}") if response.status_code != 200: report.append( "Server at {} returned status code {} and no JSON data. Re-trying request in {:.2f} seconds." .format(domain, response.status_code, error_pause_duration)) if response.status_code in [429, 504]: # SEND MESSAGE time.sleep(error_pause_duration) continue elif response.status_code == 200: response_json = response.json() report.append("COMPLETE QUERY RESPONSE FOR PLACE:") report.append(str(response_json)) if len(response_json): bbox = [float(x) for x in response_json[0]["boundingbox"]] bbox = [bbox[2], bbox[0], bbox[3], bbox[1]] report.append( f"PLACE FOUND:{response_json[0]['display_name']}") return (bbox, report) else: bbox = None if attempts == max_attempts - 1 and bbox is None: report.append( "Reached maximum download attempts. Please wait a few minutes and try again" ) else: report.append("We got an error for place query.") return (bbox, report)
def parameters(self) -> dict: return Parameters().parameters
import requests from aequilibrae.parameters import Parameters par = Parameters().parameters['osm'] accept_language = par['accept_language'] memory = 0 user_agent = "AequilibraE (https://github.com/aequilibrae/aequilibrae-GUI)" referer = "AequilibraE (https://github.com/aequilibrae/aequilibrae-GUI)" http_headers = requests.utils.default_headers() http_headers.update({ "User-Agent": user_agent, "referer": referer, "Accept-Language": accept_language, "format": "json" })
def get_node_fields(): p = Parameters() fields = p.parameters["network"]["nodes"]["fields"] fields = [list(x.keys())[0] for x in fields] return fields + ["osm_id"]
def __init__(self): self.path_to_file: str = None self.source: str = None self.parameters = Parameters().parameters self.conn: sqlite3.Connection = None self.network: Network = None