def setUp(self) -> None:
os.environ['PATH'] = os.path.join(
tempfile.gettempdir(), 'temp_data') + ';' + os.environ['PATH']
self.temp_proj_folder = os.path.join(tempfile.gettempdir(),
uuid.uuid4().hex)
copytree(no_triggers_project, self.temp_proj_folder)
self.proj = Project()
self.proj.open(self.temp_proj_folder)
self.curr = self.proj.conn.cursor()
# Modes to add
sql = 'INSERT INTO modes (mode_name, mode_id) VALUES (?, ?);'
for mid in ['p', 'l', 'g', 'x', 'y', 'd', 'k', 'a', 'r', 'n', 'm']:
self.curr.execute(sql, [f'mode_{mid}', mid])
self.proj.conn.commit()
curr = self.proj.conn.cursor()
self.rtree = True
try:
curr.execute("SELECT rtreecheck('idx_nodes_geometry');")
except Exception as e:
self.rtree = False
warn(f'RTREE not available --> {e.args}')
root = os.path.dirname(os.path.realpath(__file__)).replace('tests', '')
qry_file = os.path.join(
root, "database_specification/triggers/modes_table_triggers.sql")
with open(qry_file, "r") as sql_file:
self.queries = sql_file.read()
self.queries = [cmd for cmd in self.queries.split("#")]
def setUp(self) -> None:
self.file = os.path.join(gettempdir(),
"aequilibrae_project_test.sqlite")
self.project = Project()
self.project.new(self.file)
self.source = self.file
self.file2 = os.path.join(gettempdir(),
"aequilibrae_project_test2.sqlite")
self.conn = sqlite3.connect(self.file2)
self.conn.enable_load_extension(True)
plat = platform.platform()
pth = os.getcwd()
if "WINDOWS" in plat.upper():
par = Parameters()
spatialite_path = par.parameters["system"]["spatialite_path"]
if os.path.isfile(
os.path.join(spatialite_path, "mod_spatialite.dll")):
os.chdir(spatialite_path)
try:
self.conn.load_extension("mod_spatialite")
except Exception as e:
warn(
f"AequilibraE might not work as intended without spatialite. {e.args}"
)
os.chdir(pth)
self.network = Network(self)
def doWork(self):
self.emit_messages(message="Initializing project", value=0, max_val=1)
self.project = Project()
self.project.new(self.output_file)
self.project.conn.close()
self.project.conn = qgis.utils.spatialite_connect(self.output_file)
self.project.network.conn = self.project.conn
self.project.network.create_empty_tables()
# Add the required extra fields to the link layer
self.emit_messages(
message="Adding extra network data fields to database",
value=0,
max_val=1)
self.additional_fields_to_layers('links', self.link_layer,
self.link_fields)
self.additional_fields_to_layers('nodes', self.node_layer,
self.node_fields)
conn = qgis.utils.spatialite_connect(self.output_file)
self.transfer_layer_features("links", self.link_layer,
self.link_fields)
self.transfer_layer_features("nodes", self.node_layer,
self.node_fields)
self.emit_messages(message="Creating layer triggers",
value=0,
max_val=1)
self.project.network.add_triggers()
self.emit_messages(message="Spatial indices", value=0, max_val=1)
self.project.network.add_spatial_index()
self.ProgressText.emit("DONE")
def setUp(self) -> None:
os.environ['PATH'] = os.path.join(gettempdir(), 'temp_data') + ';' + os.environ['PATH']
self.proj_dir = os.path.join(gettempdir(), uuid.uuid4().hex)
copytree(siouxfalls_project, self.proj_dir)
self.project = Project()
self.project.open(self.proj_dir)
def setUp(self) -> None:
os.environ['PATH'] = os.path.join(gettempdir(), 'temp_data') + ';' + os.environ['PATH']
self.proj_path = os.path.join(gettempdir(), f'aeq_{uuid.uuid4().hex}')
copytree(siouxfalls_project, self.proj_path)
self.siouxfalls = Project()
self.siouxfalls.open(self.proj_path)
remove_triggers(self.siouxfalls.conn)
add_triggers(self.siouxfalls.conn)
def test_opening_wrong_folder(self):
temp_proj_folder = os.path.join(tempfile.gettempdir(),
uuid.uuid4().hex)
self.proj.close()
with self.assertRaises(FileNotFoundError):
proj = Project()
proj.open(temp_proj_folder)
self.proj.open(self.temp_proj_folder)
class TestNetworkTriggers(TestCase):
def setUp(self) -> None:
os.environ['PATH'] = os.path.join(gettempdir(), 'temp_data') + ';' + os.environ['PATH']
self.proj_path = os.path.join(gettempdir(), f'aeq_{uuid.uuid4().hex}')
copytree(siouxfalls_project, self.proj_path)
self.siouxfalls = Project()
self.siouxfalls.open(self.proj_path)
remove_triggers(self.siouxfalls.conn)
add_triggers(self.siouxfalls.conn)
def tearDown(self) -> None:
self.siouxfalls.close()
def test_delete_links_delete_nodes(self):
items = self.siouxfalls.network.count_nodes()
self.assertEqual(24, items, 'Wrong number of nodes found')
links = self.siouxfalls.network.links
nodes = self.siouxfalls.network.nodes
node = nodes.get(1)
node.is_centroid = 0
node.save()
# We have completely disconnected 2 nodes (1 and 2)
for i in [1, 2, 3, 4, 5, 14]:
link = links.get(i)
link.delete()
# Since node 1 is no longer a centroid, we should have only 23 nodes in the network
items = self.siouxfalls.network.count_nodes()
self.assertEqual(23, items, 'Wrong number of nodes found')
def test_add_regular_link(self):
# Add a regular link to see if it fails when creating it
# It happened at some point
curr = self.siouxfalls.conn.cursor()
data = [123456, 'c', 'default', LineString([Point(0, 0), Point(1, 1)]).wkb]
sql = 'insert into links (link_id, modes, link_type, geometry) Values(?,?,?,GeomFromWKB(?, 4326));'
curr.execute(sql, data)
self.siouxfalls.conn.commit()
def test_add_regular_node_change_centroid_id(self):
# Add a regular link to see if it fails when creating it
# It happened at some point
curr = self.siouxfalls.conn.cursor()
network = self.siouxfalls.network
nodes = network.count_nodes()
data = [987654, 1, Point(0, 0).wkb]
sql = 'insert into nodes (node_id, is_centroid, geometry) Values(?,?,GeomFromWKB(?, 4326));'
curr.execute(sql, data)
self.siouxfalls.conn.commit()
self.assertEqual(nodes + 1, network.count_nodes(), 'Failed to insert node')
curr.execute('Update nodes set is_centroid=0 where node_id=?', data[:1])
self.siouxfalls.conn.commit()
self.assertEqual(nodes, network.count_nodes(), 'Failed to delete node when changing centroid flag')
def setUp(self) -> None:
proj_dir = os.path.join(gettempdir(), uuid.uuid4().hex)
copytree(siouxfalls_project, proj_dir)
self.project = Project()
self.project.open(proj_dir)
self.matrices = self.project.matrices
self.matrices.reload()
self.curr = self.project.conn.cursor()
def setUp(self) -> None:
os.environ["PATH"] = os.path.join(
tempfile.gettempdir(), "temp_data") + ";" + os.environ["PATH"]
self.temp_proj_folder = os.path.join(tempfile.gettempdir(),
uuid4().hex)
copytree(siouxfalls_project, self.temp_proj_folder)
self.project = Project()
self.project.open(self.temp_proj_folder)
self.project.network.build_graphs()
self.graph = self.project.network.graphs["c"]
def setUp(self) -> None:
os.environ["PATH"] = os.path.join(gettempdir(), "temp_data") + ";" + os.environ["PATH"]
self.proj_dir = os.path.join(gettempdir(), uuid.uuid4().hex)
copytree(siouxfalls_project, self.proj_dir)
self.project = Project()
self.project.open(self.proj_dir)
self.network = self.project.network
self.curr = self.project.conn.cursor()
def setUp(self) -> None:
self.file = os.path.join(gettempdir(),
"aequilibrae_project_test.sqlite")
self.project = Project()
self.project.new(self.file)
self.source = self.file
self.file2 = os.path.join(gettempdir(),
"aequilibrae_project_test2.sqlite")
self.conn = sqlite3.connect(self.file2)
self.conn = spatialite_connection(self.conn)
self.network = Network(self)
class TestProject(TestCase):
def setUp(self) -> None:
self.temp_proj_folder = os.path.join(tempfile.gettempdir(),
uuid.uuid4().hex)
self.proj = Project()
self.proj.new(self.temp_proj_folder)
def tearDown(self) -> None:
self.proj.close()
def test_opening_wrong_folder(self):
temp_proj_folder = os.path.join(tempfile.gettempdir(),
uuid.uuid4().hex)
self.proj.close()
with self.assertRaises(FileNotFoundError):
proj = Project()
proj.open(temp_proj_folder)
self.proj.open(self.temp_proj_folder)
def test_create_when_already_exists(self):
with self.assertRaises(Exception):
q = Project()
q.new(os.path.join(tempfile.gettempdir(), uuid.uuid4().hex))
with self.assertRaises(Exception):
q = Project()
q.open(os.path.join(tempfile.gettempdir(), uuid.uuid4().hex))
def test_creation(self):
curr = self.proj.conn.cursor()
curr.execute("""PRAGMA table_info(links);""")
fields = curr.fetchall()
fields = [x[1] for x in fields]
if 'distance' not in fields:
self.fail("Table LINKS was not created correctly")
curr = self.proj.conn.cursor()
curr.execute("""PRAGMA table_info(nodes);""")
nfields = curr.fetchall()
nfields = [x[1] for x in nfields]
if 'is_centroid' not in nfields:
self.fail('Table NODES was not created correctly')
def test_close(self):
_ = database_connection()
self.proj.close()
with self.assertRaises(FileExistsError):
_ = database_connection()
def setUp(self) -> None:
self.temp_proj_folder = os.path.join(tempfile.gettempdir(),
uuid.uuid4().hex)
copytree(no_triggers_project, self.temp_proj_folder)
self.proj = Project()
self.proj.open(self.temp_proj_folder)
self.curr = self.proj.conn.cursor()
letters = [
random.choice(string.ascii_letters + '_') for x in range(20)
]
self.random_string = ''.join(letters)
class TestLinkType(TestCase):
def setUp(self) -> None:
self.temp_proj_folder = os.path.join(tempfile.gettempdir(),
uuid.uuid4().hex)
copytree(no_triggers_project, self.temp_proj_folder)
self.proj = Project()
self.proj.open(self.temp_proj_folder)
self.curr = self.proj.conn.cursor()
letters = [
random.choice(string.ascii_letters + '_') for x in range(20)
]
self.random_string = ''.join(letters)
def tearDown(self) -> None:
self.proj.close()
def test_changing_link_type_id(self):
ltypes = self.proj.network.link_types
lt = random.choice([x for x in ltypes.all_types().values()])
with self.assertRaises(ValueError):
lt.link_type_id = 'test my description'
with self.assertRaises(ValueError):
lt.link_type_id = 'K'
def test_empty(self):
ltypes = self.proj.network.link_types
newt = ltypes.new('Z')
# a.link_type = 'just a_test'
with self.assertRaises(IntegrityError):
newt.save()
def test_save(self):
ltypes = self.proj.network.link_types
newt = ltypes.new('Z')
newt.link_type = self.random_string
newt.description = self.random_string[::-1]
newt.save()
self.curr.execute(
'select description, link_type from link_types where link_type_id="Z"'
)
desc, mname = self.curr.fetchone()
self.assertEqual(desc, self.random_string[::-1],
"Didn't save the mode description correctly")
self.assertEqual(mname, self.random_string,
"Didn't save the mode name correctly")
def run_load_project(self):
formats = ["AequilibraE Project(*.sqlite)"]
path, dtype = GetOutputFileName(
QtWidgets.QDialog(),
"AequilibraE Project",
formats,
".sqlite",
standard_path(),
)
# Cleans the project descriptor
tab_count = 1
for i in range(tab_count):
self.projectManager.removeTab(i)
if dtype is not None:
self.contents = []
self.showing.setVisible(True)
self.project = Project()
self.project.load(path)
self.project.conn = qgis.utils.spatialite_connect(path)
self.project.network.conn = self.project.conn
uri = QgsDataSourceUri()
uri.setDatabase(path)
uri.setDataSource('', 'links', 'geometry')
self.link_layer = QgsVectorLayer(uri.uri(), 'links', 'spatialite')
QgsProject.instance().addMapLayer(self.link_layer)
uri.setDataSource('', 'nodes', 'geometry')
self.node_layer = QgsVectorLayer(uri.uri(), 'nodes', 'spatialite')
QgsProject.instance().addMapLayer(self.node_layer)
descr = QWidget()
descrlayout = QVBoxLayout()
# We create a tab with the main description of the project
p1 = QLabel('Project: {}'.format(path))
p2 = QLabel('Modes: {}'.format(', '.join(
self.project.network.modes())))
p3 = QLabel('Total Links: {:,}'.format(
self.project.network.count_links()))
p4 = QLabel('Total Nodes: {:,}'.format(
self.project.network.count_nodes()))
for p in [p1, p2, p3, p4]:
descrlayout.addWidget(p)
descr.setLayout(descrlayout)
self.tabContents = [(descr, "Project")]
self.projectManager.addTab(descr, "Project")
def setUp(self) -> None:
self.matrix = AequilibraeMatrix()
self.matrix.load(siouxfalls_demand)
self.matrix.computational_view()
self.project = Project()
self.project.load(siouxfalls_project)
self.project.network.build_graphs()
self.car_graph = self.project.network.graphs['c'] # type: Graph
self.car_graph.set_graph('free_flow_time')
self.car_graph.set_blocked_centroid_flows(False)
self.assignment = TrafficAssignment()
self.assigclass = TrafficClass(self.car_graph, self.matrix)
def test_create_when_already_exists(self):
with self.assertRaises(Exception):
q = Project()
q.new(os.path.join(tempfile.gettempdir(), uuid.uuid4().hex))
with self.assertRaises(Exception):
q = Project()
q.open(os.path.join(tempfile.gettempdir(), uuid.uuid4().hex))
def test_creation(self):
test_file = temp_proj_name
with self.assertRaises(FileNotFoundError):
p = Project()
p.load(test_file)
p = Project()
p.new(test_file)
p.conn.close()
def setUp(self) -> None:
os.environ['PATH'] = os.path.join(
gettempdir(), '../../temp_data') + ';' + os.environ['PATH']
self.proj_dir = os.path.join(gettempdir(), uuid.uuid4().hex)
copytree(siouxfalls_project, self.proj_dir)
self.project = Project()
self.project.open(self.proj_dir)
self.network = self.project.network
self.curr = self.project.conn.cursor()
self.links = self.network.links
self.modes = self.network.modes
self.links = self.network.links
self.lid = randint(1, 24)
self.link = self.links.get(self.lid)
def final_steps(self):
self.project = Project(self.output_path.text(), True)
self.project.network.create_empty_tables()
curr = self.project.conn.cursor()
curr.execute("""ALTER TABLE links ADD COLUMN osm_id integer""")
curr.execute("""ALTER TABLE nodes ADD COLUMN osm_id integer""")
self.project.conn.commit()
self.project.conn.close()
self.builder = OSMBuilder(self.downloader.json, self.project.source)
self.run_thread()
def test_create_from_osm(self):
thresh = 0.05
if os.environ.get('GITHUB_WORKFLOW', 'ERROR') == 'Code coverage':
thresh = 1.01
if random() < thresh:
self.siouxfalls.close()
self.project = Project()
self.project.new(self.proj_path2)
# self.network.create_from_osm(west=153.1136245, south=-27.5095487, east=153.115, north=-27.5085, modes=["car"])
self.project.network.create_from_osm(west=-112.185,
south=36.59,
east=-112.179,
north=36.60)
curr = self.project.conn.cursor()
curr.execute("""select count(*) from links""")
lks = curr.fetchone()[0]
curr.execute("""select count(distinct osm_id) from links""")
osmids = curr.fetchone()[0]
if osmids == 0:
warn('COULD NOT RETRIEVE DATA FROM OSM')
return
if osmids >= lks:
self.fail("OSM links not broken down properly")
curr.execute("""select count(*) from nodes""")
nds = curr.fetchone()[0]
if lks > nds:
self.fail(
"We imported more links than nodes. Something wrong here")
self.project.close()
self.siouxfalls.open(self.proj_path)
else:
print('Skipped check to not load OSM servers')
def test_exclude_links(self):
p = Project()
p.load(siouxfalls_project)
p.network.build_graphs()
g = p.network.graphs['c'] # type: Graph
# excludes a link before any setting or preparation
g.exclude_links([12])
g.set_graph('distance')
r1 = PathResults()
r1.prepare(g)
r1.compute_path(1, 14)
self.assertEqual(list(r1.path), [2, 6, 10, 34])
# We exclude one link that we know was part of the last shortest path
g.exclude_links([10])
r2 = PathResults()
r2.prepare(g)
r2.compute_path(1, 14)
self.assertEqual(list(r2.path), [2, 7, 36, 34])
p.conn.close()
def signal_handler(self, val):
if val[0] == "Value":
self.progressbar.setValue(val[1])
elif val[0] == "maxValue":
self.progressbar.setRange(0, val[1])
elif val[0] == "text":
self.progress_label.setText(val[1])
elif val[0] == "finished_threaded_procedure":
self.project = Project(self.output_path.text())
self.progress_label.setText('Adding spatial indices')
self.project.network.add_spatial_index()
self.project.network.add_triggers()
l = self.project.network.count_links()
n = self.project.network.count_nodes()
self.report.append(reporter(f'{l:,} links generated'))
self.report.append(reporter(f'{n:,} nodes generated'))
self.leave()
class TestLog(TestCase):
def setUp(self) -> None:
os.environ['PATH'] = os.path.join(gettempdir(), 'temp_data') + ';' + os.environ['PATH']
self.proj_dir = os.path.join(gettempdir(), uuid.uuid4().hex)
copytree(siouxfalls_project, self.proj_dir)
self.project = Project()
self.project.open(self.proj_dir)
def tearDown(self) -> None:
self.project.close()
def test_contents(self):
log = self.project.log()
cont = log.contents()
self.assertEqual(len(cont), 4, 'Returned the wrong amount of data from the log')
def test_clear(self):
log = self.project.log()
log.clear()
with open(os.path.join(self.proj_dir, "aequilibrae.log"), 'r') as file:
q = file.readlines()
self.assertEqual(len(q), 0, 'Failed to clear the log file')
class TestLinkTypes(TestCase):
def setUp(self) -> None:
self.temp_proj_folder = os.path.join(tempfile.gettempdir(),
uuid.uuid4().hex)
copytree(no_triggers_project, self.temp_proj_folder)
self.proj = Project()
self.proj.open(self.temp_proj_folder)
self.curr = self.proj.conn.cursor()
letters = [
random.choice(string.ascii_letters + '_') for x in range(20)
]
self.random_string = ''.join(letters)
def tearDown(self) -> None:
self.proj.close()
def test_add(self):
lt = self.proj.network.link_types
existing = list(lt.all_types().keys())
newlt = lt.new('G')
newlt.link_type = 'unique_link_type'
newlt.save()
nowexisting = list(lt.all_types().keys())
n = [x for x in nowexisting if x not in existing][0]
self.assertEqual('G', n, 'Failed to add link type')
def test_delete(self):
lt = self.proj.network.link_types
existing = list(lt.all_types().keys())
deleted = random.choice(existing)
lt.delete(deleted)
remaining = list(lt.all_types().keys())
difference = [x for x in existing if x not in remaining]
self.assertEqual(deleted, difference[0], 'Failed to delete link type')
def test_get_and_get_by_name(self):
lt = self.proj.network.link_types
ltget = lt.get('y')
ltgetbn = lt.get_by_name('default')
self.assertEqual(ltget.link_type_id, ltgetbn.link_type_id,
'Get methods returned different things')
self.assertEqual(ltget.link_type, ltgetbn.link_type,
'Get methods returned different things')
self.assertEqual(ltget.description, ltgetbn.description,
'Get methods returned different things')
self.assertEqual(ltget.lanes, ltgetbn.lanes,
'Get methods returned different things')
self.assertEqual(ltget.lane_capacity, ltgetbn.lane_capacity,
'Get methods returned different things')
self.assertEqual(ltget.link_type, 'default',
'Get methods returned different things')
def test_all_types(self):
lt = self.proj.network.link_types
all_lts = set([x for x in lt.all_types().keys()])
c = self.proj.conn.cursor()
c.execute('select link_type_id from link_types')
reallts = set([x[0] for x in c.fetchall()])
diff = all_lts.symmetric_difference(reallts)
self.assertEqual(diff, set(), 'Getting all link_types failed')
class TestMatrices(TestCase):
def setUp(self) -> None:
proj_dir = os.path.join(gettempdir(), uuid.uuid4().hex)
copytree(siouxfalls_project, proj_dir)
self.project = Project()
self.project.open(proj_dir)
self.matrices = self.project.matrices
self.matrices.reload()
self.curr = self.project.conn.cursor()
def tearDown(self) -> None:
self.project.close()
del self.project
del self.matrices
def randomword(self, length):
allowed_characters = string.ascii_letters + '_123456789@!()[]{};:" -'
val = "".join(choice(allowed_characters) for i in range(length))
if val[0] == "_" or val[-1] == "_":
return self.randomword(length)
return val
def test_set_record(self):
rec = self.matrices.get_record("omx2")
with self.assertRaises(ValueError):
rec.name = "omx"
with self.assertRaises(ValueError):
rec.file_name = "sfalls_skims.omx"
rec.file_name = "SiouxFalls.omx"
self.assertEqual(
rec.cores, 1,
"Setting a file that exists did not correct the number of cores")
def test_clear_database(self):
self.__mat_count(3, "The test data started wrong")
self.matrices.clear_database()
self.__mat_count(2, "Did not clear the database appropriately")
def test_update_database(self):
self.__mat_count(3, "The test data started wrong")
self.matrices.update_database()
self.__mat_count(4, "Did not add to the database appropriately")
rec = self.matrices.get_record("omx")
existing = join(rec.fldr, rec.file_name)
new_name = "test_name.omx"
new_name1 = "test_name1.omx"
copyfile(existing, join(rec.fldr, new_name))
record = self.matrices.new_record("test_name1.omx", new_name)
record.save()
copyfile(existing, join(rec.fldr, new_name1))
self.matrices.update_database()
def test_get_matrix(self):
with self.assertRaises(Exception):
_ = self.matrices.get_matrix("omxq")
mat = self.matrices.get_matrix("oMx")
mat.computational_view()
self.assertEqual(floor(mat.matrix_view.sum()), 20309,
"Matrix loaded incorrectly")
def test_get_record(self):
rec = self.matrices.get_record("omx")
self.assertEqual(rec.cores, 4,
"record populated wrong. Number of cores")
self.assertEqual(rec.description, None,
"record populated wrong. Description")
def test_record_update_cores(self):
rec = self.matrices.get_record("omx")
rec.update_cores()
self.assertEqual(rec.cores, 2, "Cores update did not work")
def test_save_record(self):
rec = self.matrices.get_record("omx")
text = self.randomword(randint(30, 100))
rec.description = text
rec.save()
self.curr.execute('select description from matrices where name="omx";')
self.assertEqual(text,
self.curr.fetchone()[0],
"Saving matrix record description failed")
def test_delete(self):
self.matrices.delete_record("omx")
self.curr.execute('select count(*) from matrices where name="omx";')
self.assertEqual(0,
self.curr.fetchone()[0],
" Deleting matrix record failed")
with self.assertRaises(Exception):
self.matrices.get_record("omx")
def test_list(self):
df = self.matrices.list()
self.curr.execute("select count(*) from Matrices")
self.assertEqual(
df.shape[0],
self.curr.fetchone()[0],
"Returned the wrong number of matrices in the database")
self.assertEqual(df[df.status == "file missing"].shape[0], 1,
"Wrong # of records for missing matrix files")
def __mat_count(self, should_have: int, error_message: str) -> None:
self.curr.execute("Select count(*) from Matrices;")
self.assertEqual(self.curr.fetchone()[0], should_have, error_message)
class CreatesTranspoNetProcedure(WorkerThread):
def __init__(
self,
parentThread,
output_file,
node_layer,
node_fields,
link_layer,
link_fields,
):
WorkerThread.__init__(self, parentThread)
self.output_file = output_file
self.node_fields = node_fields
self.link_fields = link_fields
self.node_layer = node_layer
self.link_layer = link_layer
self.report = []
self.project: Project
def doWork(self):
self.emit_messages(message="Initializing project", value=0, max_val=1)
self.project = Project()
self.project.new(self.output_file)
self.project.conn.close()
self.project.conn = qgis.utils.spatialite_connect(self.output_file)
self.project.network.conn = self.project.conn
self.project.network.create_empty_tables()
# Add the required extra fields to the link layer
self.emit_messages(
message="Adding extra network data fields to database",
value=0,
max_val=1)
self.additional_fields_to_layers('links', self.link_layer,
self.link_fields)
self.additional_fields_to_layers('nodes', self.node_layer,
self.node_fields)
conn = qgis.utils.spatialite_connect(self.output_file)
self.transfer_layer_features("links", self.link_layer,
self.link_fields)
self.transfer_layer_features("nodes", self.node_layer,
self.node_fields)
self.emit_messages(message="Creating layer triggers",
value=0,
max_val=1)
self.project.network.add_triggers()
self.emit_messages(message="Spatial indices", value=0, max_val=1)
self.project.network.add_spatial_index()
self.ProgressText.emit("DONE")
# Adds the non-standard fields to a layer
def additional_fields_to_layers(self, table, layer, layer_fields):
curr = self.project.conn.cursor()
fields = layer.dataProvider().fields()
string_fields = []
curr.execute(f'PRAGMA table_info({table});')
field_names = curr.fetchall()
existing_fields = [f[1].lower() for f in field_names]
for f in set(layer_fields.keys()):
if f in existing_fields:
continue
field = fields[layer_fields[f]]
field_length = field.length()
if not field.isNumeric():
field_type = "char"
string_fields.append(f)
else:
if "Int" in field.typeName():
field_type = "INTEGER"
else:
field_type = "REAL"
try:
sql = "alter table " + table + " add column " + f + " " + field_type + "(" + str(
field_length) + ")"
curr.execute(sql)
self.project.conn.commit()
except:
logger.error(sql)
self.report.append("field " + str(f) + " could not be added")
curr.close()
return string_fields
def transfer_layer_features(self, table, layer, layer_fields):
self.emit_messages(message=f"Transferring features from {table} layer",
value=0,
max_val=layer.featureCount())
curr = self.project.conn.cursor()
field_titles = ", ".join(list(layer_fields.keys()))
all_modes = set()
for j, f in enumerate(layer.getFeatures()):
self.emit_messages(value=j)
attrs = []
for k, val in layer_fields.items():
if val < 0:
attrs.append("NULL")
else:
attr_val = self.convert_data(f.attributes()[val])
if not str(attr_val).isnumeric():
attrs.append(f'"{attr_val}"')
else:
attrs.append(attr_val)
attrs = ", ".join(attrs)
geom = f.geometry().asWkt().upper()
crs = str(layer.crs().authid().split(":")[1])
sql = f"INSERT INTO {table} ({field_titles} , geometry) VALUES ({attrs} , GeomFromText('{geom}', {crs}))"
if table == 'links':
all_modes.update(list(f.attributes()[layer_fields['modes']]))
try:
curr.execute(sql)
except:
logger.info(f'Failed inserting link {f.id()}')
logger.info(sql)
if f.id():
msg = f"feature with id {f.id()} could not be added to layer {table}"
else:
msg = f"feature with no node id present. It could not be added to layer {table}"
self.report.append(msg)
# We check if all modes exist
a = self.project.network.modes()
for x in all_modes:
if x not in a:
par = [
f'"automatic_{x}"', f'"{x}"',
'"Mode automatically added during project creation from layers"'
]
curr.execute(
f'INSERT INTO "modes" (mode_name, mode_id, description) VALUES({",".join(par)})'
)
logger.info(f'New mode inserted during project creation {x}')
self.project.conn.commit()
curr.close()
def convert_data(self, value):
if type(value) == NULL:
return "NULL"
else:
return str(value).replace('"', "'")
def emit_messages(self, message="", value=-1, max_val=-1):
if len(message) > 0:
self.ProgressText.emit(message)
if value >= 0:
self.ProgressValue.emit(value)
if max_val >= 0:
self.ProgressMaxValue.emit(max_val)
class TestNetwork(TestCase):
def setUp(self) -> None:
self.file = os.path.join(gettempdir(),
"aequilibrae_project_test.sqlite")
self.project = Project()
self.project.new(self.file)
self.source = self.file
self.file2 = os.path.join(gettempdir(),
"aequilibrae_project_test2.sqlite")
self.conn = sqlite3.connect(self.file2)
self.conn = spatialite_connection(self.conn)
self.network = Network(self)
def tearDown(self) -> None:
try:
self.project.conn.close()
os.unlink(self.file)
self.conn.close()
os.unlink(self.file2)
except Exception as e:
warn(f'Could not delete. {e.args}')
def test_create_from_osm(self):
thresh = 0.05
if os.environ.get('GITHUB_WORKFLOW', 'ERROR') == 'Code coverage':
thresh = 1.01
if random() < thresh:
# self.network.create_from_osm(west=153.1136245, south=-27.5095487, east=153.115, north=-27.5085, modes=["car"])
self.project.network.create_from_osm(west=-112.185,
south=36.59,
east=-112.179,
north=36.60)
curr = self.project.conn.cursor()
curr.execute("""select count(*) from links""")
lks = curr.fetchone()
curr.execute("""select count(distinct osm_id) from links""")
osmids = curr.fetchone()
if osmids >= lks:
self.fail("OSM links not broken down properly")
curr.execute("""select count(*) from nodes""")
nds = curr.fetchone()
if lks > nds:
self.fail(
"We imported more links than nodes. Something wrong here")
else:
print('Skipped check to not load OSM servers')
def test_create_empty_tables(self):
self.network.create_empty_tables()
p = Parameters().parameters["network"]
curr = self.conn.cursor()
curr.execute("""PRAGMA table_info(links);""")
fields = curr.fetchall()
fields = [x[1] for x in fields]
oneway = reduce(lambda a, b: dict(a, **b),
p["links"]["fields"]["one-way"])
owf = list(oneway.keys())
twoway = reduce(lambda a, b: dict(a, **b),
p["links"]["fields"]["two-way"])
twf = []
for k in list(twoway.keys()):
twf.extend([f"{k}_ab", f"{k}_ba"])
for f in owf + twf:
if f not in fields:
self.fail(f"Field {f} not added to links table")
curr = self.conn.cursor()
curr.execute("""PRAGMA table_info(nodes);""")
nfields = curr.fetchall()
nfields = [x[1] for x in nfields]
flds = reduce(lambda a, b: dict(a, **b), p["nodes"]["fields"])
flds = list(flds.keys())
for f in flds:
if f not in nfields:
self.fail(f"Field {f} not added to nodes table")
class TestProject(TestCase):
def setUp(self) -> None:
os.environ['PATH'] = os.path.join(
tempfile.gettempdir(), 'temp_data') + ';' + os.environ['PATH']
self.temp_proj_folder = os.path.join(tempfile.gettempdir(),
uuid.uuid4().hex)
copytree(no_triggers_project, self.temp_proj_folder)
self.proj = Project()
self.proj.open(self.temp_proj_folder)
self.curr = self.proj.conn.cursor()
# Modes to add
sql = 'INSERT INTO modes (mode_name, mode_id) VALUES (?, ?);'
for mid in ['p', 'l', 'g', 'x', 'y', 'd', 'k', 'a', 'r', 'n', 'm']:
self.curr.execute(sql, [f'mode_{mid}', mid])
self.proj.conn.commit()
curr = self.proj.conn.cursor()
self.rtree = True
try:
curr.execute("SELECT rtreecheck('idx_nodes_geometry');")
except Exception as e:
self.rtree = False
warn(f'RTREE not available --> {e.args}')
root = os.path.dirname(os.path.realpath(__file__)).replace('tests', '')
qry_file = os.path.join(
root, "database_specification/triggers/modes_table_triggers.sql")
with open(qry_file, "r") as sql_file:
self.queries = sql_file.read()
self.queries = [cmd for cmd in self.queries.split("#")]
def tearDown(self) -> None:
self.proj.close()
def __get_query(self, qry: str) -> str:
for query in self.queries:
if qry in query:
return query
raise FileNotFoundError('QUERY DOES NOT EXIST')
def test_all_tests_considered(self):
tests_added = list(self.__dir__())
tests_added = [x[5:] for x in tests_added if x[:5] == 'test_']
for trigger in self.queries:
if 'TRIGGER' in trigger.upper():
found = [x for x in tests_added if x in trigger]
if not found:
self.fail(f'Trigger not tested. {trigger}')
def test_mode_single_letter_update(self):
cmd = self.__get_query('mode_single_letter_update')
sql = "UPDATE 'modes' SET mode_id= 'ttt' where mode_id='b'"
self.curr.execute(sql)
self.curr.execute(cmd)
with self.assertRaises(sqlite3.IntegrityError):
sql = "UPDATE 'modes' SET mode_id= 'gg' where mode_id='w'"
self.curr.execute(sql)
def test_mode_single_letter_insert(self):
cmd = self.__get_query('mode_single_letter_insert')
sql = "INSERT INTO 'modes' (mode_name, mode_id) VALUES(?, ?)"
self.curr.execute(sql, ['testasdasd', 'pp'])
self.curr.execute(cmd)
with self.assertRaises(sqlite3.IntegrityError):
self.curr.execute(sql, ['test1b', 'mm'])
def test_mode_keep_if_in_use_updating(self):
cmd = self.__get_query('mode_keep_if_in_use_updating')
sql = "UPDATE 'modes' SET mode_id= 'h' where mode_id='g'"
self.curr.execute(sql)
self.curr.execute(cmd)
sql = "UPDATE 'modes' SET mode_id= 'j' where mode_id='l'"
with self.assertRaises(sqlite3.IntegrityError):
self.curr.execute(sql)
def test_mode_keep_if_in_use_deleting(self):
cmd = self.__get_query('mode_keep_if_in_use_deleting')
sql = "DELETE FROM 'modes' where mode_id='p'"
self.curr.execute(sql)
self.curr.execute(cmd)
sql = "DELETE FROM 'modes' where mode_id='c'"
with self.assertRaises(sqlite3.IntegrityError):
self.curr.execute(sql)
def test_modes_on_links_update(self):
cmd = self.__get_query('modes_on_links_update')
sql = "UPDATE 'links' SET modes= 'qwerty' where link_id=55"
self.curr.execute(sql)
self.curr.execute(cmd)
sql = "UPDATE 'links' SET modes= 'azerty' where link_id=56"
with self.assertRaises(sqlite3.IntegrityError):
self.curr.execute(sql)
def test_modes_length_on_links_update(self):
cmd = self.__get_query('modes_length_on_links_update')
sql = "UPDATE 'links' SET modes= '' where modes='wb'"
self.curr.execute(sql)
self.curr.execute(cmd)
sql = "UPDATE 'links' SET modes= '' where modes='bw'"
with self.assertRaises(sqlite3.IntegrityError):
self.curr.execute(sql)
def test_modes_on_nodes_table_update_a_node(self):
cmd = self.__get_query('modes_on_nodes_table_update_a_node')
sql = "UPDATE 'links' SET a_node= 1 where a_node=3"
self.curr.execute(sql)
sql = "SELECT modes from nodes where node_id=1"
self.curr.execute(sql)
i = self.curr.fetchone()[0]
self.assertEqual(i, 'ct')
self.curr.execute(cmd)
k = ''
for n in [2, 5]:
for f in ['a_node', 'b_node']:
self.curr.execute(f"SELECT modes from links where {f}={n}")
k += self.curr.fetchone()[0]
existing = set(k)
sql = "UPDATE 'links' SET a_node= 2 where a_node=5"
self.curr.execute(sql)
sql = "SELECT modes from nodes where node_id=2"
self.curr.execute(sql)
i = set(self.curr.fetchone()[0])
self.assertEqual(i, existing)
def test_modes_on_nodes_table_update_b_node(self):
cmd = self.__get_query('modes_on_nodes_table_update_b_node')
sql = "UPDATE 'links' SET b_node= 1 where b_node=3"
self.curr.execute(sql)
sql = "SELECT modes from nodes where node_id=1"
self.curr.execute(sql)
i = self.curr.fetchone()[0]
self.assertEqual(i, 'ct')
self.curr.execute(cmd)
sql = "UPDATE 'links' SET b_node= 2 where b_node=4"
self.curr.execute(sql)
sql = "SELECT modes from nodes where node_id=2"
self.curr.execute(sql)
i = self.curr.fetchone()[0]
self.assertEqual(i, 'ctw')
def test_modes_on_nodes_table_update_links_modes(self):
cmd = self.__get_query('modes_on_nodes_table_update_links_modes')
sql = "UPDATE 'links' SET modes= 'x' where a_node=24"
self.curr.execute(sql)
sql = "SELECT modes from nodes where node_id=24"
self.curr.execute(sql)
i = self.curr.fetchone()[0]
self.assertEqual(i, 'c')
self.curr.execute(cmd)
sql = "UPDATE 'links' SET 'modes'= 'y' where a_node=24"
self.curr.execute(sql)
sql = "SELECT modes from nodes where node_id=24"
self.curr.execute(sql)
i = self.curr.fetchone()[0]
self.assertIn('c', i)
self.assertIn('y', i)
sql = "UPDATE 'links' SET 'modes'= 'r' where b_node=24"
self.curr.execute(sql)
sql = "SELECT modes from nodes where node_id=24"
self.curr.execute(sql)
i = self.curr.fetchone()[0]
self.assertIn('r', i)
self.assertIn('y', i)
def test_modes_on_links_insert(self):
cmd = self.__get_query('modes_on_links_insert')
if self.rtree:
self.curr.execute('pragma table_info(links)')
f = self.curr.fetchall()
fields = {x[1]: x[0] for x in f}
sql = 'select * from links where link_id=10'
self.curr.execute(sql)
a = [x for x in self.curr.fetchone()]
a[fields['modes']] = 'as12'
a[fields['link_id']] = 1234
a[fields['a_node']] = 999
a[fields['b_node']] = 888
a[0] = 1234
idx = ','.join(['?'] * len(a))
self.curr.execute(f'insert into links values ({idx})', a)
self.curr.execute('delete from links where link_id=1234')
self.curr.execute(cmd)
with self.assertRaises(sqlite3.IntegrityError):
self.curr.execute(f'insert into links values ({idx})', a)
def test_modes_length_on_links_insert(self):
cmd = self.__get_query('modes_length_on_links_insert')
if self.rtree:
self.curr.execute('pragma table_info(links)')
f = self.curr.fetchall()
fields = {x[1]: x[0] for x in f}
sql = 'select * from links where link_id=70'
self.curr.execute(sql)
a = [x for x in self.curr.fetchone()]
a[fields['modes']] = ''
a[fields['link_id']] = 4321
a[fields['a_node']] = 888
a[fields['b_node']] = 999
a[0] = 4321
idx = ','.join(['?'] * len(a))
self.curr.execute(f'insert into links values ({idx})', a)
self.curr.execute('delete from links where link_id=4321')
self.curr.execute(cmd)
with self.assertRaises(sqlite3.IntegrityError):
self.curr.execute(f'insert into links values ({idx})', a)
def test_keep_at_least_one(self):
cmd = self.__get_query('mode_keep_at_least_one')
self.curr.execute('Delete from modes;')
self.curr.execute('select count(*) from modes;')
self.assertEqual(
self.curr.fetchone()[0], 0,
'We were not able to clear the m odes table as expected')
self.curr.execute(
'insert into modes(mode_id, mode_name) VALUES("k", "some_name")')
self.curr.execute(cmd)
with self.assertRaises(sqlite3.IntegrityError):
self.curr.execute('Delete from modes;')
def test_modes_on_nodes_table_update_nodes_modes(self):
cmd = self.__get_query('modes_on_nodes_table_update_nodes_modes')
self.curr.execute(
'select node_id, modes from nodes where length(modes)>0')
dt = self.curr.fetchall()
x = choice(dt)
self.curr.execute(
f'update nodes set modes="abcdefgq" where node_id={x[0]}')
self.curr.execute(
f'select node_id, modes from nodes where node_id={x[0]}')
z = self.curr.fetchone()
if z == x:
self.fail(
'Modes field on nodes layer is being preserved by unknown mechanism'
)
self.curr.execute(cmd)
y = choice(dt)
while y == x:
y = choice(dt)
# We try to force the change to make sure it was correctly filled to begin with
self.curr.execute(
f'update nodes set modes="hgfedcba" where node_id={y[0]}')
self.curr.execute(
f'select node_id, modes from nodes where node_id={y[0]}')
k = self.curr.fetchone()
self.curr.execute(
f'update nodes set modes="abcdefgq" where node_id={y[0]}')
self.curr.execute(
f'select node_id, modes from nodes where node_id={y[0]}')
z = self.curr.fetchone()
self.assertEqual(
z, k, 'Failed to preserve the information on modes for the nodes')
class TestNetwork_skimming(TestCase):
def setUp(self) -> None:
os.environ['PATH'] = os.path.join(
gettempdir(), 'temp_data') + ';' + os.environ['PATH']
self.proj_dir = os.path.join(gettempdir(), uuid.uuid4().hex)
copytree(siouxfalls_project, self.proj_dir)
self.project = Project()
self.project.open(self.proj_dir)
self.network = self.project.network
self.curr = self.project.conn.cursor()
def tearDown(self) -> None:
self.project.close()
del self.curr
try:
rmtree(self.proj_dir)
except Exception as e:
print(f'Failed to remove at {e.args}')
def test_network_skimming(self):
self.network.build_graphs()
graph = self.network.graphs['c']
graph.set_graph(cost_field="distance")
graph.set_skimming("distance")
# skimming results
res = SkimResults()
res.prepare(graph)
aux_res = MultiThreadedNetworkSkimming()
aux_res.prepare(graph, res)
_ = skimming_single_origin(12, graph, res, aux_res, 0)
skm = NetworkSkimming(graph)
skm.execute()
tot = np.nanmax(skm.results.skims.distance[:, :])
if tot > np.sum(graph.cost):
self.fail(
"Skimming was not successful. At least one np.inf returned.")
if skm.report:
self.fail("Skimming returned an error:" + str(skm.report))
skm.save_to_project('tEst_Skimming')
if not os.path.isfile(
os.path.join(self.proj_dir, 'matrices', 'test_skimming.omx')):
self.fail('Did not save project to project')
matrices = self.project.matrices
mat = matrices.get_record('TEsT_Skimming')
self.assertEqual(mat.name, 'tEst_Skimming',
'Matrix record name saved wrong')
self.assertEqual(mat.file_name, 'tEst_Skimming.omx',
'matrix file_name saved wrong')
self.assertEqual(mat.cores, 1,
'matrix saved number of matrix cores wrong')
self.assertEqual(mat.procedure, 'Network skimming',
'Matrix saved wrong procedure name')
self.assertEqual(mat.procedure_id, skm.procedure_id,
'Procedure ID saved wrong')
self.assertEqual(mat.timestamp, skm.procedure_date,
'Procedure ID saved wrong')
