def load_households(households_file: str):
open_file = multiopen(households_file, 'rt')
households = csv.reader(open_file, delimiter=',', quotechar='"')
next(households)
households = counter(households, 'Parsing household %s.')
for household in households:
yield [int(h) for h in household[0:2]] + [float(household[2])] + \
[int(h) for h in household[3:18]]
open_file.close()
def load_trips(trips_file: str):
open_file = multiopen(trips_file, 'rt')
trips = csv.reader(open_file, delimiter=',', quotechar='"')
next(trips)
trips = counter(trips, 'Parsing trip %s.')
for trip in trips:
yield [int(t) for t in trip[0:6]] + [trip[6]] + \
[int(t) for t in trip[7:15]] + [float(t) for t in trip[15:18]]
open_file.close()
def load_persons(persons_file: str):
open_file = multiopen(persons_file, 'rt')
persons = csv.reader(open_file, delimiter=',', quotechar='"')
next(persons)
persons = counter(persons, 'Loading person %s.')
for person in persons:
yield [int(p) for p in person[0:2]] + [float(person[2])] + \
[int(p) for p in person[3:18]] + [person[18]] + \
[int(p) for p in person[19:37]] + [person[37]] + [int(person[38])]
open_file.close()
def load_routes(self, planspath: str):
plansfile = multiopen(planspath, mode='rb')
plans = iter(iterparse(plansfile, events=('start', 'end')))
evt, root = next(plans)
agent = None
selected = False
mode = None
count = 0
n = 1
log.info('Fetching output plans routing data.')
for evt, elem in plans:
if evt == 'start':
if elem.tag == 'person':
agent = elem.get('id')
elif elem.tag == 'plan':
selected = elem.get('selected') == 'yes'
elif elem.tag == 'leg':
mode = elem.get('mode')
elif evt == 'end':
if elem.tag == 'route' and selected:
vehicle = elem.get('vehicleRefId')
kind = elem.get('type')
if vehicle == 'null' and kind == 'links':
start = elem.get('start_link')
end = elem.get('end_link')
distance = float(elem.get('distance'))
path = (self.links[link]
for link in elem.text.split(' '))
uuid = f'{mode}-{start}-{end}'
route = Route(self.links[start], self.links[end],
tuple(path), distance, LegMode(mode))
self.agents[agent].routes[uuid] = route
elif elem.tag == 'person':
count += 1
if count % 10000 == 0:
root.clear()
if count == n:
log.info(f'Processing route {count}.')
n <<= 1
if count != (n >> 1):
log.info(f'Processing route {count}.')
plansfile.close()
self.agents.lock()
def load_households(self, households_file):
open_file = multiopen(households_file, 'rt')
households = csv.reader(open_file, delimiter=',', quotechar='"')
next(households)
count = 0
n = 1
for household in households:
yield [int(h) for h in household[0:2]] + [float(household[2])] + \
[int(h) for h in household[3:17]]
count += 1
if count == n:
log.info(f'Parsing household {count}.')
n <<= 1
if count != n << 1:
log.info(f'Parsing household {count}.')
open_file.close()
def load_trips(self, trips_file):
open_file = multiopen(trips_file, 'rt')
trips = csv.reader(open_file, delimiter=',', quotechar='"')
next(trips)
count = 0
n = 1
for trip in trips:
yield [int(t) for t in trip[0:6]] + [trip[6]] + \
[int(t) for t in trip[7:15]] + [float(t) for t in trip[15:18]]
count += 1
if count == n:
log.info(f'Parsing trip {count}.')
n <<= 1
if count != n << 1:
log.info(f'Parsing trip {count}.')
open_file.close()
def load_persons(self, persons_file):
open_file = multiopen(persons_file, 'rt')
persons = csv.reader(open_file, delimiter=',', quotechar='"')
next(persons)
count = 0
n = 1
for person in persons:
yield [int(p) for p in person[0:2]] + [float(person[2])] + \
[int(p) for p in person[3:18]] + [person[18]] + \
[int(p) for p in person[19:37]] + [person[37]] + [int(person[38])]
count += 1
if count == n:
log.info(f'Parsing person {count}.')
n <<= 1
if count != n << 1:
log.info(f'Parsing person {count}.')
open_file.close()
def generate(self,
planspath,
vehiclespath,
modes,
sample_percent=1,
sample_size=math.inf,
transit=None,
vehicle=None,
walk=None,
bike=None,
party=None):
log.info('Creating a sample population.')
conditions = {
'transit': transit,
'vehicle': vehicle,
'walk': walk,
'bike': bike,
'party': party
}
max_size = self.database.count_rows('agents')
size = min(max_size * sample_percent, sample_size)
table = 'agents'
if size < max_size or any(cond is not None
for cond in conditions.values()):
table = 'sample'
self.create_sample(size, **conditions)
actual = self.database.count_rows('sample')
if actual < size:
log.info(f'Target sample was {size} but only found {actual} '
'agents under specified parameters.')
log.info('Fetching agents, activities and legs.')
agents = self.fetch_agents(table)
activities = self.fetch_activities(table)
legs = self.fetch_legs(table)
log.info('Iterating over plans and generating plans file.')
touch(planspath)
plansfile = multiopen(planspath, mode='wt')
plansfile.write(
'<?xml version="1.0" encoding="utf-8"?><!DOCTYPE plans'
' SYSTEM "http://www.matsim.org/files/dtd/plans_v4.dtd"><plans>')
for agent_id, plan_size in counter(agents, 'Writing plan %s.'):
plansfile.write(f'<person id="{agent_id}"><plan selected="yes">')
plansfile.write(next(activities).encode_start())
for _ in range(plan_size // 2 - 1):
leg = next(legs)
activity = next(activities)
plansfile.write(leg.encode(activity))
plansfile.write(activity.encode())
leg = next(legs)
activity = next(activities)
plansfile.write(leg.encode(activity))
plansfile.write(activity.encode_end())
plansfile.write('</plan></person>')
plansfile.flush()
plansfile.write('</plans>')
log.info('Writing vehicle definitions file.')
touch(vehiclespath)
vehiclesfile = multiopen(vehiclespath, mode='wt')
vehiclesfile.write('''<?xml version="1.0" encoding="UTF-8" ?>
<vehicleDefinitions
xmlns="http://www.matsim.org/files/dtd"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.matsim.org/files/dtd
http://www.matsim.org/files/dtd/vehicleDefinitions_v2.0.xsd">'''
)
vehiclesfile.write('''
<vehicleType id="Bus">
<attributes>
<attribute name="accessTimeInSecondsPerPerson" class="java.lang.Double">0.5</attribute>
<attribute name="doorOperationMode" class="java.lang.String">serial</attribute>
<attribute name="egressTimeInSecondsPerPerson" class="java.lang.Double">0.5</attribute>
</attributes>
<capacity seats="70" standingRoomInPersons="0"/>
<length meter="18.0"/>
<width meter="2.5"/>
<passengerCarEquivalents pce="2.8"/>
<networkMode networkMode="Bus"/>
<flowEfficiencyFactor factor="1.0"/>
</vehicleType>''')
vehiclesfile.write('''
<vehicleType id="Tram">
<attributes>
<attribute name="accessTimeInSecondsPerPerson" class="java.lang.Double">0.25</attribute>
<attribute name="doorOperationMode" class="java.lang.String">serial</attribute>
<attribute name="egressTimeInSecondsPerPerson" class="java.lang.Double">0.25</attribute>
</attributes>
<capacity seats="180" standingRoomInPersons="0"/>
<length meter="36.0"/>
<width meter="2.4"/>
<passengerCarEquivalents pce="5.2"/>
<networkMode networkMode="Tram"/>
<flowEfficiencyFactor factor="1.0"/>
</vehicleType>''')
vehiclesfile.write('''
<vehicleType id="car">
<attributes>
<attribute name="accessTimeInSecondsPerPerson" class="java.lang.Double">1.0</attribute>
<attribute name="doorOperationMode" class="java.lang.String">serial</attribute>
<attribute name="egressTimeInSecondsPerPerson" class="java.lang.Double">1.0</attribute>
</attributes>
<capacity seats="5" standingRoomInPersons="0"/>
<length meter="7.5"/>
<width meter="1.0"/>
<maximumVelocity meterPerSecond="40.0"/>
<passengerCarEquivalents pce="1.0"/>
<networkMode networkMode="car"/>
<flowEfficiencyFactor factor="1.0"/>
</vehicleType>''')
vehiclesfile.write('''
<vehicleType id="bike">
<attributes>
<attribute name="accessTimeInSecondsPerPerson" class="java.lang.Double">1.0</attribute>
<attribute name="doorOperationMode" class="java.lang.String">serial</attribute>
<attribute name="egressTimeInSecondsPerPerson" class="java.lang.Double">1.0</attribute>
</attributes>
<capacity seats="1" standingRoomInPersons="0"/>
<length meter="5.0"/>
<width meter="1.0"/>
<maximumVelocity meterPerSecond="4.4704"/>
<passengerCarEquivalents pce="0.25"/>
<networkMode networkMode="bike"/>
<flowEfficiencyFactor factor="1.0"/>
</vehicleType>''')
vehiclesfile.write('''
<vehicleType id="netwalk">
<attributes>
<attribute name="accessTimeInSecondsPerPerson" class="java.lang.Double">1.0</attribute>
<attribute name="doorOperationMode" class="java.lang.String">serial</attribute>
<attribute name="egressTimeInSecondsPerPerson" class="java.lang.Double">1.0</attribute>
</attributes>
<capacity seats="1" standingRoomInPersons="0"/>
<length meter="1.0"/>
<width meter="1.0"/>
<maximumVelocity meterPerSecond="1.4"/>
<passengerCarEquivalents pce="0.0"/>
<networkMode networkMode="netwalk"/>
<flowEfficiencyFactor factor="1.0"/>
</vehicleType>''')
vehiclesfile.write('</vehicleDefinitions>')
vehiclesfile.close()
log.info('Cleaning up.')
self.delete_sample()
def parse(self, planspath, eventspath):
log.info('Reallocating tables for simulation output data.')
self.create_tables()
log.info('Loading and building network.')
network = Network(self.database)
network.load_network(planspath)
population = Population(network)
log.info('Loading input data identifications.')
self.load_activities()
self.load_legs()
Activity.activities = self.activities
Leg.legs = self.legs
log.info('Decompressing and loading events file.')
eventsfile = multiopen(eventspath, mode='rb')
events = iter(iterparse(eventsfile, events=('start', 'end')))
evt, root = next(events)
count = 0
time = 14400
n = 14400
log.info('Iterating over simulation events and parsing data.')
for evt, elem in events:
if evt == 'end' and elem.tag == 'event':
time = int(float(elem.get('time')))
population.parse_event(elem)
count += 1
if time >= n:
log.info(f'Simulation events parsing at {hhmmss(time)} '
f'with {count} events processed.')
n += 3600
if count % 1000000 == 0:
root.clear()
log.debug('Exporting finished activities, legs and events.')
activities = population.export_activities()
events = population.export_events()
legs = population.export_legs()
log.debug('Pushing parsed event data to database.')
self.database.insert_values('output_activities', activities, 9)
self.database.insert_values('output_events', events, 8)
self.database.insert_values('output_legs', legs, 8)
self.database.connection.commit()
root.clear()
log.info('Simulation events iteration complete; cleaning up.')
log.debug('Closing final activities.')
population.cleanup(time)
log.debug('Exporting finished activities, legs and events.')
activities = population.export_activities()
events = population.export_events()
legs = population.export_legs()
agents = population.export_agents()
log.debug('Pushing parsed event data to database.')
self.database.insert_values('output_agents', agents, 3)
self.database.insert_values('output_activities', activities, 9)
self.database.insert_values('output_events', events, 8)
self.database.insert_values('output_legs', legs, 8)
log.info('Creating indexes on new tables.')
self.create_indexes()
def parse_roads(database: SqliteUtil, networkpath: str, src_epsg: int,
prj_epsg: int):
log.info('Allocating tables for network links and nodes.')
create_tables(database)
log.info('Loading network roads file.')
network = multiopen(networkpath, mode='rb')
parser = iter(iterparse(network, events=('start', 'end')))
evt, root = next(parser)
transformer = Transformer.from_crs(f'epsg:{src_epsg}',
f'epsg:{prj_epsg}',
always_xy=True,
skip_equivalent=True)
project = transformer.transform
links = []
nodes = []
count, n = 0, 1
for evt, elem in parser:
if evt == 'start':
if elem.tag == 'nodes':
log.info('Parsing nodes from network file.')
count, n = 0, 1
root.clear()
elif elem.tag == 'links':
if count != n << 1:
log.info(f'Parsing node {count}.')
log.info('Parsing links from network file.')
count, n = 0, 1
root.clear()
elif evt == 'end':
if elem.tag == 'node':
node_id = str(elem.get('id'))
x = float(elem.get('x'))
y = float(elem.get('y'))
x, y = project(x, y)
wkt = f'POINT ({x} {y})'
nodes.append((node_id, None, wkt))
count += 1
if count == n:
log.info(f'Parsing node {count}.')
n <<= 1
if count % 100000 == 0:
root.clear()
elif elem.tag == 'link':
source_node = str(elem.get('from'))
terminal_node = str(elem.get('to'))
links.append(
(str(elem.get('id')), source_node, terminal_node,
float(elem.get('length')), float(elem.get('freespeed')),
float(elem.get('capacity')), float(elem.get('permlanes')),
int(elem.get('oneway')), str(elem.get('modes')), None,
None))
count += 1
if count == n:
log.info(f'Parsing link {count}.')
n <<= 1
if count % 100000 == 0:
root.clear()
if count != n << 1:
log.info(f'Parsing link {count}.')
network.close()
log.info('Writing parsed links and nodes to database.')
database.insert_values('nodes', nodes, 3)
database.insert_values('links', links, 11)
database.connection.commit()
log.info('Creating indexes on new tables.')
create_indexes(database)
def parse(self, networkpath):
log.info('Fetching exposure geospatial data.')
centroids_list = self.fetch_centroids()
log.info('Loading centroids into spatial index.')
centroids = {}
for uuid, centroid in counter(centroids_list, 'Loading centroid %s.'):
x, y = map(float, centroid[7:-1].split(' '))
centroids[uuid] = Centroid(uuid, x, y)
centroid_idx = index.Index(centroid.entry()
for centroid in centroids.values())
del centroids_list
log.info('Fetching network region data.')
regions_list = self.fetch_regions()
log.info('Loading regions into spatial index.')
regions = {}
for uuid, region in counter(regions_list, 'Loading region %s.'):
polygon = loads(region)
setattr(polygon, 'maz', uuid)
regions[uuid] = Region(uuid, polygon)
region_idx = STRtree(region.region for region in regions.values())
del regions_list
def get_centroid(node):
return next(centroid_idx.nearest(node, 1))
def get_region(node: Point):
regions = region_idx.query(node)
region = None
if len(regions):
region = regions[0].maz
return region
log.info('Loading network roads file.')
network = multiopen(networkpath, mode='rb')
parser = iter(iterparse(network, events=('start', 'end')))
evt, root = next(parser)
links = []
nodes = []
points = {}
count = 0
n = 1
for evt, elem in parser:
if evt == 'start':
if elem.tag == 'nodes':
log.info('Parsing nodes from network file.')
count = 0
n = 1
elif elem.tag == 'links':
if count != n << 1:
log.info(f'Parsed node {count}.')
log.info('Parsing links from network file.')
count = 0
n = 1
elif evt == 'end':
if elem.tag == 'node':
node_id = str(elem.get('id'))
x = float(elem.get('x'))
y = float(elem.get('y'))
point = Point(x, y)
region = get_region(point)
centroid = get_centroid((x, y, x, y))
nodes.append((node_id, region, centroid, dumps(point)))
points[node_id] = point
count += 1
if count == n:
log.info(f'Parsing node {count}.')
n <<= 1
elif elem.tag == 'link':
source_node = str(elem.get('from'))
terminal_node = str(elem.get('to'))
line = LineString(
(points[source_node], points[terminal_node]))
links.append(
(str(elem.get('id')), source_node, terminal_node,
float(elem.get('length')),
float(elem.get('freespeed')),
float(elem.get('capacity')),
float(elem.get('permlanes')), int(elem.get('oneway')),
str(elem.get('modes')), dumps(line)))
count += 1
if count == n:
log.info(f'Parsing link {count}.')
n <<= 1
if count % 100000:
root.clear()
if count != n << 1:
log.info(f'Parsing link {count}.')
network.close()
log.info('Writing parsed links and nodes to database.')
self.create_tables()
self.database.insert_values('nodes', nodes, 4)
self.database.insert_values('links', links, 10)
self.database.connection.commit()
