def load_households(households_file: str): open_file = multiopen(households_file, 'rt') households = csv.reader(open_file, delimiter=',', quotechar='"') next(households) households = counter(households, 'Parsing household %s.') for household in households: yield [int(h) for h in household[0:2]] + [float(household[2])] + \ [int(h) for h in household[3:18]] open_file.close()
def load_trips(trips_file: str): open_file = multiopen(trips_file, 'rt') trips = csv.reader(open_file, delimiter=',', quotechar='"') next(trips) trips = counter(trips, 'Parsing trip %s.') for trip in trips: yield [int(t) for t in trip[0:6]] + [trip[6]] + \ [int(t) for t in trip[7:15]] + [float(t) for t in trip[15:18]] open_file.close()
def load_persons(persons_file: str): open_file = multiopen(persons_file, 'rt') persons = csv.reader(open_file, delimiter=',', quotechar='"') next(persons) persons = counter(persons, 'Loading person %s.') for person in persons: yield [int(p) for p in person[0:2]] + [float(person[2])] + \ [int(p) for p in person[3:18]] + [person[18]] + \ [int(p) for p in person[19:37]] + [person[37]] + [int(person[38])] open_file.close()
def load_routes(self, planspath: str): plansfile = multiopen(planspath, mode='rb') plans = iter(iterparse(plansfile, events=('start', 'end'))) evt, root = next(plans) agent = None selected = False mode = None count = 0 n = 1 log.info('Fetching output plans routing data.') for evt, elem in plans: if evt == 'start': if elem.tag == 'person': agent = elem.get('id') elif elem.tag == 'plan': selected = elem.get('selected') == 'yes' elif elem.tag == 'leg': mode = elem.get('mode') elif evt == 'end': if elem.tag == 'route' and selected: vehicle = elem.get('vehicleRefId') kind = elem.get('type') if vehicle == 'null' and kind == 'links': start = elem.get('start_link') end = elem.get('end_link') distance = float(elem.get('distance')) path = (self.links[link] for link in elem.text.split(' ')) uuid = f'{mode}-{start}-{end}' route = Route(self.links[start], self.links[end], tuple(path), distance, LegMode(mode)) self.agents[agent].routes[uuid] = route elif elem.tag == 'person': count += 1 if count % 10000 == 0: root.clear() if count == n: log.info(f'Processing route {count}.') n <<= 1 if count != (n >> 1): log.info(f'Processing route {count}.') plansfile.close() self.agents.lock()
def load_households(self, households_file): open_file = multiopen(households_file, 'rt') households = csv.reader(open_file, delimiter=',', quotechar='"') next(households) count = 0 n = 1 for household in households: yield [int(h) for h in household[0:2]] + [float(household[2])] + \ [int(h) for h in household[3:17]] count += 1 if count == n: log.info(f'Parsing household {count}.') n <<= 1 if count != n << 1: log.info(f'Parsing household {count}.') open_file.close()
def load_trips(self, trips_file): open_file = multiopen(trips_file, 'rt') trips = csv.reader(open_file, delimiter=',', quotechar='"') next(trips) count = 0 n = 1 for trip in trips: yield [int(t) for t in trip[0:6]] + [trip[6]] + \ [int(t) for t in trip[7:15]] + [float(t) for t in trip[15:18]] count += 1 if count == n: log.info(f'Parsing trip {count}.') n <<= 1 if count != n << 1: log.info(f'Parsing trip {count}.') open_file.close()
def load_persons(self, persons_file): open_file = multiopen(persons_file, 'rt') persons = csv.reader(open_file, delimiter=',', quotechar='"') next(persons) count = 0 n = 1 for person in persons: yield [int(p) for p in person[0:2]] + [float(person[2])] + \ [int(p) for p in person[3:18]] + [person[18]] + \ [int(p) for p in person[19:37]] + [person[37]] + [int(person[38])] count += 1 if count == n: log.info(f'Parsing person {count}.') n <<= 1 if count != n << 1: log.info(f'Parsing person {count}.') open_file.close()
def generate(self, planspath, vehiclespath, modes, sample_percent=1, sample_size=math.inf, transit=None, vehicle=None, walk=None, bike=None, party=None): log.info('Creating a sample population.') conditions = { 'transit': transit, 'vehicle': vehicle, 'walk': walk, 'bike': bike, 'party': party } max_size = self.database.count_rows('agents') size = min(max_size * sample_percent, sample_size) table = 'agents' if size < max_size or any(cond is not None for cond in conditions.values()): table = 'sample' self.create_sample(size, **conditions) actual = self.database.count_rows('sample') if actual < size: log.info(f'Target sample was {size} but only found {actual} ' 'agents under specified parameters.') log.info('Fetching agents, activities and legs.') agents = self.fetch_agents(table) activities = self.fetch_activities(table) legs = self.fetch_legs(table) log.info('Iterating over plans and generating plans file.') touch(planspath) plansfile = multiopen(planspath, mode='wt') plansfile.write( '<?xml version="1.0" encoding="utf-8"?><!DOCTYPE plans' ' SYSTEM "http://www.matsim.org/files/dtd/plans_v4.dtd"><plans>') for agent_id, plan_size in counter(agents, 'Writing plan %s.'): plansfile.write(f'<person id="{agent_id}"><plan selected="yes">') plansfile.write(next(activities).encode_start()) for _ in range(plan_size // 2 - 1): leg = next(legs) activity = next(activities) plansfile.write(leg.encode(activity)) plansfile.write(activity.encode()) leg = next(legs) activity = next(activities) plansfile.write(leg.encode(activity)) plansfile.write(activity.encode_end()) plansfile.write('</plan></person>') plansfile.flush() plansfile.write('</plans>') log.info('Writing vehicle definitions file.') touch(vehiclespath) vehiclesfile = multiopen(vehiclespath, mode='wt') vehiclesfile.write('''<?xml version="1.0" encoding="UTF-8" ?> <vehicleDefinitions xmlns="http://www.matsim.org/files/dtd" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.matsim.org/files/dtd http://www.matsim.org/files/dtd/vehicleDefinitions_v2.0.xsd">''' ) vehiclesfile.write(''' <vehicleType id="Bus"> <attributes> <attribute name="accessTimeInSecondsPerPerson" class="java.lang.Double">0.5</attribute> <attribute name="doorOperationMode" class="java.lang.String">serial</attribute> <attribute name="egressTimeInSecondsPerPerson" class="java.lang.Double">0.5</attribute> </attributes> <capacity seats="70" standingRoomInPersons="0"/> <length meter="18.0"/> <width meter="2.5"/> <passengerCarEquivalents pce="2.8"/> <networkMode networkMode="Bus"/> <flowEfficiencyFactor factor="1.0"/> </vehicleType>''') vehiclesfile.write(''' <vehicleType id="Tram"> <attributes> <attribute name="accessTimeInSecondsPerPerson" class="java.lang.Double">0.25</attribute> <attribute name="doorOperationMode" class="java.lang.String">serial</attribute> <attribute name="egressTimeInSecondsPerPerson" class="java.lang.Double">0.25</attribute> </attributes> <capacity seats="180" standingRoomInPersons="0"/> <length meter="36.0"/> <width meter="2.4"/> <passengerCarEquivalents pce="5.2"/> <networkMode networkMode="Tram"/> <flowEfficiencyFactor factor="1.0"/> </vehicleType>''') vehiclesfile.write(''' <vehicleType id="car"> <attributes> <attribute name="accessTimeInSecondsPerPerson" class="java.lang.Double">1.0</attribute> <attribute name="doorOperationMode" class="java.lang.String">serial</attribute> <attribute name="egressTimeInSecondsPerPerson" class="java.lang.Double">1.0</attribute> </attributes> <capacity seats="5" standingRoomInPersons="0"/> <length meter="7.5"/> <width meter="1.0"/> <maximumVelocity meterPerSecond="40.0"/> <passengerCarEquivalents pce="1.0"/> <networkMode networkMode="car"/> <flowEfficiencyFactor factor="1.0"/> </vehicleType>''') vehiclesfile.write(''' <vehicleType id="bike"> <attributes> <attribute name="accessTimeInSecondsPerPerson" class="java.lang.Double">1.0</attribute> <attribute name="doorOperationMode" class="java.lang.String">serial</attribute> <attribute name="egressTimeInSecondsPerPerson" class="java.lang.Double">1.0</attribute> </attributes> <capacity seats="1" standingRoomInPersons="0"/> <length meter="5.0"/> <width meter="1.0"/> <maximumVelocity meterPerSecond="4.4704"/> <passengerCarEquivalents pce="0.25"/> <networkMode networkMode="bike"/> <flowEfficiencyFactor factor="1.0"/> </vehicleType>''') vehiclesfile.write(''' <vehicleType id="netwalk"> <attributes> <attribute name="accessTimeInSecondsPerPerson" class="java.lang.Double">1.0</attribute> <attribute name="doorOperationMode" class="java.lang.String">serial</attribute> <attribute name="egressTimeInSecondsPerPerson" class="java.lang.Double">1.0</attribute> </attributes> <capacity seats="1" standingRoomInPersons="0"/> <length meter="1.0"/> <width meter="1.0"/> <maximumVelocity meterPerSecond="1.4"/> <passengerCarEquivalents pce="0.0"/> <networkMode networkMode="netwalk"/> <flowEfficiencyFactor factor="1.0"/> </vehicleType>''') vehiclesfile.write('</vehicleDefinitions>') vehiclesfile.close() log.info('Cleaning up.') self.delete_sample()
def parse(self, planspath, eventspath): log.info('Reallocating tables for simulation output data.') self.create_tables() log.info('Loading and building network.') network = Network(self.database) network.load_network(planspath) population = Population(network) log.info('Loading input data identifications.') self.load_activities() self.load_legs() Activity.activities = self.activities Leg.legs = self.legs log.info('Decompressing and loading events file.') eventsfile = multiopen(eventspath, mode='rb') events = iter(iterparse(eventsfile, events=('start', 'end'))) evt, root = next(events) count = 0 time = 14400 n = 14400 log.info('Iterating over simulation events and parsing data.') for evt, elem in events: if evt == 'end' and elem.tag == 'event': time = int(float(elem.get('time'))) population.parse_event(elem) count += 1 if time >= n: log.info(f'Simulation events parsing at {hhmmss(time)} ' f'with {count} events processed.') n += 3600 if count % 1000000 == 0: root.clear() log.debug('Exporting finished activities, legs and events.') activities = population.export_activities() events = population.export_events() legs = population.export_legs() log.debug('Pushing parsed event data to database.') self.database.insert_values('output_activities', activities, 9) self.database.insert_values('output_events', events, 8) self.database.insert_values('output_legs', legs, 8) self.database.connection.commit() root.clear() log.info('Simulation events iteration complete; cleaning up.') log.debug('Closing final activities.') population.cleanup(time) log.debug('Exporting finished activities, legs and events.') activities = population.export_activities() events = population.export_events() legs = population.export_legs() agents = population.export_agents() log.debug('Pushing parsed event data to database.') self.database.insert_values('output_agents', agents, 3) self.database.insert_values('output_activities', activities, 9) self.database.insert_values('output_events', events, 8) self.database.insert_values('output_legs', legs, 8) log.info('Creating indexes on new tables.') self.create_indexes()
def parse_roads(database: SqliteUtil, networkpath: str, src_epsg: int, prj_epsg: int): log.info('Allocating tables for network links and nodes.') create_tables(database) log.info('Loading network roads file.') network = multiopen(networkpath, mode='rb') parser = iter(iterparse(network, events=('start', 'end'))) evt, root = next(parser) transformer = Transformer.from_crs(f'epsg:{src_epsg}', f'epsg:{prj_epsg}', always_xy=True, skip_equivalent=True) project = transformer.transform links = [] nodes = [] count, n = 0, 1 for evt, elem in parser: if evt == 'start': if elem.tag == 'nodes': log.info('Parsing nodes from network file.') count, n = 0, 1 root.clear() elif elem.tag == 'links': if count != n << 1: log.info(f'Parsing node {count}.') log.info('Parsing links from network file.') count, n = 0, 1 root.clear() elif evt == 'end': if elem.tag == 'node': node_id = str(elem.get('id')) x = float(elem.get('x')) y = float(elem.get('y')) x, y = project(x, y) wkt = f'POINT ({x} {y})' nodes.append((node_id, None, wkt)) count += 1 if count == n: log.info(f'Parsing node {count}.') n <<= 1 if count % 100000 == 0: root.clear() elif elem.tag == 'link': source_node = str(elem.get('from')) terminal_node = str(elem.get('to')) links.append( (str(elem.get('id')), source_node, terminal_node, float(elem.get('length')), float(elem.get('freespeed')), float(elem.get('capacity')), float(elem.get('permlanes')), int(elem.get('oneway')), str(elem.get('modes')), None, None)) count += 1 if count == n: log.info(f'Parsing link {count}.') n <<= 1 if count % 100000 == 0: root.clear() if count != n << 1: log.info(f'Parsing link {count}.') network.close() log.info('Writing parsed links and nodes to database.') database.insert_values('nodes', nodes, 3) database.insert_values('links', links, 11) database.connection.commit() log.info('Creating indexes on new tables.') create_indexes(database)
def parse(self, networkpath): log.info('Fetching exposure geospatial data.') centroids_list = self.fetch_centroids() log.info('Loading centroids into spatial index.') centroids = {} for uuid, centroid in counter(centroids_list, 'Loading centroid %s.'): x, y = map(float, centroid[7:-1].split(' ')) centroids[uuid] = Centroid(uuid, x, y) centroid_idx = index.Index(centroid.entry() for centroid in centroids.values()) del centroids_list log.info('Fetching network region data.') regions_list = self.fetch_regions() log.info('Loading regions into spatial index.') regions = {} for uuid, region in counter(regions_list, 'Loading region %s.'): polygon = loads(region) setattr(polygon, 'maz', uuid) regions[uuid] = Region(uuid, polygon) region_idx = STRtree(region.region for region in regions.values()) del regions_list def get_centroid(node): return next(centroid_idx.nearest(node, 1)) def get_region(node: Point): regions = region_idx.query(node) region = None if len(regions): region = regions[0].maz return region log.info('Loading network roads file.') network = multiopen(networkpath, mode='rb') parser = iter(iterparse(network, events=('start', 'end'))) evt, root = next(parser) links = [] nodes = [] points = {} count = 0 n = 1 for evt, elem in parser: if evt == 'start': if elem.tag == 'nodes': log.info('Parsing nodes from network file.') count = 0 n = 1 elif elem.tag == 'links': if count != n << 1: log.info(f'Parsed node {count}.') log.info('Parsing links from network file.') count = 0 n = 1 elif evt == 'end': if elem.tag == 'node': node_id = str(elem.get('id')) x = float(elem.get('x')) y = float(elem.get('y')) point = Point(x, y) region = get_region(point) centroid = get_centroid((x, y, x, y)) nodes.append((node_id, region, centroid, dumps(point))) points[node_id] = point count += 1 if count == n: log.info(f'Parsing node {count}.') n <<= 1 elif elem.tag == 'link': source_node = str(elem.get('from')) terminal_node = str(elem.get('to')) line = LineString( (points[source_node], points[terminal_node])) links.append( (str(elem.get('id')), source_node, terminal_node, float(elem.get('length')), float(elem.get('freespeed')), float(elem.get('capacity')), float(elem.get('permlanes')), int(elem.get('oneway')), str(elem.get('modes')), dumps(line))) count += 1 if count == n: log.info(f'Parsing link {count}.') n <<= 1 if count % 100000: root.clear() if count != n << 1: log.info(f'Parsing link {count}.') network.close() log.info('Writing parsed links and nodes to database.') self.create_tables() self.database.insert_values('nodes', nodes, 4) self.database.insert_values('links', links, 10) self.database.connection.commit()