def parse_regions(database: SqliteUtil, regions_file: str, src_epsg: int, prj_epsg: int): log.info('Allocating tables for regions.') create_tables(database) transformer = Transformer.from_crs(f'epsg:{src_epsg}', f'epsg:{prj_epsg}', always_xy=True, skip_equivalent=True) project = transformer.transform log.info('Parsing regions from shapefile.') parser = shapefile.Reader(regions_file) iter_regions = counter(iter(parser), 'Parsing region %s.') regions = [] for item in iter_regions: points = (project(*point) for point in item.shape.points) polygon = Polygon(points) regions.append( (item.record.MAZ_ID_10, item.record.TAZ_2015, item.record.Sq_miles, dumps(polygon.centroid), dumps(polygon))) parser.close() log.info('Writing parsed regions to database.') database.insert_values('regions', regions, 5) database.connection.commit() log.info('Creating indexes on new tables.') create_indexes(database)
def load_links(database: SqliteUtil): query = ''' SELECT links.link_id, links.mrt_temperature, nodes1.point AS source_point, nodes2.point AS terminal_point FROM links INNER JOIN nodes AS nodes1 ON links.source_node = nodes1.node_id INNER JOIN nodes AS nodes2 ON links.terminal_node = nodes2.node_id; ''' database.cursor.execute(query) rows = database.fetch_rows() rows = counter(rows, 'Loading link %s.') bounds = lambda x, y: min(x) > 0.5e6 and max(x) < 0.85e6 and \ min(y) > 0.8e6 and max(y) < 1.0e6 links = [] for link_id, profile, src_pt, term_pt in rows: line = LineString((xy(src_pt), xy(term_pt))) x, y = line.coords.xy if bounds(x, y): link = Link(link_id, line, profile) links.append(link) return links
def load_temperatures(self): log.info('Loading network daymet temperature data.') temperatures = counter(self.fetch_temperatures(), 'Loading temperature %s.') for temperature_id, _, temperature in temperatures: self.temperatures[temperature_id].append(temperature) self.temperatures.lock()
def load_links(self): log.info('Loading network road link data.') query = ''' SELECT link_id, length, freespeed, modes, air_temperature, mrt_temperature FROM links; ''' self.database.cursor.execute(query) result = self.database.cursor.fetchall() links = counter(result, 'Loading link %s.') for link_id, length, speed, modes, air_temp, mrt_temp in links: modes_set = set(NetworkMode(mode) for mode in modes.split(',')) air_temperature = self.air_temperatures[air_temp] mrt_temperature = None if mrt_temp is not None: mrt_temperature = self.mrt_temperatures[mrt_temp] link = Link( link_id, length, speed, modes_set, air_temperature, mrt_temperature ) self.links[link_id] = link
def load_nodes(self): log.info('Loading network road node data.') nodes = counter(self.fetch_nodes(), 'Loading node %s.') for node_id, maz, centroid_id, point in nodes: centroid = self.centroids[centroid_id] x, y = xy(point) self.nodes[node_id] = Node(node_id, maz, centroid, x, y)
def load_temperatures(database: SqliteUtil, kind: str, max_idx: int, min_idx: int): query = f''' SELECT mrt_temperatures.temperature_id, mrt_temperatures.temperature_idx, mrt_temperatures.{kind}, COUNT(*) AS util FROM mrt_temperatures INNER JOIN links ON links.mrt_temperature = mrt_temperatures.temperature_id INNER JOIN output_events ON output_events.link_id = links.link_id WHERE output_events.start >= mrt_temperatures.temperature_idx * 900 AND output_events.end < mrt_temperatures.temperature_idx * 900 + 900 GROUP BY temperature_id, temperature_idx; ''' database.cursor.execute(query) rows = database.fetch_rows() rows = counter(rows, 'Loading temperature profile %s.') temps = defaultdict(lambda: [None] * (max_idx - min_idx + 1)) for uuid, idx, temp, util in rows: if util > 0: temps[uuid][idx - min_idx] = temp return temps
def load_nodes(self): log.info('Loading network road node data.') nodes = counter(self.fetch_nodes(), 'Loading node %s.') for node in nodes: node_id = node[0] maz = node[1] x, y = xy(node[2]) self.nodes[node_id] = Node(node_id, maz, x, y)
def load_centroids(self): log.info('Loading network daymet centroid data.') centroids = counter(self.fetch_centroids(), 'Loading centroid %s.') for centroid_id, temperature_id, center in centroids: temperatures = self.temperatures[temperature_id] x, y = xy(center) self.centroids[centroid_id] = Centroid(centroid_id, temperatures, x, y)
def load_households(households_file: str): open_file = multiopen(households_file, 'rt') households = csv.reader(open_file, delimiter=',', quotechar='"') next(households) households = counter(households, 'Parsing household %s.') for household in households: yield [int(h) for h in household[0:2]] + [float(household[2])] + \ [int(h) for h in household[3:18]] open_file.close()
def load_trips(trips_file: str): open_file = multiopen(trips_file, 'rt') trips = csv.reader(open_file, delimiter=',', quotechar='"') next(trips) trips = counter(trips, 'Parsing trip %s.') for trip in trips: yield [int(t) for t in trip[0:6]] + [trip[6]] + \ [int(t) for t in trip[7:15]] + [float(t) for t in trip[15:18]] open_file.close()
def load_links(self): log.info('Loading network road link data.') links = counter(self.fetch_links(), 'Loading link %s.') for link in links: link_id = link[0] src_node = self.nodes[link[1]] term_node = self.nodes[link[2]] length = link[3] freespeed = link[4] modes = set(NetworkMode(mode) for mode in link[5].split(',')) self.links[link_id] = Link(link_id, src_node, term_node, length, freespeed, modes)
def load_persons(persons_file: str): open_file = multiopen(persons_file, 'rt') persons = csv.reader(open_file, delimiter=',', quotechar='"') next(persons) persons = counter(persons, 'Loading person %s.') for person in persons: yield [int(p) for p in person[0:2]] + [float(person[2])] + \ [int(p) for p in person[3:18]] + [person[18]] + \ [int(p) for p in person[19:37]] + [person[37]] + [int(person[38])] open_file.close()
def load_temperatures(self, kind: str = 'mrt'): log.info('Loading network air temperature data.') query = ''' SELECT temperature_id, temperature_idx, temperature FROM air_temperatures; ''' self.database.cursor.execute(query) rows = self.database.fetch_rows() temps = defaultdict(lambda: [None]*96) rows = counter(rows, 'Loading air temperature %s.') for temperature_id, temperature_idx, temperature in rows: temps[temperature_id][temperature_idx] = temperature for uuid, values in temps.items(): self.air_temperatures[uuid] = Temperature(uuid, values) log.info('Loading network mrt temperature data.') query = f''' SELECT temperature_id, temperature_idx, {kind} FROM mrt_temperatures; ''' self.database.cursor.execute(query) rows = self.database.cursor.fetchall() temps = defaultdict(lambda: [None]*96) rows = counter(rows, 'Loading mrt temperature %s.') for temperature_id, temperature_idx, temperature in rows: temps[temperature_id][temperature_idx] = temperature for uuid, values in temps.items(): self.mrt_temperatures[uuid] = Temperature(uuid, values)
def load_parcels(self): log.info('Loading network parcel data.') query = ''' SELECT apn, air_temperature FROM parcels; ''' self.database.cursor.execute(query) rows = self.database.fetch_rows() rows = counter(rows, 'Loading parcel %s.') for apn, temperature in rows: temp = self.air_temperatures[temperature] parcel = Parcel(apn, temp) self.parcels[apn] = parcel
def load_regions(database: SqliteUtil): query = ''' SELECT maz, region FROM regions; ''' database.cursor.execute(query) rows = database.fetch_rows() rows = counter(rows, 'Loading region %s.') regions = [] for maz, polygon in rows: region = Region(maz, polygon) regions.append(region) return regions
def parse_temperatures(csvfile: str) \ -> Tuple[List[Tuple[float,float,float]],int]: log.info(f'Opening {csvfile}.') csv_file = open(csvfile, 'r') iter_temps = csv.reader(csv_file, delimiter=',', quotechar='"') next(iter_temps) iter_temps = counter(iter_temps, 'Parsing temperature %s.') temps = [] peek = next(iter_temps) iter_temps.send(peek) secs = hhmm_to_secs(peek[2]) for _, _, _, mrt, pet, utci in iter_temps: temps.append((float(mrt), float(pet), float(utci))) csv_file.close() return temps, secs
def load_nodes(database: SqliteUtil) -> Dict[str,Node]: query = ''' SELECT node_id, point FROM nodes; ''' database.cursor.execute(query) rows = database.fetch_rows() rows = counter(rows, 'Loading node %s.') nodes: Dict[str,Node] = {} for uuid, point in rows: x, y = xy(point) node = Node(uuid, x, y) nodes[uuid] = node return nodes
def load_links(database: SqliteUtil): query = ''' SELECT links.link_id, nodes.point FROM links INNER JOIN nodes ON links.source_node = nodes.node_id; ''' database.cursor.execute(query) rows = database.fetch_rows() rows = counter(rows, 'Loading link %s.') links = [] for link_id, point in rows: x, y = xy(point) link = Link(link_id, x, y) links.append(link) return links
def load_parcels(database: SqliteUtil): query = ''' SELECT apn, type, cooling, center FROM parcels; ''' database.cursor.execute(query) rows = database.fetch_rows() rows = counter(rows, 'Loading parcel %s.') parcels = [] for apn, kind, cooling, center in rows: x, y = xy(center) parcel = Parcel(apn, kind, bool(cooling), x, y) parcels.append(parcel) return parcels
def load_links(database: SqliteUtil, nodes: Dict[str,Node]) -> Dict[str,Link]: query = ''' SELECT link_id, source_node, terminal_node, freespeed, permlanes FROM links; ''' database.cursor.execute(query) rows = database.fetch_rows() rows = counter(rows, 'Loading link %s.') links: Dict[str,Link] = {} for uuid, src, term, speed, lanes in rows: source_node = nodes[src] terminal_node = nodes[term] link = Link(uuid, source_node, terminal_node, lanes, speed) links[uuid] = link return links
def load_parcels(self): parcels = self.fetch_parcels() self.residential_parcels = defaultdict(lambda x: []) self.commercial_parcels = defaultdict(lambda x: []) self.default_parcels = {} self.other_parcels = defaultdict(lambda x: []) for apn, maz, kind in counter(parcels, 'Loading parcel %s.'): if kind == 'residential': self.residential_parcels[maz].append(Parcel(apn)) elif kind == 'commercial': self.commercial_parcels[maz].append(Parcel(apn)) elif kind == 'default': self.default_parcels[maz] = Parcel(apn) elif kind == 'other': self.other_parcels[maz].append(Parcel(apn)) self.residential_parcels.lock() self.commercial_parcels.lock() self.other_parcels.lock() self.mazs = set(self.default_parcels.keys()) self.offset = defaultdict(lambda x: 0)
def parse_points(csvfile: str, src_epsg: int, prj_epsg: int) \ -> Tuple[List[Point],int]: log.info(f'Opening {csvfile}.') csv_file = open(csvfile, 'r') iter_points = csv.reader(csv_file, delimiter=',', quotechar='"') next(iter_points) iter_points = counter(iter_points, 'Parsing point %s.') transformer = Transformer.from_crs(f'epsg:{src_epsg}', f'epsg:{prj_epsg}', always_xy=True, skip_equivalent=True) project = transformer.transform points = [] peek = next(iter_points) iter_points.send(peek) secs = hhmm_to_secs(peek[2]) for uuid, (lat, lon, _, mrt, pet, utci) in enumerate(iter_points): x, y = project(lon, lat) point = Point(uuid, x, y, float(mrt), float(pet), float(utci)) points.append(point) csv_file.close() return points, secs
def generate(self, planspath, vehiclespath, modes, sample_percent=1, sample_size=math.inf, transit=None, vehicle=None, walk=None, bike=None, party=None): log.info('Creating a sample population.') conditions = { 'transit': transit, 'vehicle': vehicle, 'walk': walk, 'bike': bike, 'party': party } max_size = self.database.count_rows('agents') size = min(max_size * sample_percent, sample_size) table = 'agents' if size < max_size or any(cond is not None for cond in conditions.values()): table = 'sample' self.create_sample(size, **conditions) actual = self.database.count_rows('sample') if actual < size: log.info(f'Target sample was {size} but only found {actual} ' 'agents under specified parameters.') log.info('Fetching agents, activities and legs.') agents = self.fetch_agents(table) activities = self.fetch_activities(table) legs = self.fetch_legs(table) log.info('Iterating over plans and generating plans file.') touch(planspath) plansfile = multiopen(planspath, mode='wt') plansfile.write( '<?xml version="1.0" encoding="utf-8"?><!DOCTYPE plans' ' SYSTEM "http://www.matsim.org/files/dtd/plans_v4.dtd"><plans>') for agent_id, plan_size in counter(agents, 'Writing plan %s.'): plansfile.write(f'<person id="{agent_id}"><plan selected="yes">') plansfile.write(next(activities).encode_start()) for _ in range(plan_size // 2 - 1): leg = next(legs) activity = next(activities) plansfile.write(leg.encode(activity)) plansfile.write(activity.encode()) leg = next(legs) activity = next(activities) plansfile.write(leg.encode(activity)) plansfile.write(activity.encode_end()) plansfile.write('</plan></person>') plansfile.flush() plansfile.write('</plans>') log.info('Writing vehicle definitions file.') touch(vehiclespath) vehiclesfile = multiopen(vehiclespath, mode='wt') vehiclesfile.write('''<?xml version="1.0" encoding="UTF-8" ?> <vehicleDefinitions xmlns="http://www.matsim.org/files/dtd" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.matsim.org/files/dtd http://www.matsim.org/files/dtd/vehicleDefinitions_v2.0.xsd">''' ) vehiclesfile.write(''' <vehicleType id="Bus"> <attributes> <attribute name="accessTimeInSecondsPerPerson" class="java.lang.Double">0.5</attribute> <attribute name="doorOperationMode" class="java.lang.String">serial</attribute> <attribute name="egressTimeInSecondsPerPerson" class="java.lang.Double">0.5</attribute> </attributes> <capacity seats="70" standingRoomInPersons="0"/> <length meter="18.0"/> <width meter="2.5"/> <passengerCarEquivalents pce="2.8"/> <networkMode networkMode="Bus"/> <flowEfficiencyFactor factor="1.0"/> </vehicleType>''') vehiclesfile.write(''' <vehicleType id="Tram"> <attributes> <attribute name="accessTimeInSecondsPerPerson" class="java.lang.Double">0.25</attribute> <attribute name="doorOperationMode" class="java.lang.String">serial</attribute> <attribute name="egressTimeInSecondsPerPerson" class="java.lang.Double">0.25</attribute> </attributes> <capacity seats="180" standingRoomInPersons="0"/> <length meter="36.0"/> <width meter="2.4"/> <passengerCarEquivalents pce="5.2"/> <networkMode networkMode="Tram"/> <flowEfficiencyFactor factor="1.0"/> </vehicleType>''') vehiclesfile.write(''' <vehicleType id="car"> <attributes> <attribute name="accessTimeInSecondsPerPerson" class="java.lang.Double">1.0</attribute> <attribute name="doorOperationMode" class="java.lang.String">serial</attribute> <attribute name="egressTimeInSecondsPerPerson" class="java.lang.Double">1.0</attribute> </attributes> <capacity seats="5" standingRoomInPersons="0"/> <length meter="7.5"/> <width meter="1.0"/> <maximumVelocity meterPerSecond="40.0"/> <passengerCarEquivalents pce="1.0"/> <networkMode networkMode="car"/> <flowEfficiencyFactor factor="1.0"/> </vehicleType>''') vehiclesfile.write(''' <vehicleType id="bike"> <attributes> <attribute name="accessTimeInSecondsPerPerson" class="java.lang.Double">1.0</attribute> <attribute name="doorOperationMode" class="java.lang.String">serial</attribute> <attribute name="egressTimeInSecondsPerPerson" class="java.lang.Double">1.0</attribute> </attributes> <capacity seats="1" standingRoomInPersons="0"/> <length meter="5.0"/> <width meter="1.0"/> <maximumVelocity meterPerSecond="4.4704"/> <passengerCarEquivalents pce="0.25"/> <networkMode networkMode="bike"/> <flowEfficiencyFactor factor="1.0"/> </vehicleType>''') vehiclesfile.write(''' <vehicleType id="netwalk"> <attributes> <attribute name="accessTimeInSecondsPerPerson" class="java.lang.Double">1.0</attribute> <attribute name="doorOperationMode" class="java.lang.String">serial</attribute> <attribute name="egressTimeInSecondsPerPerson" class="java.lang.Double">1.0</attribute> </attributes> <capacity seats="1" standingRoomInPersons="0"/> <length meter="1.0"/> <width meter="1.0"/> <maximumVelocity meterPerSecond="1.4"/> <passengerCarEquivalents pce="0.0"/> <networkMode networkMode="netwalk"/> <flowEfficiencyFactor factor="1.0"/> </vehicleType>''') vehiclesfile.write('</vehicleDefinitions>') vehiclesfile.close() log.info('Cleaning up.') self.delete_sample()
def load_activities(self): log.info('Loading input activity definitions.') activities = counter(self.fetch_activities(), 'Loading activity %s.') for activitiy_id, agent_id in activities: self.activities[str(agent_id)].append(activitiy_id)
def load_legs(self): log.info('Loading input leg definitions.') legs = counter(self.fetch_legs(), 'Loading leg %s.') for leg_id, agent_id in legs: self.legs[str(agent_id)].append(leg_id)
def export_links(database: SqliteUtil, filepath: str, src_epsg: int, prj_epsg: int): transformer = Transformer.from_crs(f'epsg:{src_epsg}', f'epsg:{prj_epsg}', always_xy=True, skip_equivalent=True) project = transformer.transform prjpath = os.path.splitext(filepath)[0] + '.prj' with open(prjpath, 'w') as prjfile: info = get_wkt_string(prj_epsg) prjfile.write(info) query = ''' SELECT links.link_id, links.source_node, links.terminal_node, links.length, links.freespeed, links.capacity, links.permlanes, links.oneway, links.modes, links.air_temperature, links.mrt_temperature, nodes1.point, nodes2.point FROM links INNER JOIN nodes AS nodes1 ON links.source_node = nodes1.node_id INNER JOIN nodes AS nodes2 ON links.terminal_node = nodes2.node_id; ''' database.cursor.execute(query) rows = database.fetch_rows() rows = counter(rows, 'Exporting link %s.') links = shapefile.Writer(filepath, ) links.field('link_id', 'C') links.field('source_node', 'C') links.field('terminal_node', 'C') links.field('length', 'N') links.field('freespeed', 'N') links.field('capacity', 'N') links.field('permlanes', 'N') links.field('oneway', 'N') links.field('modes', 'C') links.field('air_temperature', 'N') links.field('mrt_temperature', 'N') for row in rows: props = row[:-2] pt1, pt2 = row[-2:] x1, y1 = project(*xy(pt1)) x2, y2 = project(*xy(pt2)) try: links.record(*props) except: print(props) breakpoint() exit() links.line([((x1, y1), (x2, y2))]) if links.recNum != links.shpNum: log.error('Record/shape misalignment; shapefile exporting failure.') raise RuntimeError links.close()
def map_mrt_temperature(database: SqliteUtil, kind: str): log.info('Profiling temperature extrema.') max_temp, min_temp, max_idx, min_idx = load_extrema(database, kind) log.info('Loading network links.') links = load_links(database) os.makedirs('result/mrt_temperatures/', exist_ok=True) log.info('Loading temperatures.') temps = defaultdict(lambda: [None] * (max_idx - min_idx + 1)) query = f''' SELECT temperature_id, temperature_idx, {kind} FROM mrt_temperatures; ''' database.cursor.execute(query) rows = database.fetch_rows() rows = counter(rows, 'Loading temperature profile %s.') for uuid, idx, temp in rows: temps[uuid][idx - min_idx] = temp def generate(): for link in links: temp = temps[link.profile] yield (link.id, *temp, link.line) log.info('Forming dataframes.') cols = [f'temp_{idx}' for idx in range(min_idx, max_idx + 1)] df = pd.DataFrame(generate(), columns=('id', *cols, 'line')) df['line'] = gpd.GeoSeries(df['line'], crs='EPSG:2223') gpdf = gpd.GeoDataFrame(df, geometry='line', crs='EPSG:2223') gpdf = gpdf.to_crs(epsg=3857) del links, temps, df for idx in range(min_idx, max_idx + 1): fig, ax = plt.subplots(1, figsize=(20, 12)) log.info(f'Plotting network visual.') plot = gpdf.plot(column=f'temp_{idx}', cmap='YlOrRd', linewidth=0.5, ax=ax, alpha=1) ax.set_title(f'Maricopa {kind.upper()} Temperatures {idx_to_hhmm(idx)}', fontdict={'fontsize': '18', 'fontweight' : '3'}) ctx.add_basemap(plot, source=ctx.providers.Stamen.TonerLite) sm = plt.cm.ScalarMappable(cmap='YlOrRd', norm=plt.Normalize(vmin=min_temp, vmax=max_temp)) sm._A = [] cbar = fig.colorbar(sm) log.info(f'Saving map.') fig.savefig(f'result/mrt_temperatures1/{idx}.png', bbox_inches='tight') plt.clf() plt.close()
def parse_parcels(database: SqliteUtil, residence_file: str, commerce_file: str, parcel_file: str, cooling_file: str, src_epsg: int, prj_epsg: int): boundaries = {} cooling = {} parcels = [] apns = set() transformer = Transformer.from_crs(f'epsg:{src_epsg}', f'epsg:{prj_epsg}', always_xy=True, skip_equivalent=True) project = transformer.transform log.info('Allocating tables for parcels.') create_tables(database) log.info('Parsing parcel boudaries from shapefile.') parser = shapefile.Reader(parcel_file) iter_boundaries = counter(iter(parser), 'Parsing parcel boundary %s.') for parcel in iter_boundaries: if len(parcel.shape.points): apn = parcel.record['APN'] points = (project(*pt) for pt in parcel.shape.points) polygon = Polygon(points) boundaries[apn] = polygon parser.close() log.info('Loading cooling information from csv file.') with open(cooling_file, 'r') as open_file: lines = csv.reader(open_file, delimiter=',', quotechar='"') next(lines) for desc, _, cool in lines: cooling[desc] = bool(cool) log.info('Parsing residential parcels from database file.') parser = shapefile.Reader(residence_file) iter_parcels = counter(parser.iterRecords(), 'Parsing residential parcel %s.') for record in iter_parcels: apn = record['APN'] if apn in boundaries and apn not in apn: cool = True polygon = boundaries[apn] parcel = Parcel(apn, 'residential', cool, polygon) parcels.append(parcel) apns.add(apn) parser.close() log.info('Parsing comercial parcels from database file.') parser = shapefile.Reader(commerce_file) iter_parcels = counter(parser.iterRecords(), 'Parsing commercial parcel %s.') for record in iter_parcels: apn = record['APN'] if apn in boundaries and apn not in apns: desc = record['DESCRIPT'] cool = cooling[desc] polygon = boundaries[apn] parcel = Parcel(apn, 'commercial', cool, polygon) parcels.append(parcel) apns.add(apn) parser.close() log.info('Parsing extraneous parcels from shapefile.') other = set(boundaries.keys()) - apns other = counter(other, 'Parsing extraneous parcel %s.') for apn in other: polygon = boundaries[apn] parcel = Parcel(apn, 'other', True, polygon) parcels.append(parcel) def load(): for idx, parcel in enumerate(parcels): pt = parcel.polygon.centroid yield (idx, (pt.x, pt.y, pt.x, pt.y), None) log.info('Building spatial index from parcel data.') index = Index(load()) log.info('Loading network region data.') regions = load_regions(database) log.info('Scanning regions and mapping mazs to parcels.') iter_regions = counter(regions, 'Sacnning region %s.') for region in iter_regions: apn = f'maz-{region.maz}' parcel = Parcel(apn, 'default', True, region.polygon) parcel.maz = region.maz parcels.append(parcel) result = index.intersection(region.polygon.bounds) for idx in result: parcel = parcels[idx] if region.polygon.contains(parcel.polygon.centroid): if parcel.maz is not None: warning = 'Parcel %s is in both region %s and %s' \ '; the latter region will be kept.' log.warning(warning % (parcel.apn, parcel.maz, region.maz)) parcel.maz = region.maz del regions def dump(): for parcel in parcels: yield (parcel.apn, parcel.maz, parcel.kind, int(parcel.cooling), None, None, dumps(parcel.polygon.centroid), dumps(parcel.polygon)) log.info('Writing parsed parcels to database.') database.insert_values('parcels', dump(), 8) database.connection.commit() log.info('Creating indexes on new tables.') create_indexes(database)
def parse_temperatures(database: SqliteUtil, tmin_files: List[str], tmax_files: List[str], steps: int, day: int, src_epsg: int, prj_epsg: int): log.info('Allocating tables for air temperatures.') create_tables(database) files = zip(tmax_files, tmin_files) profile_count = 0 point_count = 0 temperatures = [] points = [] profiles = {} n = 1 transformer = Transformer.from_crs(f'epsg:{src_epsg}', f'epsg:{prj_epsg}', always_xy=True, skip_equivalent=True) project = transformer.transform def apply(id: int, temp: Callable): for step in range(steps): prop = step / steps row = (id, step, int(86400 * prop), temp(24 * prop)) yield row log.info('Loading temperatures from netCDF4 files.') for tmax_file, tmin_file in files: tmaxnc = Dataset(tmax_file, 'r') tminnc = Dataset(tmin_file, 'r') lons = tmaxnc.variables['lon'] lats = tmaxnc.variables['lat'] shape = tmaxnc.variables['tmax'].shape tmaxs = tmaxnc.variables['tmax'][day] tmins = tminnc.variables['tmin'][day] for i in range(shape[1]): for j in range(shape[2]): tmax = tmaxs[i][j] tmin = tmins[i][j] if tmax != -9999.0: x, y = project(lons[i][j], lats[i][j]) idx = f'{tmax}-{tmin}' if idx not in profiles: temp = iterpolation(tmin, tmax, 5, 15) temperatures.extend(apply(profile_count, temp)) profiles[idx] = profile_count profile_count += 1 profile = profiles[idx] point = Point(point_count, x, y, profile) points.append(point) point_count += 1 if point_count == n: log.info( f'Loading air temperature reading {point_count}.') n <<= 1 tmaxnc.close() tminnc.close() if point_count != n >> 1: log.info(f'Loading air temperature reading {point_count}.') def load(): for point in points: x, y = point.x, point.y yield (point.id, (x, y, x, y), point.profile) log.info('Starting network update for air temperatures.') log.info('Building spatial index from temperature profile locations.') index = Index(load()) used = set() log.info('Loading network links.') links = load_links(database) log.info('Applying temperature profiles to links.') iter_links = counter(links, 'Applying profile to link %s.') for link in iter_links: result = index.nearest((link.x, link.y, link.x, link.y), objects=True) profile = next(result).object link.air_temperature = profile used.add(profile) def dump_links(): for link in links: yield (link.id, link.air_temperature) log.info('Writing updated links to database.') database.insert_values('temp_links', dump_links(), 2) database.connection.commit() del links log.info('Loading network parcels.') parcels = load_parcels(database) residential = profile_count temperatures.extend(apply(profile_count, lambda x: 26.6667)) profile_count += 1 commercial = profile_count temperatures.extend(apply(profile_count, lambda x: 26.6667)) profile_count += 1 other = profile_count temperatures.extend(apply(profile_count, lambda x: 26.6667)) profile_count += 1 used.add(residential) used.add(commercial) used.add(other) log.info('Applying temperature profiles to parcels.') iter_parcels = counter(parcels, 'Applying profile to parcel %s.') for parcel in iter_parcels: if not parcel.cooling: x, y = xy(parcel.center) result = index.nearest((x, y, x, y), objects=True) profile = next(result).object parcel.air_temperature = profile used.add(profile) elif parcel.kind == 'residential': parcel.air_temperature = residential elif parcel.kind == 'commercial': parcel.air_temperature = commercial else: parcel.air_temperature = other def dump_parcels(): for parcel in parcels: yield (parcel.apn, parcel.air_temperature) log.info('Writing updated parcels to database.') database.insert_values('temp_parcels', dump_parcels(), 2) database.connection.commit() del parcels def dump_temperatures(): for temp in temperatures: if temp[0] in used: yield temp log.info('Writing parsed air temperatures to database.') database.insert_values('air_temperatures', dump_temperatures(), 4) database.connection.commit() del temperatures log.info('Merging, dropping and renaming old tables.') query = ''' CREATE INDEX temp_links_link ON temp_links(link_id); ''' database.cursor.execute(query) query = ''' CREATE TABLE temp_links_merged AS SELECT links.link_id, links.source_node, links.terminal_node, links.length, links.freespeed, links.capacity, links.permlanes, links.oneway, links.modes, temp_links.air_temperature, links.mrt_temperature FROM links INNER JOIN temp_links USING(link_id); ''' database.cursor.execute(query) query = ''' CREATE INDEX temp_parcels_parcel ON temp_parcels(apn); ''' database.cursor.execute(query) query = ''' CREATE TABLE temp_parcels_merged AS SELECT parcels.apn, parcels.maz, parcels.type, parcels.cooling, temp_parcels.air_temperature, parcels.mrt_temperature, parcels.center, parcels.region FROM parcels INNER JOIN temp_parcels USING(apn); ''' database.cursor.execute(query) original = database.count_rows('links') merged = database.count_rows('temp_links_merged') if original != merged: log.error('Original links and updated links tables ' 'do not align; quiting to prevent data loss.') raise RuntimeError else: database.drop_table('links', 'temp_links') query = ''' ALTER TABLE temp_links_merged RENAME TO links; ''' database.cursor.execute(query) original = database.count_rows('parcels') merged = database.count_rows('temp_parcels_merged') if original != merged: log.error('Original parcels and updated parcels tables ' 'do not align; quiting to prevent data loss.') raise RuntimeError else: database.drop_table('parcels', 'temp_parcels') query = ''' ALTER TABLE temp_parcels_merged RENAME TO parcels; ''' database.cursor.execute(query) database.connection.commit() log.info('Creating indexes on new tables.') create_indexes(database) log.info('Writing process metadata.')
def parse(self, networkpath): log.info('Fetching exposure geospatial data.') centroids_list = self.fetch_centroids() log.info('Loading centroids into spatial index.') centroids = {} for uuid, centroid in counter(centroids_list, 'Loading centroid %s.'): x, y = map(float, centroid[7:-1].split(' ')) centroids[uuid] = Centroid(uuid, x, y) centroid_idx = index.Index(centroid.entry() for centroid in centroids.values()) del centroids_list log.info('Fetching network region data.') regions_list = self.fetch_regions() log.info('Loading regions into spatial index.') regions = {} for uuid, region in counter(regions_list, 'Loading region %s.'): polygon = loads(region) setattr(polygon, 'maz', uuid) regions[uuid] = Region(uuid, polygon) region_idx = STRtree(region.region for region in regions.values()) del regions_list def get_centroid(node): return next(centroid_idx.nearest(node, 1)) def get_region(node: Point): regions = region_idx.query(node) region = None if len(regions): region = regions[0].maz return region log.info('Loading network roads file.') network = multiopen(networkpath, mode='rb') parser = iter(iterparse(network, events=('start', 'end'))) evt, root = next(parser) links = [] nodes = [] points = {} count = 0 n = 1 for evt, elem in parser: if evt == 'start': if elem.tag == 'nodes': log.info('Parsing nodes from network file.') count = 0 n = 1 elif elem.tag == 'links': if count != n << 1: log.info(f'Parsed node {count}.') log.info('Parsing links from network file.') count = 0 n = 1 elif evt == 'end': if elem.tag == 'node': node_id = str(elem.get('id')) x = float(elem.get('x')) y = float(elem.get('y')) point = Point(x, y) region = get_region(point) centroid = get_centroid((x, y, x, y)) nodes.append((node_id, region, centroid, dumps(point))) points[node_id] = point count += 1 if count == n: log.info(f'Parsing node {count}.') n <<= 1 elif elem.tag == 'link': source_node = str(elem.get('from')) terminal_node = str(elem.get('to')) line = LineString( (points[source_node], points[terminal_node])) links.append( (str(elem.get('id')), source_node, terminal_node, float(elem.get('length')), float(elem.get('freespeed')), float(elem.get('capacity')), float(elem.get('permlanes')), int(elem.get('oneway')), str(elem.get('modes')), dumps(line))) count += 1 if count == n: log.info(f'Parsing link {count}.') n <<= 1 if count % 100000: root.clear() if count != n << 1: log.info(f'Parsing link {count}.') network.close() log.info('Writing parsed links and nodes to database.') self.create_tables() self.database.insert_values('nodes', nodes, 4) self.database.insert_values('links', links, 10) self.database.connection.commit()