def parse_regions(database: SqliteUtil, regions_file: str, src_epsg: int,
prj_epsg: int):
log.info('Allocating tables for regions.')
create_tables(database)
transformer = Transformer.from_crs(f'epsg:{src_epsg}',
f'epsg:{prj_epsg}',
always_xy=True,
skip_equivalent=True)
project = transformer.transform
log.info('Parsing regions from shapefile.')
parser = shapefile.Reader(regions_file)
iter_regions = counter(iter(parser), 'Parsing region %s.')
regions = []
for item in iter_regions:
points = (project(*point) for point in item.shape.points)
polygon = Polygon(points)
regions.append(
(item.record.MAZ_ID_10, item.record.TAZ_2015, item.record.Sq_miles,
dumps(polygon.centroid), dumps(polygon)))
parser.close()
log.info('Writing parsed regions to database.')
database.insert_values('regions', regions, 5)
database.connection.commit()
log.info('Creating indexes on new tables.')
create_indexes(database)
def load_links(database: SqliteUtil):
query = '''
SELECT
links.link_id,
links.mrt_temperature,
nodes1.point AS source_point,
nodes2.point AS terminal_point
FROM links
INNER JOIN nodes AS nodes1
ON links.source_node = nodes1.node_id
INNER JOIN nodes AS nodes2
ON links.terminal_node = nodes2.node_id;
'''
database.cursor.execute(query)
rows = database.fetch_rows()
rows = counter(rows, 'Loading link %s.')
bounds = lambda x, y: min(x) > 0.5e6 and max(x) < 0.85e6 and \
min(y) > 0.8e6 and max(y) < 1.0e6
links = []
for link_id, profile, src_pt, term_pt in rows:
line = LineString((xy(src_pt), xy(term_pt)))
x, y = line.coords.xy
if bounds(x, y):
link = Link(link_id, line, profile)
links.append(link)
return links
def load_temperatures(self):
log.info('Loading network daymet temperature data.')
temperatures = counter(self.fetch_temperatures(),
'Loading temperature %s.')
for temperature_id, _, temperature in temperatures:
self.temperatures[temperature_id].append(temperature)
self.temperatures.lock()
def load_links(self):
log.info('Loading network road link data.')
query = '''
SELECT
link_id,
length,
freespeed,
modes,
air_temperature,
mrt_temperature
FROM links;
'''
self.database.cursor.execute(query)
result = self.database.cursor.fetchall()
links = counter(result, 'Loading link %s.')
for link_id, length, speed, modes, air_temp, mrt_temp in links:
modes_set = set(NetworkMode(mode) for mode in modes.split(','))
air_temperature = self.air_temperatures[air_temp]
mrt_temperature = None
if mrt_temp is not None:
mrt_temperature = self.mrt_temperatures[mrt_temp]
link = Link(
link_id,
length,
speed,
modes_set,
air_temperature,
mrt_temperature
)
self.links[link_id] = link
def load_nodes(self):
log.info('Loading network road node data.')
nodes = counter(self.fetch_nodes(), 'Loading node %s.')
for node_id, maz, centroid_id, point in nodes:
centroid = self.centroids[centroid_id]
x, y = xy(point)
self.nodes[node_id] = Node(node_id, maz, centroid, x, y)
def load_temperatures(database: SqliteUtil, kind: str,
max_idx: int, min_idx: int):
query = f'''
SELECT
mrt_temperatures.temperature_id,
mrt_temperatures.temperature_idx,
mrt_temperatures.{kind},
COUNT(*) AS util
FROM mrt_temperatures
INNER JOIN links
ON links.mrt_temperature = mrt_temperatures.temperature_id
INNER JOIN output_events
ON output_events.link_id = links.link_id
WHERE output_events.start >= mrt_temperatures.temperature_idx * 900
AND output_events.end < mrt_temperatures.temperature_idx * 900 + 900
GROUP BY temperature_id, temperature_idx;
'''
database.cursor.execute(query)
rows = database.fetch_rows()
rows = counter(rows, 'Loading temperature profile %s.')
temps = defaultdict(lambda: [None] * (max_idx - min_idx + 1))
for uuid, idx, temp, util in rows:
if util > 0:
temps[uuid][idx - min_idx] = temp
return temps
def load_nodes(self):
log.info('Loading network road node data.')
nodes = counter(self.fetch_nodes(), 'Loading node %s.')
for node in nodes:
node_id = node[0]
maz = node[1]
x, y = xy(node[2])
self.nodes[node_id] = Node(node_id, maz, x, y)
def load_centroids(self):
log.info('Loading network daymet centroid data.')
centroids = counter(self.fetch_centroids(), 'Loading centroid %s.')
for centroid_id, temperature_id, center in centroids:
temperatures = self.temperatures[temperature_id]
x, y = xy(center)
self.centroids[centroid_id] = Centroid(centroid_id, temperatures,
x, y)
def load_households(households_file: str):
open_file = multiopen(households_file, 'rt')
households = csv.reader(open_file, delimiter=',', quotechar='"')
next(households)
households = counter(households, 'Parsing household %s.')
for household in households:
yield [int(h) for h in household[0:2]] + [float(household[2])] + \
[int(h) for h in household[3:18]]
open_file.close()
def load_trips(trips_file: str):
open_file = multiopen(trips_file, 'rt')
trips = csv.reader(open_file, delimiter=',', quotechar='"')
next(trips)
trips = counter(trips, 'Parsing trip %s.')
for trip in trips:
yield [int(t) for t in trip[0:6]] + [trip[6]] + \
[int(t) for t in trip[7:15]] + [float(t) for t in trip[15:18]]
open_file.close()
def load_links(self):
log.info('Loading network road link data.')
links = counter(self.fetch_links(), 'Loading link %s.')
for link in links:
link_id = link[0]
src_node = self.nodes[link[1]]
term_node = self.nodes[link[2]]
length = link[3]
freespeed = link[4]
modes = set(NetworkMode(mode) for mode in link[5].split(','))
self.links[link_id] = Link(link_id, src_node, term_node, length,
freespeed, modes)
def load_persons(persons_file: str):
open_file = multiopen(persons_file, 'rt')
persons = csv.reader(open_file, delimiter=',', quotechar='"')
next(persons)
persons = counter(persons, 'Loading person %s.')
for person in persons:
yield [int(p) for p in person[0:2]] + [float(person[2])] + \
[int(p) for p in person[3:18]] + [person[18]] + \
[int(p) for p in person[19:37]] + [person[37]] + [int(person[38])]
open_file.close()
def load_temperatures(self, kind: str = 'mrt'):
log.info('Loading network air temperature data.')
query = '''
SELECT
temperature_id,
temperature_idx,
temperature
FROM air_temperatures;
'''
self.database.cursor.execute(query)
rows = self.database.fetch_rows()
temps = defaultdict(lambda: [None]*96)
rows = counter(rows, 'Loading air temperature %s.')
for temperature_id, temperature_idx, temperature in rows:
temps[temperature_id][temperature_idx] = temperature
for uuid, values in temps.items():
self.air_temperatures[uuid] = Temperature(uuid, values)
log.info('Loading network mrt temperature data.')
query = f'''
SELECT
temperature_id,
temperature_idx,
{kind}
FROM mrt_temperatures;
'''
self.database.cursor.execute(query)
rows = self.database.cursor.fetchall()
temps = defaultdict(lambda: [None]*96)
rows = counter(rows, 'Loading mrt temperature %s.')
for temperature_id, temperature_idx, temperature in rows:
temps[temperature_id][temperature_idx] = temperature
for uuid, values in temps.items():
self.mrt_temperatures[uuid] = Temperature(uuid, values)
def load_parcels(self):
log.info('Loading network parcel data.')
query = '''
SELECT
apn,
air_temperature
FROM parcels;
'''
self.database.cursor.execute(query)
rows = self.database.fetch_rows()
rows = counter(rows, 'Loading parcel %s.')
for apn, temperature in rows:
temp = self.air_temperatures[temperature]
parcel = Parcel(apn, temp)
self.parcels[apn] = parcel
def load_regions(database: SqliteUtil):
query = '''
SELECT
maz,
region
FROM regions;
'''
database.cursor.execute(query)
rows = database.fetch_rows()
rows = counter(rows, 'Loading region %s.')
regions = []
for maz, polygon in rows:
region = Region(maz, polygon)
regions.append(region)
return regions
def parse_temperatures(csvfile: str) \
-> Tuple[List[Tuple[float,float,float]],int]:
log.info(f'Opening {csvfile}.')
csv_file = open(csvfile, 'r')
iter_temps = csv.reader(csv_file, delimiter=',', quotechar='"')
next(iter_temps)
iter_temps = counter(iter_temps, 'Parsing temperature %s.')
temps = []
peek = next(iter_temps)
iter_temps.send(peek)
secs = hhmm_to_secs(peek[2])
for _, _, _, mrt, pet, utci in iter_temps:
temps.append((float(mrt), float(pet), float(utci)))
csv_file.close()
return temps, secs
def load_nodes(database: SqliteUtil) -> Dict[str,Node]:
query = '''
SELECT
node_id,
point
FROM nodes;
'''
database.cursor.execute(query)
rows = database.fetch_rows()
rows = counter(rows, 'Loading node %s.')
nodes: Dict[str,Node] = {}
for uuid, point in rows:
x, y = xy(point)
node = Node(uuid, x, y)
nodes[uuid] = node
return nodes
def load_links(database: SqliteUtil):
query = '''
SELECT
links.link_id,
nodes.point
FROM links
INNER JOIN nodes
ON links.source_node = nodes.node_id;
'''
database.cursor.execute(query)
rows = database.fetch_rows()
rows = counter(rows, 'Loading link %s.')
links = []
for link_id, point in rows:
x, y = xy(point)
link = Link(link_id, x, y)
links.append(link)
return links
def load_parcels(database: SqliteUtil):
query = '''
SELECT
apn,
type,
cooling,
center
FROM parcels;
'''
database.cursor.execute(query)
rows = database.fetch_rows()
rows = counter(rows, 'Loading parcel %s.')
parcels = []
for apn, kind, cooling, center in rows:
x, y = xy(center)
parcel = Parcel(apn, kind, bool(cooling), x, y)
parcels.append(parcel)
return parcels
def load_links(database: SqliteUtil, nodes: Dict[str,Node]) -> Dict[str,Link]:
query = '''
SELECT
link_id,
source_node,
terminal_node,
freespeed,
permlanes
FROM links;
'''
database.cursor.execute(query)
rows = database.fetch_rows()
rows = counter(rows, 'Loading link %s.')
links: Dict[str,Link] = {}
for uuid, src, term, speed, lanes in rows:
source_node = nodes[src]
terminal_node = nodes[term]
link = Link(uuid, source_node, terminal_node, lanes, speed)
links[uuid] = link
return links
def load_parcels(self):
parcels = self.fetch_parcels()
self.residential_parcels = defaultdict(lambda x: [])
self.commercial_parcels = defaultdict(lambda x: [])
self.default_parcels = {}
self.other_parcels = defaultdict(lambda x: [])
for apn, maz, kind in counter(parcels, 'Loading parcel %s.'):
if kind == 'residential':
self.residential_parcels[maz].append(Parcel(apn))
elif kind == 'commercial':
self.commercial_parcels[maz].append(Parcel(apn))
elif kind == 'default':
self.default_parcels[maz] = Parcel(apn)
elif kind == 'other':
self.other_parcels[maz].append(Parcel(apn))
self.residential_parcels.lock()
self.commercial_parcels.lock()
self.other_parcels.lock()
self.mazs = set(self.default_parcels.keys())
self.offset = defaultdict(lambda x: 0)
def parse_points(csvfile: str, src_epsg: int, prj_epsg: int) \
-> Tuple[List[Point],int]:
log.info(f'Opening {csvfile}.')
csv_file = open(csvfile, 'r')
iter_points = csv.reader(csv_file, delimiter=',', quotechar='"')
next(iter_points)
iter_points = counter(iter_points, 'Parsing point %s.')
transformer = Transformer.from_crs(f'epsg:{src_epsg}',
f'epsg:{prj_epsg}', always_xy=True, skip_equivalent=True)
project = transformer.transform
points = []
peek = next(iter_points)
iter_points.send(peek)
secs = hhmm_to_secs(peek[2])
for uuid, (lat, lon, _, mrt, pet, utci) in enumerate(iter_points):
x, y = project(lon, lat)
point = Point(uuid, x, y, float(mrt), float(pet), float(utci))
points.append(point)
csv_file.close()
return points, secs
def generate(self,
planspath,
vehiclespath,
modes,
sample_percent=1,
sample_size=math.inf,
transit=None,
vehicle=None,
walk=None,
bike=None,
party=None):
log.info('Creating a sample population.')
conditions = {
'transit': transit,
'vehicle': vehicle,
'walk': walk,
'bike': bike,
'party': party
}
max_size = self.database.count_rows('agents')
size = min(max_size * sample_percent, sample_size)
table = 'agents'
if size < max_size or any(cond is not None
for cond in conditions.values()):
table = 'sample'
self.create_sample(size, **conditions)
actual = self.database.count_rows('sample')
if actual < size:
log.info(f'Target sample was {size} but only found {actual} '
'agents under specified parameters.')
log.info('Fetching agents, activities and legs.')
agents = self.fetch_agents(table)
activities = self.fetch_activities(table)
legs = self.fetch_legs(table)
log.info('Iterating over plans and generating plans file.')
touch(planspath)
plansfile = multiopen(planspath, mode='wt')
plansfile.write(
'<?xml version="1.0" encoding="utf-8"?><!DOCTYPE plans'
' SYSTEM "http://www.matsim.org/files/dtd/plans_v4.dtd"><plans>')
for agent_id, plan_size in counter(agents, 'Writing plan %s.'):
plansfile.write(f'<person id="{agent_id}"><plan selected="yes">')
plansfile.write(next(activities).encode_start())
for _ in range(plan_size // 2 - 1):
leg = next(legs)
activity = next(activities)
plansfile.write(leg.encode(activity))
plansfile.write(activity.encode())
leg = next(legs)
activity = next(activities)
plansfile.write(leg.encode(activity))
plansfile.write(activity.encode_end())
plansfile.write('</plan></person>')
plansfile.flush()
plansfile.write('</plans>')
log.info('Writing vehicle definitions file.')
touch(vehiclespath)
vehiclesfile = multiopen(vehiclespath, mode='wt')
vehiclesfile.write('''<?xml version="1.0" encoding="UTF-8" ?>
<vehicleDefinitions
xmlns="http://www.matsim.org/files/dtd"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.matsim.org/files/dtd
http://www.matsim.org/files/dtd/vehicleDefinitions_v2.0.xsd">'''
)
vehiclesfile.write('''
<vehicleType id="Bus">
<attributes>
<attribute name="accessTimeInSecondsPerPerson" class="java.lang.Double">0.5</attribute>
<attribute name="doorOperationMode" class="java.lang.String">serial</attribute>
<attribute name="egressTimeInSecondsPerPerson" class="java.lang.Double">0.5</attribute>
</attributes>
<capacity seats="70" standingRoomInPersons="0"/>
<length meter="18.0"/>
<width meter="2.5"/>
<passengerCarEquivalents pce="2.8"/>
<networkMode networkMode="Bus"/>
<flowEfficiencyFactor factor="1.0"/>
</vehicleType>''')
vehiclesfile.write('''
<vehicleType id="Tram">
<attributes>
<attribute name="accessTimeInSecondsPerPerson" class="java.lang.Double">0.25</attribute>
<attribute name="doorOperationMode" class="java.lang.String">serial</attribute>
<attribute name="egressTimeInSecondsPerPerson" class="java.lang.Double">0.25</attribute>
</attributes>
<capacity seats="180" standingRoomInPersons="0"/>
<length meter="36.0"/>
<width meter="2.4"/>
<passengerCarEquivalents pce="5.2"/>
<networkMode networkMode="Tram"/>
<flowEfficiencyFactor factor="1.0"/>
</vehicleType>''')
vehiclesfile.write('''
<vehicleType id="car">
<attributes>
<attribute name="accessTimeInSecondsPerPerson" class="java.lang.Double">1.0</attribute>
<attribute name="doorOperationMode" class="java.lang.String">serial</attribute>
<attribute name="egressTimeInSecondsPerPerson" class="java.lang.Double">1.0</attribute>
</attributes>
<capacity seats="5" standingRoomInPersons="0"/>
<length meter="7.5"/>
<width meter="1.0"/>
<maximumVelocity meterPerSecond="40.0"/>
<passengerCarEquivalents pce="1.0"/>
<networkMode networkMode="car"/>
<flowEfficiencyFactor factor="1.0"/>
</vehicleType>''')
vehiclesfile.write('''
<vehicleType id="bike">
<attributes>
<attribute name="accessTimeInSecondsPerPerson" class="java.lang.Double">1.0</attribute>
<attribute name="doorOperationMode" class="java.lang.String">serial</attribute>
<attribute name="egressTimeInSecondsPerPerson" class="java.lang.Double">1.0</attribute>
</attributes>
<capacity seats="1" standingRoomInPersons="0"/>
<length meter="5.0"/>
<width meter="1.0"/>
<maximumVelocity meterPerSecond="4.4704"/>
<passengerCarEquivalents pce="0.25"/>
<networkMode networkMode="bike"/>
<flowEfficiencyFactor factor="1.0"/>
</vehicleType>''')
vehiclesfile.write('''
<vehicleType id="netwalk">
<attributes>
<attribute name="accessTimeInSecondsPerPerson" class="java.lang.Double">1.0</attribute>
<attribute name="doorOperationMode" class="java.lang.String">serial</attribute>
<attribute name="egressTimeInSecondsPerPerson" class="java.lang.Double">1.0</attribute>
</attributes>
<capacity seats="1" standingRoomInPersons="0"/>
<length meter="1.0"/>
<width meter="1.0"/>
<maximumVelocity meterPerSecond="1.4"/>
<passengerCarEquivalents pce="0.0"/>
<networkMode networkMode="netwalk"/>
<flowEfficiencyFactor factor="1.0"/>
</vehicleType>''')
vehiclesfile.write('</vehicleDefinitions>')
vehiclesfile.close()
log.info('Cleaning up.')
self.delete_sample()
def load_activities(self):
log.info('Loading input activity definitions.')
activities = counter(self.fetch_activities(), 'Loading activity %s.')
for activitiy_id, agent_id in activities:
self.activities[str(agent_id)].append(activitiy_id)
def load_legs(self):
log.info('Loading input leg definitions.')
legs = counter(self.fetch_legs(), 'Loading leg %s.')
for leg_id, agent_id in legs:
self.legs[str(agent_id)].append(leg_id)
def export_links(database: SqliteUtil, filepath: str, src_epsg: int,
prj_epsg: int):
transformer = Transformer.from_crs(f'epsg:{src_epsg}',
f'epsg:{prj_epsg}',
always_xy=True,
skip_equivalent=True)
project = transformer.transform
prjpath = os.path.splitext(filepath)[0] + '.prj'
with open(prjpath, 'w') as prjfile:
info = get_wkt_string(prj_epsg)
prjfile.write(info)
query = '''
SELECT
links.link_id,
links.source_node,
links.terminal_node,
links.length,
links.freespeed,
links.capacity,
links.permlanes,
links.oneway,
links.modes,
links.air_temperature,
links.mrt_temperature,
nodes1.point,
nodes2.point
FROM links
INNER JOIN nodes AS nodes1
ON links.source_node = nodes1.node_id
INNER JOIN nodes AS nodes2
ON links.terminal_node = nodes2.node_id;
'''
database.cursor.execute(query)
rows = database.fetch_rows()
rows = counter(rows, 'Exporting link %s.')
links = shapefile.Writer(filepath, )
links.field('link_id', 'C')
links.field('source_node', 'C')
links.field('terminal_node', 'C')
links.field('length', 'N')
links.field('freespeed', 'N')
links.field('capacity', 'N')
links.field('permlanes', 'N')
links.field('oneway', 'N')
links.field('modes', 'C')
links.field('air_temperature', 'N')
links.field('mrt_temperature', 'N')
for row in rows:
props = row[:-2]
pt1, pt2 = row[-2:]
x1, y1 = project(*xy(pt1))
x2, y2 = project(*xy(pt2))
try:
links.record(*props)
except:
print(props)
breakpoint()
exit()
links.line([((x1, y1), (x2, y2))])
if links.recNum != links.shpNum:
log.error('Record/shape misalignment; shapefile exporting failure.')
raise RuntimeError
links.close()
def map_mrt_temperature(database: SqliteUtil, kind: str):
log.info('Profiling temperature extrema.')
max_temp, min_temp, max_idx, min_idx = load_extrema(database, kind)
log.info('Loading network links.')
links = load_links(database)
os.makedirs('result/mrt_temperatures/', exist_ok=True)
log.info('Loading temperatures.')
temps = defaultdict(lambda: [None] * (max_idx - min_idx + 1))
query = f'''
SELECT
temperature_id,
temperature_idx,
{kind}
FROM mrt_temperatures;
'''
database.cursor.execute(query)
rows = database.fetch_rows()
rows = counter(rows, 'Loading temperature profile %s.')
for uuid, idx, temp in rows:
temps[uuid][idx - min_idx] = temp
def generate():
for link in links:
temp = temps[link.profile]
yield (link.id, *temp, link.line)
log.info('Forming dataframes.')
cols = [f'temp_{idx}' for idx in range(min_idx, max_idx + 1)]
df = pd.DataFrame(generate(), columns=('id', *cols, 'line'))
df['line'] = gpd.GeoSeries(df['line'], crs='EPSG:2223')
gpdf = gpd.GeoDataFrame(df, geometry='line', crs='EPSG:2223')
gpdf = gpdf.to_crs(epsg=3857)
del links, temps, df
for idx in range(min_idx, max_idx + 1):
fig, ax = plt.subplots(1, figsize=(20, 12))
log.info(f'Plotting network visual.')
plot = gpdf.plot(column=f'temp_{idx}', cmap='YlOrRd', linewidth=0.5,
ax=ax, alpha=1)
ax.set_title(f'Maricopa {kind.upper()} Temperatures {idx_to_hhmm(idx)}',
fontdict={'fontsize': '18', 'fontweight' : '3'})
ctx.add_basemap(plot, source=ctx.providers.Stamen.TonerLite)
sm = plt.cm.ScalarMappable(cmap='YlOrRd',
norm=plt.Normalize(vmin=min_temp, vmax=max_temp))
sm._A = []
cbar = fig.colorbar(sm)
log.info(f'Saving map.')
fig.savefig(f'result/mrt_temperatures1/{idx}.png', bbox_inches='tight')
plt.clf()
plt.close()
def parse_parcels(database: SqliteUtil, residence_file: str,
commerce_file: str, parcel_file: str, cooling_file: str,
src_epsg: int, prj_epsg: int):
boundaries = {}
cooling = {}
parcels = []
apns = set()
transformer = Transformer.from_crs(f'epsg:{src_epsg}',
f'epsg:{prj_epsg}',
always_xy=True,
skip_equivalent=True)
project = transformer.transform
log.info('Allocating tables for parcels.')
create_tables(database)
log.info('Parsing parcel boudaries from shapefile.')
parser = shapefile.Reader(parcel_file)
iter_boundaries = counter(iter(parser), 'Parsing parcel boundary %s.')
for parcel in iter_boundaries:
if len(parcel.shape.points):
apn = parcel.record['APN']
points = (project(*pt) for pt in parcel.shape.points)
polygon = Polygon(points)
boundaries[apn] = polygon
parser.close()
log.info('Loading cooling information from csv file.')
with open(cooling_file, 'r') as open_file:
lines = csv.reader(open_file, delimiter=',', quotechar='"')
next(lines)
for desc, _, cool in lines:
cooling[desc] = bool(cool)
log.info('Parsing residential parcels from database file.')
parser = shapefile.Reader(residence_file)
iter_parcels = counter(parser.iterRecords(),
'Parsing residential parcel %s.')
for record in iter_parcels:
apn = record['APN']
if apn in boundaries and apn not in apn:
cool = True
polygon = boundaries[apn]
parcel = Parcel(apn, 'residential', cool, polygon)
parcels.append(parcel)
apns.add(apn)
parser.close()
log.info('Parsing comercial parcels from database file.')
parser = shapefile.Reader(commerce_file)
iter_parcels = counter(parser.iterRecords(),
'Parsing commercial parcel %s.')
for record in iter_parcels:
apn = record['APN']
if apn in boundaries and apn not in apns:
desc = record['DESCRIPT']
cool = cooling[desc]
polygon = boundaries[apn]
parcel = Parcel(apn, 'commercial', cool, polygon)
parcels.append(parcel)
apns.add(apn)
parser.close()
log.info('Parsing extraneous parcels from shapefile.')
other = set(boundaries.keys()) - apns
other = counter(other, 'Parsing extraneous parcel %s.')
for apn in other:
polygon = boundaries[apn]
parcel = Parcel(apn, 'other', True, polygon)
parcels.append(parcel)
def load():
for idx, parcel in enumerate(parcels):
pt = parcel.polygon.centroid
yield (idx, (pt.x, pt.y, pt.x, pt.y), None)
log.info('Building spatial index from parcel data.')
index = Index(load())
log.info('Loading network region data.')
regions = load_regions(database)
log.info('Scanning regions and mapping mazs to parcels.')
iter_regions = counter(regions, 'Sacnning region %s.')
for region in iter_regions:
apn = f'maz-{region.maz}'
parcel = Parcel(apn, 'default', True, region.polygon)
parcel.maz = region.maz
parcels.append(parcel)
result = index.intersection(region.polygon.bounds)
for idx in result:
parcel = parcels[idx]
if region.polygon.contains(parcel.polygon.centroid):
if parcel.maz is not None:
warning = 'Parcel %s is in both region %s and %s' \
'; the latter region will be kept.'
log.warning(warning % (parcel.apn, parcel.maz, region.maz))
parcel.maz = region.maz
del regions
def dump():
for parcel in parcels:
yield (parcel.apn, parcel.maz, parcel.kind, int(parcel.cooling),
None, None, dumps(parcel.polygon.centroid),
dumps(parcel.polygon))
log.info('Writing parsed parcels to database.')
database.insert_values('parcels', dump(), 8)
database.connection.commit()
log.info('Creating indexes on new tables.')
create_indexes(database)
def parse_temperatures(database: SqliteUtil, tmin_files: List[str],
tmax_files: List[str], steps: int, day: int,
src_epsg: int, prj_epsg: int):
log.info('Allocating tables for air temperatures.')
create_tables(database)
files = zip(tmax_files, tmin_files)
profile_count = 0
point_count = 0
temperatures = []
points = []
profiles = {}
n = 1
transformer = Transformer.from_crs(f'epsg:{src_epsg}',
f'epsg:{prj_epsg}',
always_xy=True,
skip_equivalent=True)
project = transformer.transform
def apply(id: int, temp: Callable):
for step in range(steps):
prop = step / steps
row = (id, step, int(86400 * prop), temp(24 * prop))
yield row
log.info('Loading temperatures from netCDF4 files.')
for tmax_file, tmin_file in files:
tmaxnc = Dataset(tmax_file, 'r')
tminnc = Dataset(tmin_file, 'r')
lons = tmaxnc.variables['lon']
lats = tmaxnc.variables['lat']
shape = tmaxnc.variables['tmax'].shape
tmaxs = tmaxnc.variables['tmax'][day]
tmins = tminnc.variables['tmin'][day]
for i in range(shape[1]):
for j in range(shape[2]):
tmax = tmaxs[i][j]
tmin = tmins[i][j]
if tmax != -9999.0:
x, y = project(lons[i][j], lats[i][j])
idx = f'{tmax}-{tmin}'
if idx not in profiles:
temp = iterpolation(tmin, tmax, 5, 15)
temperatures.extend(apply(profile_count, temp))
profiles[idx] = profile_count
profile_count += 1
profile = profiles[idx]
point = Point(point_count, x, y, profile)
points.append(point)
point_count += 1
if point_count == n:
log.info(
f'Loading air temperature reading {point_count}.')
n <<= 1
tmaxnc.close()
tminnc.close()
if point_count != n >> 1:
log.info(f'Loading air temperature reading {point_count}.')
def load():
for point in points:
x, y = point.x, point.y
yield (point.id, (x, y, x, y), point.profile)
log.info('Starting network update for air temperatures.')
log.info('Building spatial index from temperature profile locations.')
index = Index(load())
used = set()
log.info('Loading network links.')
links = load_links(database)
log.info('Applying temperature profiles to links.')
iter_links = counter(links, 'Applying profile to link %s.')
for link in iter_links:
result = index.nearest((link.x, link.y, link.x, link.y), objects=True)
profile = next(result).object
link.air_temperature = profile
used.add(profile)
def dump_links():
for link in links:
yield (link.id, link.air_temperature)
log.info('Writing updated links to database.')
database.insert_values('temp_links', dump_links(), 2)
database.connection.commit()
del links
log.info('Loading network parcels.')
parcels = load_parcels(database)
residential = profile_count
temperatures.extend(apply(profile_count, lambda x: 26.6667))
profile_count += 1
commercial = profile_count
temperatures.extend(apply(profile_count, lambda x: 26.6667))
profile_count += 1
other = profile_count
temperatures.extend(apply(profile_count, lambda x: 26.6667))
profile_count += 1
used.add(residential)
used.add(commercial)
used.add(other)
log.info('Applying temperature profiles to parcels.')
iter_parcels = counter(parcels, 'Applying profile to parcel %s.')
for parcel in iter_parcels:
if not parcel.cooling:
x, y = xy(parcel.center)
result = index.nearest((x, y, x, y), objects=True)
profile = next(result).object
parcel.air_temperature = profile
used.add(profile)
elif parcel.kind == 'residential':
parcel.air_temperature = residential
elif parcel.kind == 'commercial':
parcel.air_temperature = commercial
else:
parcel.air_temperature = other
def dump_parcels():
for parcel in parcels:
yield (parcel.apn, parcel.air_temperature)
log.info('Writing updated parcels to database.')
database.insert_values('temp_parcels', dump_parcels(), 2)
database.connection.commit()
del parcels
def dump_temperatures():
for temp in temperatures:
if temp[0] in used:
yield temp
log.info('Writing parsed air temperatures to database.')
database.insert_values('air_temperatures', dump_temperatures(), 4)
database.connection.commit()
del temperatures
log.info('Merging, dropping and renaming old tables.')
query = '''
CREATE INDEX temp_links_link
ON temp_links(link_id);
'''
database.cursor.execute(query)
query = '''
CREATE TABLE temp_links_merged
AS SELECT
links.link_id,
links.source_node,
links.terminal_node,
links.length,
links.freespeed,
links.capacity,
links.permlanes,
links.oneway,
links.modes,
temp_links.air_temperature,
links.mrt_temperature
FROM links
INNER JOIN temp_links
USING(link_id);
'''
database.cursor.execute(query)
query = '''
CREATE INDEX temp_parcels_parcel
ON temp_parcels(apn);
'''
database.cursor.execute(query)
query = '''
CREATE TABLE temp_parcels_merged
AS SELECT
parcels.apn,
parcels.maz,
parcels.type,
parcels.cooling,
temp_parcels.air_temperature,
parcels.mrt_temperature,
parcels.center,
parcels.region
FROM parcels
INNER JOIN temp_parcels
USING(apn);
'''
database.cursor.execute(query)
original = database.count_rows('links')
merged = database.count_rows('temp_links_merged')
if original != merged:
log.error('Original links and updated links tables '
'do not align; quiting to prevent data loss.')
raise RuntimeError
else:
database.drop_table('links', 'temp_links')
query = '''
ALTER TABLE temp_links_merged
RENAME TO links;
'''
database.cursor.execute(query)
original = database.count_rows('parcels')
merged = database.count_rows('temp_parcels_merged')
if original != merged:
log.error('Original parcels and updated parcels tables '
'do not align; quiting to prevent data loss.')
raise RuntimeError
else:
database.drop_table('parcels', 'temp_parcels')
query = '''
ALTER TABLE temp_parcels_merged
RENAME TO parcels;
'''
database.cursor.execute(query)
database.connection.commit()
log.info('Creating indexes on new tables.')
create_indexes(database)
log.info('Writing process metadata.')
def parse(self, networkpath):
log.info('Fetching exposure geospatial data.')
centroids_list = self.fetch_centroids()
log.info('Loading centroids into spatial index.')
centroids = {}
for uuid, centroid in counter(centroids_list, 'Loading centroid %s.'):
x, y = map(float, centroid[7:-1].split(' '))
centroids[uuid] = Centroid(uuid, x, y)
centroid_idx = index.Index(centroid.entry()
for centroid in centroids.values())
del centroids_list
log.info('Fetching network region data.')
regions_list = self.fetch_regions()
log.info('Loading regions into spatial index.')
regions = {}
for uuid, region in counter(regions_list, 'Loading region %s.'):
polygon = loads(region)
setattr(polygon, 'maz', uuid)
regions[uuid] = Region(uuid, polygon)
region_idx = STRtree(region.region for region in regions.values())
del regions_list
def get_centroid(node):
return next(centroid_idx.nearest(node, 1))
def get_region(node: Point):
regions = region_idx.query(node)
region = None
if len(regions):
region = regions[0].maz
return region
log.info('Loading network roads file.')
network = multiopen(networkpath, mode='rb')
parser = iter(iterparse(network, events=('start', 'end')))
evt, root = next(parser)
links = []
nodes = []
points = {}
count = 0
n = 1
for evt, elem in parser:
if evt == 'start':
if elem.tag == 'nodes':
log.info('Parsing nodes from network file.')
count = 0
n = 1
elif elem.tag == 'links':
if count != n << 1:
log.info(f'Parsed node {count}.')
log.info('Parsing links from network file.')
count = 0
n = 1
elif evt == 'end':
if elem.tag == 'node':
node_id = str(elem.get('id'))
x = float(elem.get('x'))
y = float(elem.get('y'))
point = Point(x, y)
region = get_region(point)
centroid = get_centroid((x, y, x, y))
nodes.append((node_id, region, centroid, dumps(point)))
points[node_id] = point
count += 1
if count == n:
log.info(f'Parsing node {count}.')
n <<= 1
elif elem.tag == 'link':
source_node = str(elem.get('from'))
terminal_node = str(elem.get('to'))
line = LineString(
(points[source_node], points[terminal_node]))
links.append(
(str(elem.get('id')), source_node, terminal_node,
float(elem.get('length')),
float(elem.get('freespeed')),
float(elem.get('capacity')),
float(elem.get('permlanes')), int(elem.get('oneway')),
str(elem.get('modes')), dumps(line)))
count += 1
if count == n:
log.info(f'Parsing link {count}.')
n <<= 1
if count % 100000:
root.clear()
if count != n << 1:
log.info(f'Parsing link {count}.')
network.close()
log.info('Writing parsed links and nodes to database.')
self.create_tables()
self.database.insert_values('nodes', nodes, 4)
self.database.insert_values('links', links, 10)
self.database.connection.commit()
