def set_network_grid_intersection_lengths(self):
logger.debug("set_network_grid_intersection_lengths called")
if RTREE:
# segment the network into edges intersecting grid squares
lines = list(self.graph.lines_iter())
# create an index ahead of time for fast lookups
geo_idx = index.Index()
logger.debug("Creating index")
for i, l in enumerate(lines):
geo_idx.insert(i, l.bounds)
self.network_length_in_grid = []
logger.debug("Starting iteration over %d sample units",
self.n_sample_units)
tic = time()
for i, t in enumerate(self.sample_units):
sq = shapely_rectangle_from_vertices(*t)
length_tally = 0.
for j in geo_idx.intersection(sq.bounds):
length_tally += sq.intersection(lines[j]).length
self.network_length_in_grid.append(length_tally)
logger.debug("Complete in %f s", time() - tic)
self.network_length_in_grid = np.array(self.network_length_in_grid)
else:
# this older version is SLOW. Avoid if at all possible
# segment the network into edges intersecting grid squares
lines = list(self.graph.lines_iter())
self.network_length_in_grid = []
for t in self.sample_units:
sq = shapely_rectangle_from_vertices(*t)
this_int = [
sq.intersection(t) for t in lines if sq.intersects(t)
]
this_length = sum([t.length for t in this_int])
self.network_length_in_grid.append(this_length)
self.network_length_in_grid = np.array(self.network_length_in_grid)
def run_validation(all_data=None,
cs=(100, 800),
t0=300,
sample_unit_size=50,
n_validation=50):
"""
Run a validation on the simulated data and pretrained SEPP model
:param all_data:
:param cs: Column spacings to use. Optional - default is [100, 800]
:param t0: Initial cutoff
:param sample_unit_size: Size of grid square to use for validation
:param n_validation: Number of validation iterations to use
:return:
"""
if all_data is None:
all_data = load_saved_data()
domain = shapely_rectangle_from_vertices(*domain_extent)
vb_res = {}
for c in cs:
try:
the_sepp = all_data[c]['sepp'][0]
the_data = all_data[c]['data'][0]
txy = the_data.time.adddim(the_data.space.to_cartesian())
vb = validate.SeppValidationPreTrainedModel(
txy,
the_sepp,
include_predictions=False,
include=('full_static', ))
vb.set_t_cutoff(t0)
vb.set_sample_units(sample_unit_size)
vb_res[c] = vb.run(1, n_iter=n_validation)
except Exception:
pass
return vb_res
def plot_sepp_bg_surface(sepp_obj, domain=None, **kwargs):
k = sepp_obj.bg_kde
# if x_range is None:
# x_range = [min(sepp_obj.data.toarray(1)), max(sepp_obj.data.toarray(1))]
# y_range = [min(sepp_obj.data.toarray(2)), max(sepp_obj.data.toarray(2))]
# loc = DataArray.from_meshgrid(*np.meshgrid(
# np.linspace(x_range[0], x_range[1], npt),
# np.linspace(y_range[0], y_range[1], npt),
# ))
# z = k.partial_marginal_pdf(loc)
# if ax is None:
# fig = plt.figure()
# ax = fig.add_subplot(111)
# ax.contourf(loc.toarray(0), loc.toarray(1), z, 50)
# ax.set_aspect('equal')
# if domain is not None:
func = lambda x, y: k.partial_marginal_pdf(DataArray.from_meshgrid(x, y))
if domain is None:
xmin, xmax = min(sepp_obj.data.toarray(1)), max(
sepp_obj.data.toarray(1))
ymin, ymax = min(sepp_obj.data.toarray(2)), max(
sepp_obj.data.toarray(2))
domain = shapely_rectangle_from_vertices(xmin, ymin, xmax, ymax)
plot_surface_function_on_polygon(domain, func=func, **kwargs)
plt.axis('equal')
def create_grid_squares_shapefile(outfile, side_length=250):
mpoly = get_boundary()
ipoly, fpoly, full = spatial.create_spatial_grid(mpoly, side_length)
field_description = {'id': {'fieldType': 'N'}}
id = range(1, len(fpoly) + 1)
full_polys = [spatial.shapely_rectangle_from_vertices(*t) for t in fpoly]
spatial.write_polygons_to_shapefile(outfile,
full_polys,
field_description=field_description,
id=id)
def plot(self,
show_sample_units=True,
show_sample_points=False,
show_prediction=True,
fmax=0.9,
cmap='Reds',
ax=None,
**kwargs):
if ax is None:
fig = plt.figure()
ax = fig.add_subplot(111)
ax.set_aspect('equal')
plot_shapely_geos(self.poly, facecolor='none', edgecolor='k', ax=ax)
if show_prediction:
# create dictionary of segment colours for plotting
# this requires creating a norm instance and using that to index a colourmap
cmapper = colour_mapper(self.prediction_values, fmax=fmax, vmin=0)
for pv, grid in zip(self.prediction_values, self.sample_units):
sq = shapely_rectangle_from_vertices(*grid)
plot_shapely_geos(sq,
ax=ax,
facecolor=cmapper.to_rgba(pv),
edgecolor='none',
alpha=kwargs.pop('alpha', 0.4))
if show_sample_units:
plot_shapely_geos([
shapely_rectangle_from_vertices(*grid)
for grid in self.sample_units
],
ax=ax,
facecolor='none')
if show_sample_points:
plt.scatter(*self.sample_points.separate, s=10, c='k', marker='o')
# remove x and y ticks as these rarely add anything
ax.set_xticks([])
ax.set_yticks([])
plt.draw()
def run_anisotropic_k(net_pts, dmax=400, nsim=50):
bds = net_pts.graph.extent
xr = bds[2] - bds[0]
yr = bds[3] - bds[1]
buff = 0.01
bd_poly = shapely_rectangle_from_vertices(
bds[0] - xr * buff,
bds[1] - yr * buff,
bds[2] + xr * buff,
bds[3] + yr * buff,
)
xy = net_pts.to_cartesian()
# default behaviour; start at pi/8 and move in intervals of pi/4
rk = ripley.RipleyKAnisotropicC2(xy, dmax, bd_poly)
rk.process()
u = np.linspace(0, dmax, 100)
k_obs = rk.compute_k(u)
k_sim = rk.run_permutation(u, niter=nsim)
return u, k_obs, k_sim
def plot_compare_network_planar(net_v,
planar_v,
cutoff_t=None,
poly=None,
training=None,
testing=None,
cmap='Reds',
fmax=0.99,
bounds=None,
show_grid_ranking=False):
"""
Produce plots comparing network and grid-based prediction methods.
:param net_v: Validation object
:param planar_v: Validation object
:param cutoff_t: cutoff time, defaults to whatever is currently assigned
:param poly: If supplied, plot the boundary poly
:param training: If supplied, plot training data (i.e. past crimes)
:param testing: If supplied, plot testing data (i.e. future crimes)
:param cmap:
:param fmax: Maximum colour level as a percentile
:param bounds: If supplied, zoom plot to this region
:param show_grid_ranking: If True, overlay numbers to show the order in which grid squares will be selected
:return:
"""
net_obj = net_v.data.graph
# compute grid and net based prediction values
net_z = net_v.model.predict(cutoff_t + 1., net_v.sample_points)
planar_z = planar_v.predict(cutoff_t + 1.)
# aggregate to grid - this is stolen directly from the ROC code
planar_grid_values = []
tally = 0
for n in planar_v.roc.n_sample_point_per_unit:
planar_grid_values.append(np.mean(planar_z[tally:tally + n]))
tally += n
planar_grid_values = np.array(planar_grid_values)
# get max values
idx = bisect.bisect_left(np.linspace(0, 1, net_z.size), fmax)
net_vmax = sorted(net_z)[idx]
idx = bisect.bisect_left(np.linspace(0, 1, planar_grid_values.size), fmax)
planar_vmax = sorted(planar_grid_values)[idx]
# get training data sizes if applicable
# these are proportional to the time component
if training:
training_x = training.toarray(1)
training_y = training.toarray(2)
training_t = training.toarray(0)
training_size = 500 * np.exp(-(cutoff_t - training_t) / 28.)
# get testing data sizes if applicable
if testing:
testing_x = testing.toarray(1)
testing_y = testing.toarray(2)
testing_t = testing.toarray(0)
testing_size = 500 * np.exp(-(testing_t - cutoff_t) / 3.)
fig, axs = plt.subplots(1, 2, sharex=True, sharey=True, figsize=(15, 7))
# left hand plot
# plot basic network structure
net_plots.plot_network_edge_lines(net_obj.lines_iter(), ax=axs[0])
# overlay shaded grid
grid = [
shapely_rectangle_from_vertices(*t) for t in planar_v.roc.sample_units
]
plotting.spatial.plot_shaded_regions(grid,
planar_grid_values,
ax=axs[0],
fmax=fmax,
cmap=cmap,
alpha=0.8,
colorbar=False,
scale_bar=None)
if poly:
plotting.spatial.plot_shapely_geos(poly,
fc='none',
ec='b',
lw=1.5,
ax=axs[0])
if training:
# zorder to elevate above the grid surface
axs[0].scatter(training_x,
training_y,
edgecolors='b',
facecolors='none',
s=training_size,
lw=1.5,
zorder=10)
if testing:
# zorder to elevate above the grid surface
axs[0].scatter(testing_x,
testing_y,
edgecolors='g',
facecolors='none',
s=testing_size,
lw=1.5,
zorder=10)
# right hand plot
axs[1].axis('off')
# unfilled grid
plotting.spatial.plot_shapely_geos(grid,
fc='None',
ec='k',
alpha=0.8,
ax=axs[1])
# scatter density
xy = net_v.sample_points.to_cartesian()
axs[1].scatter(xy.toarray(0),
xy.toarray(1),
c=net_z,
s=20,
cmap=cmap,
edgecolor='none')
net_plots.plot_network_edge_lines(net_obj.lines_iter(), ax=axs[1])
if poly:
plotting.spatial.plot_shapely_geos(poly,
fc='none',
ec='b',
lw=1.5,
ax=axs[1])
if training:
axs[1].scatter(training_x,
training_y,
edgecolors='b',
facecolors='none',
s=training_size,
lw=1.5)
if testing:
axs[1].scatter(testing_x,
testing_y,
edgecolors='g',
facecolors='none',
s=testing_size,
lw=1.5)
plt.tight_layout(pad=0., h_pad=0., w_pad=0., rect=(0, 0, 1, 1))
if bounds is not None:
axs[0].axis(bounds)
if show_grid_ranking:
# compute ranking
grid_ranking = np.argsort(planar_grid_values)
for i in range(grid_ranking.size):
c = grid[i].centroid
axs[0].text(c.x,
c.y,
str(i + 1),
fontsize=20,
color='k',
zorder=11,
horizontalalignment='center',
verticalalignment='center',
clip_on=True)
ax.set_position([0, 0, 1, 1])
# GRID-BASED prediction
data_txy = all_data.time.adddim(all_data.space.to_cartesian(), type=models.CartesianSpaceTimeData)
sk_planar = hotspot.STLinearSpaceExponentialTime(radius=h, mean_time=t_decay)
vb_planar = validation.ValidationIntegration(data_txy, sk_planar, spatial_domain=boro_poly, include_predictions=True)
vb_planar.set_t_cutoff(INITAL_CUTOFF)
vb_planar.set_sample_units(grid_length, n_samples)
res = vb_planar.run(1, n_iter=1)
zp = res['prediction_values'][0]
fig = plt.figure(figsize=figsize)
ax = fig.add_subplot(111)
plots.plot_network_edge_lines(itn_net.lines_iter(), ax=ax) # specify ax
# overlay shaded grid
grid = [shapely_rectangle_from_vertices(*t) for t in vb_planar.roc.sample_units]
plotting.spatial.plot_shaded_regions(grid,
zp,
ax=ax,
fmax=0.98,
cmap=cmap,
colorbar=False,
scale_bar=None)
ax.axis(axlims)
ax.set_position([0, 0, 1, 1])
# ADMINISTRATIVE UNIT based predictions
wards = cad.get_camden_wards(as_shapely=True)
sample_points = [spatial.random_points_within_poly(w, n_samples) for w in wards]
sample_xy = np.concatenate([np.array(t).transpose() for t in sample_points])
res = []
for fn in files:
with open(os.path.join(OUT_DIR, 'birmingham', fn), 'r') as f:
res.append(pickle.load(f))
return res
if __name__ == '__main__':
# load existing results: net sample units and prediction values, grid sample units and prediction values
# grid sample units have been recomputed using compute_planar_grid
grid_units, grid_vals, net_units, net_vals = load_prediction_values_and_sample_units(
)
# reduce the network to enable faster plotting
bbox = shapely_rectangle_from_vertices(XMIN, YMIN, XMAX, YMAX)
net = net_units[0].graph
net_reduced = net.within_boundary(bbox)
# find remaining sample units
fids = set([e.fid for e in net_reduced.edges()])
net_idx = [i for i, e in enumerate(net_units) if e.fid in fids]
net_units_reduced = [net_units[i] for i in net_idx]
net_lines_reduced = [
t.linestring.intersection(bbox) for t in net_units_reduced
]
net_vals_reduced = net_vals[:, net_idx]
# repeat for grid results
grid_idx = [
i for i, p in enumerate(grid_units['polys']) if p.intersects(bbox)
pred_t = 180.
arr_fun = lambda x, y: plots.txy_to_cartesian_data_array(np.ones_like(x) * pred_t, x, y)
pred_fun = lambda x, y: r.predict(arr_fun(x, y))
xx, yy, zz = spatial.plot_surface_function_on_polygon(domain, pred_fun, ax=ax, fmax=0.98, cmap=cm, dx=50)
ax.axis('off')
plt.tight_layout()
fig.savefig('continuous_heatmap.png', dpi=300)
fig.savefig('continuous_heatmap.pdf', dpi=300)
# (2) top 10 % grid squares
ipolys, full_extents, full_grid_square = create_spatial_grid(domain, 200)
grid_values = []
for xmin, ymin, xmax, ymax in full_extents:
i, j = np.where((xx >= xmin) & (xx < xmax) & (yy >= ymin) & (yy < ymax))
grid_values.append(zz[i, j].sum())
sort_idx = np.argsort(grid_values)[::-1]
top_10_pct = sort_idx[:int(len(sort_idx) * 0.1)]
top_10_pct_grids = [shapely_rectangle_from_vertices(*full_extents[i]) for i in top_10_pct]
fig = plt.figure(figsize=(5, 6))
ax = fig.add_subplot(111)
net.plot_network(ax=ax)
spatial.plot_shapely_geos(domain, ax=ax)
spatial.plot_shapely_geos(top_10_pct_grids, ax=ax, fc='r', ec='none', lw=1.5, alpha=0.7)
ax.axis('off')
plt.tight_layout()
fig.savefig('top_10_pct_grids.png', dpi=300)
fig.savefig('top_10_pct_grids.pdf', dpi=300)
from point_process import simulate, estimation, models
from analysis.spatial import shapely_rectangle_from_vertices
from scripts.rosser_cheng_isotropic_sepp.ripleys_k_analysis import run_anisotropic_k
from scripts import OUT_DIR
import numpy as np
import os
import dill
from matplotlib import pyplot as plt
from matplotlib.patches import Circle
subdir = os.path.join(OUT_DIR, 'anisotropy_simulation_study',
'patchy_background')
domain_extent = [0., 0., 5000., 5000.]
boundary = shapely_rectangle_from_vertices(*domain_extent)
col_spacings = [100., 200., 400., 800.]
row_space = 100.
sim_bg_sigma = 100.
def simulate_data_and_train():
niter = 10
sim_t_total = 1000.
sim_num_to_prune = 400
# sim_bg_sigma = 100.
train_kwargs = {
'niter': 100,
}