def calculate_green_spaces(division_dir, green_spaces_dir):
division_path = find_shp_path(division_dir)
if type(green_spaces_dir) != list:
green_spaces_dir = [green_spaces_dir]
green_spaces_paths = [find_shp_path(i) for i in green_spaces_dir]
# create spatial index for green spaces
green_spaces = []
for green_spaces_path in green_spaces_paths:
green_spaces.extend(list(iter_shp_as_shapely(green_spaces_path)))
green_spaces_idx = create_spatial_index(green_spaces)
divisions = list(iter_shp_as_shapely(division_path))
total_divisions = len(divisions)
green_relevance = {}
for index, (id_division, division_shp) in enumerate(divisions):
area = division_shp.area
area_without_green = calc_area(division_shp, green_spaces,
green_spaces_idx)
area_green = area - area_without_green
green_relevance[id_division] = area_green / area
progress_msg = unicode(index) + " of " + unicode(total_divisions)
print(progress_msg, end="\r" * len(progress_msg))
return green_relevance
def create_buffered_shp(directory, distance, buffer_dir=BUFFER_DIR,
resolution=16, recalculate=False,
context_shp_or_polygon=None):
context_polygon = create_context_polygon(context_shp_or_polygon)
shp_path = pf.find_shp_path(directory)
shp_name = _create_shp_name(shp_path, distance)
# print("\n".join([directory, shp_name, buffer_dir]))
buffer_shp_path = _create_shp_path(directory, shp_name, buffer_dir)
# print(buffer_shp_path)
if not os.path.isfile(buffer_shp_path + ".shp") or recalculate:
# read shapefile with pyshp
sf_est = shapefile.Reader(shp_path)
# create buffers from shapely shapes
buffer_shapes = []
for shape in geo_utils.get_shapely_shapes(sf_est):
if not context_polygon or context_polygon.contains(shape):
buffer_shapes.append(create_buffer(shape, distance,
resolution,
context_polygon))
write_shapefile(sf_est, buffer_shapes, buffer_shp_path)
utils.copy_prj(shp_path, buffer_shp_path)
def read_dbf(shp_dir_or_dbf_path):
if os.path.isdir(shp_dir_or_dbf_path):
dbf_path = find_shp_path(shp_dir_or_dbf_path) + ".dbf"
else:
dbf_path = shp_dir_or_dbf_path
dbf = Dbf5(dbf_path)
return dbf.to_dataframe()
def create_buffered_shp(directory,
distance,
buffer_dir=BUFFER_DIR,
resolution=16,
recalculate=False,
context_shp_or_polygon=None):
context_polygon = create_context_polygon(context_shp_or_polygon)
shp_path = pf.find_shp_path(directory)
shp_name = _create_shp_name(shp_path, distance)
# print("\n".join([directory, shp_name, buffer_dir]))
buffer_shp_path = _create_shp_path(directory, shp_name, buffer_dir)
# print(buffer_shp_path)
if not os.path.isfile(buffer_shp_path + ".shp") or recalculate:
# read shapefile with pyshp
sf_est = shapefile.Reader(shp_path)
# create buffers from shapely shapes
buffer_shapes = []
for shape in geo_utils.get_shapely_shapes(sf_est):
if not context_polygon or context_polygon.contains(shape):
buffer_shapes.append(
create_buffer(shape, distance, resolution,
context_polygon))
write_shapefile(sf_est, buffer_shapes, buffer_shp_path)
utils.copy_prj(shp_path, buffer_shp_path)
def read_dbf(shp_dir_or_dbf_path):
if os.path.isdir(shp_dir_or_dbf_path):
dbf_path = find_shp_path(shp_dir_or_dbf_path) + ".dbf"
else:
dbf_path = shp_dir_or_dbf_path
dbf = Dbf5(dbf_path)
return dbf.to_dataframe()
def prj_to_proj4(prj_or_shp_path):
if prj_or_shp_path[-4:] == ".prj":
prj_path = prj_or_shp_path
else:
prj_path = find_shp_path(prj_or_shp_path) + ".prj"
crs = pycrs.loader.from_file(prj_path)
return crs
def test_find_shp_path(self):
directory = "shp/transporte/subte-estaciones"
shp_path = pf.find_shp_path(directory)
exp_shp_path = os.path.join(
pf.get_project_dir(),
"shp/transporte/subte-estaciones/estaciones_de_subte")
self.assertEqual(shp_path, exp_shp_path)
def test_find_shp_path(self):
directory = "shp/transporte/subte-estaciones"
shp_path = pf.find_shp_path(directory)
exp_shp_path = os.path.join(
pf.get_project_dir(),
"shp/transporte/subte-estaciones/estaciones_de_subte")
self.assertEqual(shp_path, exp_shp_path)
def count_points(points_shp_path, polygons_shp_path):
points_shp_path = pf.find_shp_path(points_shp_path)
polygons_shp_path = pf.find_shp_path(polygons_shp_path)
points = dict(iter_shp_as_shapely(points_shp_path))
polygons = dict(iter_shp_as_shapely(polygons_shp_path))
count = {id_polygon: 0 for id_polygon in polygons.iterkeys()}
for id_polygon, polygon in polygons.iteritems():
remove_points = []
for id_point, point in points.iteritems():
if polygon.contains(point):
count[id_polygon] += 1
remove_points.append(id_point)
for remove_point in remove_points:
del points[remove_point]
return count
def count_points(points_shp_path, polygons_shp_path):
points_shp_path = pf.find_shp_path(points_shp_path)
polygons_shp_path = pf.find_shp_path(polygons_shp_path)
points = dict(iter_shp_as_shapely(points_shp_path))
polygons = dict(iter_shp_as_shapely(polygons_shp_path))
count = {id_polygon: 0 for id_polygon in polygons.iterkeys()}
for id_polygon, polygon in polygons.iteritems():
remove_points = []
for id_point, point in points.iteritems():
if polygon.contains(point):
count[id_polygon] += 1
remove_points.append(id_point)
for remove_point in remove_points:
del points[remove_point]
return count
def create_shp_paths_dict(shps_dir, replacements=None):
"""Create a dict with paths to shps inside shps_dir directory."""
shp_paths_dict = {}
abs_shp_dirs = [os.path.join(shps_dir, shp_dir) for shp_dir in os.listdir(shps_dir)]
for shp_dir in (i for i in abs_shp_dirs if os.path.isdir(i) and i[0] != "."):
buffer_tag = shp_dir.split("-")[-1]
shp_name = os.path.basename(shp_dir).replace("-" + buffer_tag, "")
if replacements and shp_name in replacements:
shp_name = replacements[shp_name]
shp_path = pf.find_shp_path(os.path.join(shps_dir, shp_dir))
shp_paths_dict[shp_name + "-" + buffer_tag] = shp_path
return shp_paths_dict
def create_shp_paths_dict(shps_dir, replacements=None):
"""Create a dict with paths to shps inside shps_dir directory."""
shp_paths_dict = {}
abs_shp_dirs = [os.path.join(shps_dir, shp_dir) for shp_dir in
os.listdir(shps_dir)]
for shp_dir in (i for i in abs_shp_dirs if
os.path.isdir(i) and i[0] != "."):
buffer_tag = shp_dir.split("-")[-1]
shp_name = os.path.basename(shp_dir).replace("-" + buffer_tag, "")
if replacements and shp_name in replacements:
shp_name = replacements[shp_name]
shp_path = pf.find_shp_path(os.path.join(shps_dir, shp_dir))
shp_paths_dict[shp_name + "-" + buffer_tag] = shp_path
return shp_paths_dict
def create_shp_paths_dict(shps_dir, replacements=None):
"""Create a dict with paths to shps inside shps_dir directory."""
shp_paths_dict = {}
abs_shp_dirs = [os.path.join(shps_dir, shp_dir) for shp_dir in
os.listdir(shps_dir)]
for shp_dir in (i for i in abs_shp_dirs if
os.path.isdir(i) and i[0] != "."):
shp_name = os.path.basename(shp_dir).split("_")[1]
if replacements and shp_name in replacements:
shp_name = replacements[shp_name]
shp_path = find_shp_path(shp_dir)
shp_paths_dict[shp_name] = shp_path
return shp_paths_dict
import pandas as pd
from shapely.geometry import LineString, Point, Polygon
from shapely.ops import cascaded_union
from create_buffers import write_shapefile
# from calculate_weights import find_intersections
from path_finders import get_division_path, find_shp_path, get_indicators_path
from create_indicators import get_or_create_indicators_df
import time
import shapefile
from path_finders import get_transport_shp_path, get_division_path
from geo_utils import iter_shp_as_shapely, get_shapely_shapes
from shapely.geometry import LineString, Point, Polygon, MultiPolygon, MultiPoint
from global_vars import IDS_GCBA, AREA_LEVEL_SHP_NAME
SHP_INDIC_RADIO = find_shp_path(os.path.join("indicadores",
"radios_censo_2010"))
# AUXILIARY
def find_intersections(shape, shapes):
"""Find the shapes that intersect shape."""
# correct any topological errors of self-intersection
shape = shape.buffer(0)
for shp in shapes:
if shape.intersects(shp):
yield shp
def create_spatial_index(points, tolerance):
"""Create and rtree optimized spatial index from points.
def recalculate_indicators(new_shp_dir,
area_level,
skip=None,
subcategory=None,
omit_fields=None,
by_area=None):
skip = skip or []
by_area = by_area or []
# some fields are omitted always
if omit_fields:
if not type(omit_fields) == list:
omit_fields = [omit_fields]
omit_fields = OMIT_FIELDS
omit_fields.extend(omit_fields)
new_shp_path = find_shp_path(new_shp_dir)
shp_name = os.path.basename(new_shp_path)
sf = shapefile.Reader(new_shp_path)
df_indicators = get_indicators(area_level)
weights = get_weights(new_shp_path, area_level)
w = shapefile.Writer(shapefile.POLYGON)
indicators = _get_indicator_names(df_indicators)
for field in sf.fields[1:]:
w.field(*field)
for indicator in indicators:
field = [str(indicator), str("N"), 20, 18]
# print(indicator)
w.field(*field)
w.field(str("area_km2"), str("N"), 20, 18)
w.field(str("hab_km2"), str("N"), 20, 18)
# print(w.fields)
for record_shape in sf.iterShapeRecords():
record = record_shape.record
shape = record_shape.shape
# print(record[0])
if type(record[0]) == int:
id_record = unicode(record[0])
else:
id_record = unicode(record[0].decode("utf-8"))
if len(weights[id_record]) > 0:
calculated_indicators = _calc_indicators(indicators, df_indicators,
weights[id_record],
area_level, skip, by_area)
# print(calculated_indicators)
record.extend(calculated_indicators)
area = calculate_area(shape) / 1000000
record.append(area)
population = calculated_indicators[indicators.index(POPULATION)]
pop_density = population / area
record.append(pop_density)
w.record(*record)
w.poly(shapeType=shapefile.POLYGON, parts=[shape.points])
path = get_indicators_shp_path(shp_name, subcategory)
w.save(path)
utils.copy_prj(new_shp_path.decode("utf-8"), path)
def calculate_intersect_weights(division_dir,
buffer_dir,
weights_path,
empty_dirs=None,
force_buffer_sum=True):
"""Calculate perecentage of division_dir shapes intersecting buffer_dir.
Find which shapes in the division shapefile intersect with each shape in
buffer, and how much surface of the division shape is intersecting as well
as how much of the buffer shape is being intersected by that division
shape.
If a list with "empty shapefiles" is provided, the surfaces from
those shps that intersect with the buffer will not be taken into account
for the calculation of weights. They will be subtracted both from divisions
and buffers.
Args:
division_dir (str): Directory where shapefile with "inner shapes" is
in this case the polygons used to divide the greater shape.
buffer_dir (str): Directory where shapefile with "container shapes" is,
in this case the buffers calculated over points or lines.
empty_dirs (list): Paths to shapefiles with surfaces that shouldn't be
taken into account in the calculation.
Returns:
dict: The perecentage of intersected surface is returned as calculated
over total division area and total buffer area in a dictionary like
this dict[id_buffer][id_division][division]
>>> {
"id_buffer1":
{"id_division1": {"division": %_intersect_division_surface,
"buffer": %_intersect_buffer_surface},
"id_division2": {"division": %_intersect_division_surface,
"buffer": %_intersect_buffer_surface}
},
}
"""
division_path = find_shp_path(division_dir)
buffer_path = find_shp_path(buffer_dir)
if empty_dirs:
empty_paths = [find_shp_path(shp) for shp in empty_dirs]
# create spatial index for divisions
divisions = list(geo_utils.iter_shp_as_shapely(division_path))
divisions_idx = create_spatial_index(divisions)
# create spatial index for empty shapes
if empty_dirs:
empty_shps = []
for empty_path in empty_paths:
empty_shps.extend(list(geo_utils.iter_shp_as_shapely(empty_path)))
empty_idx = create_spatial_index(empty_shps)
weighted_intersections = {}
for id_buffer, buffer_shp in geo_utils.iter_shp_as_shapely(buffer_path):
buffer_shp = buffer_shp.buffer(0)
weighted_intersections[id_buffer] = {}
if empty_dirs:
buffer_area = calc_area(buffer_shp, empty_shps, empty_idx)
else:
buffer_area = buffer_shp.area
assert buffer_area > 0, "Buffer area can't be 0 " + unicode(id_buffer)
intersect_generator = find_intersections(buffer_shp, divisions,
divisions_idx)
for id_division, division_shp in intersect_generator:
division_shp = division_shp.buffer(0)
if empty_dirs:
division_area = calc_area(division_shp, empty_shps, empty_idx)
else:
division_area = division_shp.area
try:
intersect = buffer_shp.intersection(division_shp)
if empty_dirs:
intersect_area = calc_area(intersect, empty_shps,
empty_idx)
else:
intersect_area = intersect.area
except Exception as inst:
print("id_divison:", id_division,
"couldn't be intersected with id_buffer:", id_buffer)
print(inst, "\n")
weighted_intersections[id_buffer][id_division] = {
"division": round(intersect_area / division_area, 30),
"buffer": round(intersect_area / buffer_area, 30)
}
if force_buffer_sum:
total_w = sum([
i["buffer"]
for i in weighted_intersections[id_buffer].itervalues()
])
for id_division in weighted_intersections[id_buffer]:
weighted_intersections[id_buffer][id_division][
"buffer"] /= total_w
save_to_json(weighted_intersections, weights_path)
return weighted_intersections
def calculate_intersect_weights(division_dir, buffer_dir, weights_path, empty_dirs=None, force_buffer_sum=True):
"""Calculate perecentage of division_dir shapes intersecting buffer_dir.
Find which shapes in the division shapefile intersect with each shape in
buffer, and how much surface of the division shape is intersecting as well
as how much of the buffer shape is being intersected by that division
shape.
If a list with "empty shapefiles" is provided, the surfaces from
those shps that intersect with the buffer will not be taken into account
for the calculation of weights. They will be subtracted both from divisions
and buffers.
Args:
division_dir (str): Directory where shapefile with "inner shapes" is
in this case the polygons used to divide the greater shape.
buffer_dir (str): Directory where shapefile with "container shapes" is,
in this case the buffers calculated over points or lines.
weights_path (str): Json path where calculated weights will be saved.
empty_dirs (list): Paths to shapefiles with surfaces that shouldn't be
taken into account in the calculation.
force_buffer_sum (bool): Recalculate weights of a buffer so they always
sum 1.0.
Returns:
dict: The perecentage of intersected surface is returned as calculated
over total division area and total buffer area in a dictionary like
this dict[id_buffer][id_division][division]
>>> {
"id_buffer1":
{"id_division1": {"division": %_intersect_division_surface,
"buffer": %_intersect_buffer_surface},
"id_division2": {"division": %_intersect_division_surface,
"buffer": %_intersect_buffer_surface}
},
}
"""
division_path = pf.find_shp_path(division_dir)
buffer_path = pf.find_shp_path(buffer_dir)
if empty_dirs:
empty_paths = [pf.find_shp_path(shp) for shp in empty_dirs]
# create spatial index for divisions
divisions = list(geo_utils.iter_shp_as_shapely(division_path))
divisions_idx = create_spatial_index(divisions)
# create spatial index for empty shapes
if empty_dirs:
empty_shps = []
for empty_path in empty_paths:
empty_shps.extend(list(geo_utils.iter_shp_as_shapely(empty_path)))
empty_idx = create_spatial_index(empty_shps)
weighted_intersections = {}
for id_buffer, buffer_shp in geo_utils.iter_shp_as_shapely(buffer_path):
buffer_shp = buffer_shp.buffer(0)
weighted_intersections[id_buffer] = {}
if empty_dirs:
buffer_area = calc_area(buffer_shp, empty_shps, empty_idx)
else:
buffer_area = buffer_shp.area
assert buffer_area > 0, "Buffer area can't be 0 " + unicode(id_buffer)
intersect_generator = find_intersections(buffer_shp, divisions, divisions_idx)
for id_division, division_shp in intersect_generator:
division_shp = division_shp.buffer(0)
if empty_dirs:
division_area = calc_area(division_shp, empty_shps, empty_idx)
else:
division_area = division_shp.area
try:
intersect = buffer_shp.intersection(division_shp)
if empty_dirs:
intersect_area = calc_area(intersect, empty_shps, empty_idx)
else:
intersect_area = intersect.area
except Exception as inst:
print("id_divison:", id_division, "couldn't be intersected with id_buffer:", id_buffer)
print(inst, "\n")
weighted_intersections[id_buffer][id_division] = {
"division": round(intersect_area / division_area, 30),
"buffer": round(intersect_area / buffer_area, 30),
}
if force_buffer_sum:
total_w = sum([i["buffer"] for i in weighted_intersections[id_buffer].itervalues()])
for id_division in weighted_intersections[id_buffer]:
weighted_intersections[id_buffer][id_division]["buffer"] /= total_w
save_to_json(weighted_intersections, weights_path)
return weighted_intersections
def recalculate_indicators(new_shp_dir, area_level, skip=None,
subcategory=None, omit_fields=None, by_area=None):
skip = skip or []
by_area = by_area or []
# some fields are omitted always
if omit_fields:
if not type(omit_fields) == list:
omit_fields = [omit_fields]
omit_fields = OMIT_FIELDS
omit_fields.extend(omit_fields)
new_shp_path = find_shp_path(new_shp_dir)
shp_name = os.path.basename(new_shp_path)
sf = shapefile.Reader(new_shp_path)
df_indicators = get_indicators(area_level)
weights = get_weights(new_shp_path, area_level)
w = shapefile.Writer(shapefile.POLYGON)
indicators = _get_indicator_names(df_indicators)
for field in sf.fields[1:]:
w.field(*field)
for indicator in indicators:
field = [str(indicator), str("N"), 20, 18]
# print(indicator)
w.field(*field)
w.field(str("area_km2"), str("N"), 20, 18)
w.field(str("hab_km2"), str("N"), 20, 18)
# print(w.fields)
for record_shape in sf.iterShapeRecords():
record = record_shape.record
shape = record_shape.shape
# print(record[0])
if type(record[0]) == int:
id_record = unicode(record[0])
else:
id_record = unicode(record[0].decode("utf-8"))
if len(weights[id_record]) > 0:
calculated_indicators = _calc_indicators(indicators,
df_indicators,
weights[id_record],
area_level,
skip, by_area)
# print(calculated_indicators)
record.extend(calculated_indicators)
area = calculate_area(shape) / 1000000
record.append(area)
population = calculated_indicators[indicators.index(POPULATION)]
pop_density = population / area
record.append(pop_density)
w.record(*record)
w.poly(shapeType=shapefile.POLYGON, parts=[shape.points])
path = get_indicators_shp_path(shp_name, subcategory)
w.save(path)
utils.copy_prj(new_shp_path.decode("utf-8"), path)
import pandas as pd
from shapely.geometry import LineString, Point, Polygon
from shapely.ops import cascaded_union
from create_buffers import write_shapefile
# from calculate_weights import find_intersections
from path_finders import get_division_path, find_shp_path, get_indicators_path
from create_indicators import get_or_create_indicators_df
import time
import shapefile
from path_finders import get_transport_shp_path, get_division_path
from geo_utils import iter_shp_as_shapely, get_shapely_shapes
from shapely.geometry import LineString, Point, Polygon, MultiPolygon, MultiPoint
from global_vars import IDS_GCBA, AREA_LEVEL_SHP_NAME
SHP_INDIC_RADIO = find_shp_path(
os.path.join("indicadores", "radios_censo_2010"))
# AUXILIARY
def find_intersections(shape, shapes):
"""Find the shapes that intersect shape."""
# correct any topological errors of self-intersection
shape = shape.buffer(0)
for shp in shapes:
if shape.intersects(shp):
yield shp
def create_spatial_index(points, tolerance):
"""Create and rtree optimized spatial index from points.
