def _get_city(self, feature, side):
city = ''
if self.fix_cities:
from ebpub.db.models import get_city_locations
overlapping_cities = list(get_city_locations().filter(location__intersects=feature.geom.geos))
if overlapping_cities:
city = overlapping_cities[0].name
if self.verbose:
print >> sys.stderr, "overriding city to %s" % city
else:
fid = feature.get('TFID' + side)
if fid in self.faces_db:
face = self.faces_db[fid]
# Handle both 2010 and older census files.
pid = face.get('PLACEFP10') or face['PLACEFP']
if pid in self.places:
place = self.places[pid]
# Handle both 2010 and earlier Census files.
city = place.get('NAME10') or place['NAME']
return city
def _get_city(self, feature, side):
city = ''
if self.fix_cities:
from ebpub.db.models import get_city_locations
overlapping_cities = list(get_city_locations().filter(location__intersects=feature.geom.geos))
if overlapping_cities:
city = overlapping_cities[0].name
logger.debug("overriding city to %s" % city)
else:
fid = feature.get('TFID' + side)
if fid in self.faces_db:
face = self.faces_db[fid]
# Handle both 2010 and older census files.
# If none of these work, we simply get no city.
pid = face.get('PLACEFP10') or face.get('PLACEFP00') or face.get('PLACEFP')
if pid in self.places:
place = self.places[pid]
# Handle both 2010 and earlier Census files.
city = place.get('NAME10') or place['NAME']
return city
def proper_city(block):
"""
Returns the "proper" city for block, as a string.
This function is necessary because in the Block model there are
two sides of the street, and the city on the left side could
differ from the city on the right. This function uses knowledge
about metros and cities to return the canonical city
for our purposes for a block.
In some blocks, this may return an empty string - eg. in
unincorporated areas of rural counties, or when the city simply
isn't one we know anything about.
"""
from ebpub.db.models import get_city_locations
metro = get_metro()
if metro["multiple_cities"]:
cities = set([l.name.upper() for l in get_city_locations()])
else:
cities = set([metro["city_name"].upper()])
# Determine the block's city, which because of blocks that
# border two different municipalities, and because of metros
# with multiple cities like NYC and Miami-Dade, means checking
# both sides of the block and comparing with known city names.
block_city = u""
if block.left_city != block.right_city:
# Note that if both left_city and right_city are valid, then we
# return the left_city.
if block.left_city in cities:
block_city = block.left_city
elif block.right_city in cities:
block_city = block.right_city
elif block.left_city in cities:
block_city = block.left_city
if not block_city:
# We may be in some other area that isn't a city we know about.
# That shouldn't prevent us from doing anything useful with this block.
block_city = _first_not_false(block.left_city, block.right_city, u"")
return block_city
def proper_city(block):
"""
Returns the "proper" city for block, as a string.
This function is necessary because in the Block model there are
two sides of the street, and the city on the left side could
differ from the city on the right. This function uses knowledge
about metros and cities to return the canonical city
for our purposes for a block.
In some blocks, this may return an empty string - eg. in
unincorporated areas of rural counties, or when the city simply
isn't one we know anything about.
"""
from ebpub.db.models import get_city_locations
metro = get_metro()
if metro['multiple_cities']:
cities = set([l.name.upper() for l in get_city_locations()])
else:
cities = set([metro['city_name'].upper()])
# Determine the block's city, which because of blocks that
# border two different municipalities, and because of metros
# with multiple cities like NYC and Miami-Dade, means checking
# both sides of the block and comparing with known city names.
block_city = u''
if block.left_city != block.right_city:
# Note that if both left_city and right_city are valid, then we
# return the left_city.
if block.left_city in cities:
block_city = block.left_city
elif block.right_city in cities:
block_city = block.right_city
elif block.left_city in cities:
block_city = block.left_city
if not block_city:
# We may be in some other area that isn't a city we know about.
# That shouldn't prevent us from doing anything useful with this block.
block_city = _first_not_false(block.left_city, block.right_city, u'')
return block_city
def proper_city(block):
"""
Returns the "proper" city for block, as a string.
This function is necessary because in the Block model there are
two sides of the street, and the city on the left side could
differ from the city on the right. This function uses knowledge
about metros and cities to return the canonical city
for our purposes for a block.
Note that if ImproperCity is raised, it implies that there is a
mismatch between the block data and our understanding about what
should be in there. i.e., neither the left nor right side city is
one of our metros or city within a multiple-city metro.
"""
from ebpub.db.models import get_city_locations
metro = get_metro()
if metro["multiple_cities"]:
cities = set([l.name.upper() for l in get_city_locations()])
else:
cities = set([metro["city_name"].upper()])
# Determine the block's city, which because of blocks that
# border two different municipalities, and because of metros
# with multiple cities like NYC and Miami-Dade, means checking
# both sides of the block and comparing with known city names.
block_city = None
if block.left_city != block.right_city:
# Note that if both left_city and right_city are valid, then we
# return the left_city.
if block.left_city in cities:
block_city = block.left_city
elif block.right_city in cities:
block_city = block.right_city
elif block.left_city in cities:
block_city = block.left_city
if block_city is None:
raise ImproperCity("Error: Unknown city '%s' from block %s (%s)" % (block.left_city, block.id, block))
return block_city
def _get_city(self, feature, side):
city = ''
if self.fix_cities:
from ebpub.db.models import get_city_locations
overlapping_cities = list(get_city_locations().filter(
location__intersects=feature.geom.geos))
if overlapping_cities:
city = overlapping_cities[0].name
logger.debug("overriding city to %s" % city)
else:
fid = feature.get('TFID' + side)
if fid in self.faces_db:
face = self.faces_db[fid]
# Handle both 2010 and older census files.
# If none of these work, we simply get no city.
pid = face.get('PLACEFP10') or face.get(
'PLACEFP00') or face.get('PLACEFP')
if pid in self.places:
place = self.places[pid]
# Handle both 2010 and earlier Census files.
city = place.get('NAME10') or place['NAME']
return city
def gen_blocks(self, feature):
block_fields = {}
block_fields['right_zip'] = feature.get('RZIP')
block_fields['left_zip'] = feature.get('LZIP')
block_fields['right_from_num'] = feature.get('FROMRIGHT')
block_fields['right_to_num'] = feature.get('TORIGHT')
block_fields['left_from_num'] = feature.get('FROMLEFT')
block_fields['left_to_num'] = feature.get('TOLEFT')
block_fields['street'] = feature.get('STREET').upper().strip()
block_fields['prefix'] = feature.get('PRETYPE').upper().strip()
block_fields['suffix'] = feature.get('TYPE').upper().strip()
# For predir and postdir, OpenBlock expects either:
# 1) N, S, E, or W;
# 2) 2-letter combinations of these (NE, SW, etc); or
# 3) blank/Null
# In our current shapefiles, most of the data fits this paradigm, but
# there are a few '0' entries present, presumably intended to mean
# Null.
block_fields['predir'] = re.sub('[^NESW]', '', feature.get('PREDIR').upper().strip())
block_fields['postdir'] = re.sub('[^NESW]', '', feature.get('SUFDIR').upper().strip())
for side in ['left', 'right']:
if block_fields['%s_from_num' % side] == 0 and not (block_fields['%s_to_num' % side] % 2):
block_fields['%s_from_num' % side] = 2
# As of OpenBlock 1.2 these values must be strings, else they get turned into None
for side in ['left', 'right']:
block_fields['%s_from_num' % side] = str(block_fields['%s_from_num' % side])
block_fields['%s_to_num' % side] = str(block_fields['%s_to_num' % side])
cities = list(get_city_locations().filter(location__intersects=geos_with_projection(feature.geom, 4326)))
city_name = cities[0].name.upper() if cities else ''
for side in ('right', 'left'):
block_fields['%s_city' % side] = city_name
block_fields['%s_state' % side] = 'NC'
yield block_fields.copy()
def populate_intersections(*args, **kwargs):
# On average, there are 2.3 blocks per intersection. So for
# example in the case of Chicago, where there are 788,496 blocks,
# we'd expect to see approximately 340,000 intersections
Intersection.objects.all().delete()
logger.info("Starting to populate intersections")
metro = get_metro()
zipcodes = Location.objects.filter(location_type__name__istartswith="zip").exclude(name__startswith='Unknown')
def lookup_zipcode(pt):
for zipcode in zipcodes:
if zipcode.location.contains(pt):
return zipcode
intersections_seen = {}
for i in Intersection.objects.all():
intersections_seen[i.pretty_name] = i.id
intersections_seen[i.reverse_pretty_name()] = i.id
for bi in BlockIntersection.objects.iterator():
street_name = make_dir_street_name(bi.block)
i_street_name = make_dir_street_name(bi.intersecting_block)
# This tuple enables us to skip over intersections
# we've already seen. Since intersections are
# symmetrical---eg., "N. Kimball Ave. & W. Diversey
# Ave." == "W. Diversey Ave. & N. Kimball Ave."---we
# use both orderings.
seen_intersection = (u"%s & %s" % (street_name, i_street_name),
u"%s & %s" % (i_street_name, street_name))
if seen_intersection[0] not in intersections_seen and \
seen_intersection[1] not in intersections_seen:
if bi.block.left_city != bi.block.right_city:
# If we have Locations representing cities,
# find one that contains this bi's center.
from ebpub.db.models import get_city_locations
overlapping_cities = get_city_locations().filter(location__contains=bi.location)
if overlapping_cities:
city = overlapping_cities[0].name.upper()
else:
city = metro['city_name'].upper()
else:
city = bi.block.left_city
if bi.block.left_state != bi.block.right_state:
state = metro['state'].upper()
else:
state = bi.block.left_state
if (bi.block.left_zip != bi.block.right_zip or \
bi.intersecting_block.left_zip != bi.intersecting_block.right_zip) or \
(bi.block.left_zip != bi.intersecting_block.left_zip):
zipcode_obj = lookup_zipcode(bi.location)
if zipcode_obj:
zipcode = zipcode_obj.name
else:
zipcode = bi.block.left_zip
else:
zipcode = bi.block.left_zip
intersection = intersection_from_blocks(bi.block, bi.intersecting_block, bi.location, city, state, zipcode)
intersection.save()
logging.debug("Created intersection %s" % intersection)
bi.intersection = intersection
bi.save()
intersections_seen[seen_intersection[0]] = intersection.id
intersections_seen[seen_intersection[1]] = intersection.id
else:
if not bi.intersection:
bi.intersection_id = intersections_seen[seen_intersection[0]]
bi.save()
logger.debug("Already seen intersection %s" % " / ".join(seen_intersection))
logger.info("Finished populating intersections")
total = Intersection.objects.all().count()
if not total:
logger.warn("No intersections created, maybe you forgot to do populate_block_intersections first?")
def populate_intersections(*args, **kwargs):
# On average, there are 2.3 blocks per intersection. So for
# example in the case of Chicago, where there are 788,496 blocks,
# we'd expect to see approximately 340,000 intersections
# NOTE, we can't do a delete here because that cascades to BlockIntersection,
# and then we have nothing to work with.
# So the two functions should really always be called together.
logger.info(
"Starting to populate intersections, this can take some minutes...")
logger.warn("Deleting all %d existing intersections first" %
Intersection.objects.all().count())
Intersection.objects.all().delete()
logger.info("We have %d blockintersections" %
BlockIntersection.objects.all().count())
metro = get_metro()
zipcodes = Location.objects.filter(
location_type__name__istartswith="zip").exclude(
name__startswith='Unknown')
def lookup_zipcode(pt):
for zipcode in zipcodes:
if zipcode.location.contains(pt):
return zipcode
intersections_seen = {}
for i in Intersection.objects.all():
intersections_seen[i.pretty_name] = i.id
intersections_seen[i.reverse_pretty_name()] = i.id
for bi in BlockIntersection.objects.iterator():
street_name = make_dir_street_name(bi.block)
i_street_name = make_dir_street_name(bi.intersecting_block)
# This tuple enables us to skip over intersections
# we've already seen. Since intersections are
# symmetrical---eg., "N. Kimball Ave. & W. Diversey
# Ave." == "W. Diversey Ave. & N. Kimball Ave."---we
# use both orderings.
seen_intersection = (u"%s & %s" % (street_name, i_street_name),
u"%s & %s" % (i_street_name, street_name))
if seen_intersection[0] not in intersections_seen and \
seen_intersection[1] not in intersections_seen:
if bi.block.left_city != bi.block.right_city:
# If we have Locations representing cities,
# find one that contains this bi's center.
from ebpub.db.models import get_city_locations
overlapping_cities = get_city_locations().filter(
location__contains=bi.location)
if overlapping_cities:
city = overlapping_cities[0].name.upper()
else:
city = metro['city_name'].upper()
else:
city = bi.block.left_city
if bi.block.left_state != bi.block.right_state:
state = metro['state'].upper()
else:
state = bi.block.left_state
if (bi.block.left_zip != bi.block.right_zip or \
bi.intersecting_block.left_zip != bi.intersecting_block.right_zip) or \
(bi.block.left_zip != bi.intersecting_block.left_zip):
zipcode_obj = lookup_zipcode(bi.location)
if zipcode_obj:
zipcode = zipcode_obj.name
else:
zipcode = bi.block.left_zip
else:
zipcode = bi.block.left_zip
intersection = intersection_from_blocks(bi.block,
bi.intersecting_block,
bi.location, city, state,
zipcode)
intersection.save()
logger.debug("Created intersection %s" % intersection.pretty_name)
bi.intersection = intersection
bi.save()
intersections_seen[seen_intersection[0]] = intersection.id
intersections_seen[seen_intersection[1]] = intersection.id
else:
if not bi.intersection:
bi.intersection_id = intersections_seen[seen_intersection[0]]
bi.save()
logger.debug("Already seen intersection %s" %
" / ".join(seen_intersection))
logger.info("Finished populating intersections")
total = Intersection.objects.all().count()
if not total:
logger.warn(
"No intersections created, maybe you forgot to do populate_block_intersections first?"
)
return total