def main():
wards = dict()
wardpostcodes = []
streets = csv.DictReader(open("data/streets.txt"))
council.util.truncate_all()
for row in streets:
if row.get("All Post Codes"):
postcodes = row.get("All Post Codes").split(",")
wardName = row.get("Ward")
for postcode in postcodes:
if wards.get(wardName) is None:
ward = Ward(name=wardName.replace("&", "and"))
ward.put()
wards[wardName] = ward
else:
ward = wards.get(wardName)
wardpostcodes.append(WardPostcode(ward=ward, postcode=postcode.strip().replace(" ", "").upper()))
if len(wardpostcodes) > 99:
print "Saving %s streets" % len(wardpostcodes)
db.put(wardpostcodes)
wardpostcodes = []
if len(wardpostcodes) > 0:
print "Saving %s streets" % len(wardpostcodes)
db.put(wardpostcodes)
print "Created %s wards" % len(wards)
def truncate_ward(): """Unfortunatly, you can't just truncate the tables on app engine, so this method deletes them 300 at a time, scaling that back if it gets an error.""" print "Deleting WardPostcode objects" fetchRows = 300 rows = WardPostcode.all().fetch(fetchRows) while len(rows) > 0: try: db.delete(rows) print "Deleted %s WardPostcode objects" % len(rows) except db.Timeout: fetchRows = fetchRows - 50 print "Timeout, reducing rows fetched to %s" % fetchRows if fetchRows <= 0: print "Unable to fetch rows" exit(0) rows = WardPostcode.all().fetch(fetchRows) print "Deleting Ward objects" fetchRows = 300 rows = Ward.all().fetch(fetchRows) while len(rows) > 0: try: db.delete(rows) print "Deleted %s Ward objects" % len(rows) except db.Timeout: fetchRows = fetchRows - 50 print "Timeout, reducing rows fetched to %s" % fetchRows if fetchRows <= 0: print "Unable to fetch rows" exit(0) rows = Ward.all().fetch(fetchRows)
def parseWards(self):
# Find the map element with the ID "map"
mapNode = self.dom.get_element_by_id("map")
for action, elem in etree.iterwalk(mapNode, tag="area"):
print "Downloading %s" % elem.get("alt")
# Find the ward object for this ward.
if self.wards.get(elem.get("alt")):
ward = self.wards.get(elem.get("alt"))
else:
try:
ward = Ward.all().filter("name =", elem.get("alt")).fetch(1)[0]
self.wards[elem.get("alt")] = ward
except NameError, message:
print "Unable to find %s ward (%s)" % (elem.get("alt"), message)
continue
CouncillorInfo(elem.get("href"), ward)
