def main(): wards = dict() wardpostcodes = [] streets = csv.DictReader(open("data/streets.txt")) council.util.truncate_all() for row in streets: if row.get("All Post Codes"): postcodes = row.get("All Post Codes").split(",") wardName = row.get("Ward") for postcode in postcodes: if wards.get(wardName) is None: ward = Ward(name=wardName.replace("&", "and")) ward.put() wards[wardName] = ward else: ward = wards.get(wardName) wardpostcodes.append(WardPostcode(ward=ward, postcode=postcode.strip().replace(" ", "").upper())) if len(wardpostcodes) > 99: print "Saving %s streets" % len(wardpostcodes) db.put(wardpostcodes) wardpostcodes = [] if len(wardpostcodes) > 0: print "Saving %s streets" % len(wardpostcodes) db.put(wardpostcodes) print "Created %s wards" % len(wards)
def truncate_ward(): """Unfortunatly, you can't just truncate the tables on app engine, so this method deletes them 300 at a time, scaling that back if it gets an error.""" print "Deleting WardPostcode objects" fetchRows = 300 rows = WardPostcode.all().fetch(fetchRows) while len(rows) > 0: try: db.delete(rows) print "Deleted %s WardPostcode objects" % len(rows) except db.Timeout: fetchRows = fetchRows - 50 print "Timeout, reducing rows fetched to %s" % fetchRows if fetchRows <= 0: print "Unable to fetch rows" exit(0) rows = WardPostcode.all().fetch(fetchRows) print "Deleting Ward objects" fetchRows = 300 rows = Ward.all().fetch(fetchRows) while len(rows) > 0: try: db.delete(rows) print "Deleted %s Ward objects" % len(rows) except db.Timeout: fetchRows = fetchRows - 50 print "Timeout, reducing rows fetched to %s" % fetchRows if fetchRows <= 0: print "Unable to fetch rows" exit(0) rows = Ward.all().fetch(fetchRows)
def parseWards(self): # Find the map element with the ID "map" mapNode = self.dom.get_element_by_id("map") for action, elem in etree.iterwalk(mapNode, tag="area"): print "Downloading %s" % elem.get("alt") # Find the ward object for this ward. if self.wards.get(elem.get("alt")): ward = self.wards.get(elem.get("alt")) else: try: ward = Ward.all().filter("name =", elem.get("alt")).fetch(1)[0] self.wards[elem.get("alt")] = ward except NameError, message: print "Unable to find %s ward (%s)" % (elem.get("alt"), message) continue CouncillorInfo(elem.get("href"), ward)