def update(self):
super(StreetUsePermitScraper, self).update()
for start, end, total, qs in queryset.batch(NewsItem.objects.filter(schema=self.schema)):
self.logger.debug("Updating location for %s - %s of %s" % (start, end, total))
for ni in qs:
self.set_location(ni)
ni.save()
def convert_to_json():
news_items = NewsItem.objects.filter(schema__slug="excavation-permits").order_by("id")
for start, end, total, qs in queryset.batch(news_items):
print "processing %s to %s of %s" % (start + 1, end, total)
for ni in qs:
# print ni.attributes['location_details']
cnn_list, details = ni.attributes["location_details"].split("___")
details = [repair_details(d) for d in details.split(";")]
location_details = {"cnn_list": cnn_list.split(","), "details": details}
# pprint(location_details)
ni.attributes["location_details"] = simplejson.dumps(location_details)
streets = set()
for detail in details:
if detail.has_key("street_name"):
streets.add(detail["street_name"])
else:
streets.add(detail["cross_street_1"])
location_name = ", ".join([smart_title(street) for street in streets])
if len(location_name) > 150:
location_name = VARIOUS
description = "; ".join([verbose_detail(loc) for loc in location_details["details"]])
ni.location_name = location_name
ni.description = description
ni.save()
def convert_to_json():
news_items = NewsItem.objects.filter(
schema__slug='excavation-permits').order_by('id')
for start, end, total, qs in queryset.batch(news_items):
print "processing %s to %s of %s" % (start + 1, end, total)
for ni in qs:
#print ni.attributes['location_details']
cnn_list, details = ni.attributes['location_details'].split('___')
details = [repair_details(d) for d in details.split(';')]
location_details = {
'cnn_list': cnn_list.split(','),
'details': details,
}
#pprint(location_details)
ni.attributes['location_details'] = simplejson.dumps(
location_details)
streets = set()
for detail in details:
if detail.has_key('street_name'):
streets.add(detail['street_name'])
else:
streets.add(detail['cross_street_1'])
location_name = ', '.join(
[smart_title(street) for street in streets])
if len(location_name) > 150:
location_name = VARIOUS
description = '; '.join(
[verbose_detail(loc) for loc in location_details['details']])
ni.location_name = location_name
ni.description = description
ni.save()
def convert_to_json():
news_items = NewsItem.objects.filter(schema__slug='excavation-permits').order_by('id')
for start, end, total, qs in queryset.batch(news_items):
print "processing %s to %s of %s" % (start + 1, end, total)
for ni in qs:
#print ni.attributes['location_details']
cnn_list, details = ni.attributes['location_details'].split('___')
details = [repair_details(d) for d in details.split(';')]
location_details = {
'cnn_list': cnn_list.split(','),
'details': details,
}
#pprint(location_details)
ni.attributes['location_details'] = simplejson.dumps(location_details)
streets = set()
for detail in details:
if detail.has_key('street_name'):
streets.add(detail['street_name'])
else:
streets.add(detail['cross_street_1'])
location_name = ', '.join([smart_title(street) for street in streets])
if len(location_name) > 150:
location_name = VARIOUS
description = '; '.join([verbose_detail(loc) for loc in location_details['details']])
ni.location_name = location_name
ni.description = description
ni.save()
def update(self):
super(StreetUsePermitScraper, self).update()
for start, end, total, qs in queryset.batch(
NewsItem.objects.filter(schema=self.schema)):
self.logger.debug("Updating location for %s - %s of %s" %
(start, end, total))
for ni in qs:
self.set_location(ni)
ni.save()
def deloacte_suburbs():
stats = {}
for start, end, total, qs in queryset.batch(NewsItem.objects.filter(schema__slug=SCHEMA)):
print "processing %s to %s of %s" % (start, end, total)
for ni in qs:
city = CITIES[ni.attributes['city']]
if ni.location is not None and city in SUBURBS:
ni.location = None
stats[city] = stats.get(city, 0) + 1
ni.save()
print pprint.pprint(stats)
def deloacte_suburbs():
stats = {}
for start, end, total, qs in queryset.batch(
NewsItem.objects.filter(schema__slug=SCHEMA)):
print "processing %s to %s of %s" % (start, end, total)
for ni in qs:
city = CITIES[ni.attributes['city']]
if ni.location is not None and city in SUBURBS:
ni.location = None
stats[city] = stats.get(city, 0) + 1
ni.save()
print pprint.pprint(stats)