Example #1
0
 def update(self):
     super(StreetUsePermitScraper, self).update()
     for start, end, total, qs in queryset.batch(NewsItem.objects.filter(schema=self.schema)):
         self.logger.debug("Updating location for %s - %s of %s" % (start, end, total))
         for ni in qs:
             self.set_location(ni)
             ni.save()
Example #2
0
def convert_to_json():
    news_items = NewsItem.objects.filter(schema__slug="excavation-permits").order_by("id")
    for start, end, total, qs in queryset.batch(news_items):
        print "processing %s to %s of %s" % (start + 1, end, total)
        for ni in qs:
            # print ni.attributes['location_details']
            cnn_list, details = ni.attributes["location_details"].split("___")
            details = [repair_details(d) for d in details.split(";")]
            location_details = {"cnn_list": cnn_list.split(","), "details": details}
            # pprint(location_details)
            ni.attributes["location_details"] = simplejson.dumps(location_details)

            streets = set()
            for detail in details:
                if detail.has_key("street_name"):
                    streets.add(detail["street_name"])
                else:
                    streets.add(detail["cross_street_1"])

            location_name = ", ".join([smart_title(street) for street in streets])
            if len(location_name) > 150:
                location_name = VARIOUS

            description = "; ".join([verbose_detail(loc) for loc in location_details["details"]])

            ni.location_name = location_name
            ni.description = description
            ni.save()
Example #3
0
def convert_to_json():
    news_items = NewsItem.objects.filter(
        schema__slug='excavation-permits').order_by('id')
    for start, end, total, qs in queryset.batch(news_items):
        print "processing %s to %s of %s" % (start + 1, end, total)
        for ni in qs:
            #print ni.attributes['location_details']
            cnn_list, details = ni.attributes['location_details'].split('___')
            details = [repair_details(d) for d in details.split(';')]
            location_details = {
                'cnn_list': cnn_list.split(','),
                'details': details,
            }
            #pprint(location_details)
            ni.attributes['location_details'] = simplejson.dumps(
                location_details)

            streets = set()
            for detail in details:
                if detail.has_key('street_name'):
                    streets.add(detail['street_name'])
                else:
                    streets.add(detail['cross_street_1'])

            location_name = ', '.join(
                [smart_title(street) for street in streets])
            if len(location_name) > 150:
                location_name = VARIOUS

            description = '; '.join(
                [verbose_detail(loc) for loc in location_details['details']])

            ni.location_name = location_name
            ni.description = description
            ni.save()
Example #4
0
def convert_to_json():
    news_items = NewsItem.objects.filter(schema__slug='excavation-permits').order_by('id')
    for start, end, total, qs in queryset.batch(news_items):
        print "processing %s to %s of %s" % (start + 1, end, total)
        for ni in qs:
            #print ni.attributes['location_details']
            cnn_list, details = ni.attributes['location_details'].split('___')
            details = [repair_details(d) for d in details.split(';')]
            location_details = {
                'cnn_list': cnn_list.split(','),
                'details': details,
            }
            #pprint(location_details)
            ni.attributes['location_details'] = simplejson.dumps(location_details)


            streets = set()
            for detail in details:
                if detail.has_key('street_name'):
                    streets.add(detail['street_name'])
                else:
                    streets.add(detail['cross_street_1'])

            location_name = ', '.join([smart_title(street) for street in streets])
            if len(location_name) > 150:
                location_name = VARIOUS

            description = '; '.join([verbose_detail(loc) for loc in location_details['details']])


            ni.location_name = location_name
            ni.description = description
            ni.save()
Example #5
0
 def update(self):
     super(StreetUsePermitScraper, self).update()
     for start, end, total, qs in queryset.batch(
             NewsItem.objects.filter(schema=self.schema)):
         self.logger.debug("Updating location for %s - %s of %s" %
                           (start, end, total))
         for ni in qs:
             self.set_location(ni)
             ni.save()
Example #6
0
def deloacte_suburbs():
    stats = {}
    for start, end, total, qs in queryset.batch(NewsItem.objects.filter(schema__slug=SCHEMA)):
        print "processing %s to %s of %s" % (start, end, total)
        for ni in qs:
            city = CITIES[ni.attributes['city']]
            if ni.location is not None and city in SUBURBS:
                ni.location = None
                stats[city] = stats.get(city, 0) + 1
                ni.save()
    print pprint.pprint(stats)
Example #7
0
def deloacte_suburbs():
    stats = {}
    for start, end, total, qs in queryset.batch(
            NewsItem.objects.filter(schema__slug=SCHEMA)):
        print "processing %s to %s of %s" % (start, end, total)
        for ni in qs:
            city = CITIES[ni.attributes['city']]
            if ni.location is not None and city in SUBURBS:
                ni.location = None
                stats[city] = stats.get(city, 0) + 1
                ni.save()
    print pprint.pprint(stats)