def save_random_newsitem(schema, i, block):
title = '%d Random %s %s' % (i, schema.name, uuid.uuid4())
print "Creating %r" % title
item = NewsItem()
item.title = title
item.schema = schema
item.description = gibberis.ch.freeform.random_text(get_text_corpus(), 300)
item.url = 'http://example.com/%s/%d' % (schema.slug, i)
date = random_datetime(7.0)
item.pub_date = date
item.item_date = date.date()
item.location_name = block.pretty_name
try:
item.location = block.geom.centroid
except AttributeError:
item.location = block.geom
# Populate the attributes.
attrs = {}
for schemafield in schema.schemafield_set.all():
attrs[schemafield.name] = random_schemafield_value(schemafield)
print "Added: %s at %s (%s)" % (item.title, item.location_name,
item.location.wkt)
# Need to save before we can have foreign keys from the attributes
# or subclass.
item.save()
if attrs:
item.attributes = attrs
# That implicitly saves in the old model, but not the new.
item.save()
def main():
""" Download Calendar RSS feed and update database """
url = """http://calendar.boston.com/search?acat=&cat=&commit=Search\
&new=n&rss=1&search=true&sort=0&srad=20&srss=50&ssrss=5&st=event\
&st_select=any&svt=text&swhat=&swhen=today&swhere=&trim=1"""
schema = 'events'
parser = OptionParser()
parser.add_option('-q',
'--quiet',
action="store_true",
dest="quiet",
default=False,
help="no output")
(options, args) = parser.parse_args()
if len(args) > 0:
return parser.error('script does not take any arguments')
try:
schema = Schema.objects.get(slug=schema)
except Schema.DoesNotExist:
print "Schema (%s): DoesNotExist" % schema
sys.exit(0)
feed = feedparser.parse(url)
for entry in feed.entries:
try:
item = NewsItem.objects.get(title=entry.title,
description=entry.description)
status = "Updated"
except NewsItem.DoesNotExist:
item = NewsItem()
status = "Added"
try:
item.schema = schema
item.title = entry.title
item.description = entry.description
item.url = entry.link
item.item_date = datetime.datetime(*entry.updated_parsed[:6])
item.pub_date = datetime.datetime(*entry.updated_parsed[:6])
item.location = Point(
(float(entry['geo_long']), float(entry['geo_lat'])))
item.save()
if not options.quiet:
print "%s: %s" % (status, item.title)
except ValueError:
if not options.quiet:
print "unexpected error:", sys.exc_info()[1]
def _make_items(number, schema):
items = []
from django.conf import settings
local_tz = pytz.timezone(settings.TIME_ZONE)
curdate = datetime.datetime.now().replace(microsecond=0, tzinfo=local_tz)
inc = datetime.timedelta(days=-1)
for i in range(number):
desc = '%s item %d' % (schema.slug, i)
items.append(NewsItem(schema=schema, title=desc,
description=desc,
item_date=curdate.date(),
pub_date=curdate,
location=geos.Point(0,0)))
curdate += inc
return items
def main(argv=None):
url = 'http://search.boston.com/search/api?q=*&sort=-articleprintpublicationdate&subject=boston&scope=bonzai'
schema = 'local-news'
try:
schema = Schema.objects.get(slug=schema)
except Schema.DoesNotExist:
print "Schema (%s): DoesNotExist" % schema
sys.exit(0)
f = feedparser.parse(url)
geocoder = SmartGeocoder()
for e in f.entries:
try:
item = NewsItem.objects.get(title=e.title,
description=e.description)
except NewsItem.DoesNotExist:
item = NewsItem()
item.schema = schema
item.title = e.title
item.description = e.description
item.url = e.link
#item.location_name = e['x-calconnect-street']
item.item_date = datetime.datetime(*e.updated_parsed[:6])
item.pub_date = datetime.datetime(*e.updated_parsed[:6])
try:
if 'point' in e:
x, y = e.point.split(' ')
else:
x, y = e.georss_point.split(' ')
item.location = Point((float(y), float(x)))
item.save()
except:
pass
print "Added: %s" % item.title
def _update_service_request(self, sreq):
service_request_id = self._get_request_field(sreq, 'service_request_id')
if not service_request_id:
log.info("Skipping request with no request id (may be in progress)!")
return
# pull out the location first, if we can't do this, we don't want it.
try:
point = Point(float(sreq.find('long').text),
float(sreq.find('lat').text),
srid=4326)
except:
log.debug("Skipping request with invalid location (%s)" % service_request_id)
return
if self.bounds is not None:
if not self.bounds.intersects(point):
log.debug("Skipping request at %s, outside bounds" % point)
return
try:
ni = NewsItem.objects.filter(schema=self.schema).by_attribute(self.service_request_id_field,
service_request_id).all()[0]
log.info('updating existing request %s' % service_request_id)
except IndexError:
# create the NewsItem
ni = NewsItem(schema=self.schema)
log.info('created new service request %s' % service_request_id)
ni.title = self._get_request_field(sreq, 'service_name')
ni.description = self._get_request_field(sreq, 'description')
ni.location = point
ni.location_name = self._get_request_field(sreq, 'address')
# try to reverse geocde this point
if not ni.location_name:
try:
block, distance = reverse_geocode(ni.location)
ni.location_name = block.pretty_name
except:
log.debug("Failed to reverse geocode item %s" % service_request_id)
# try to pull the requested_datetime into pubdate/itemdate
# default to now.
try:
ni.pub_date = pyrfc3339.parse(sreq.find('requested_datetime').text)
except:
ni.pub_date = datetime.datetime.utcnow()
log.info("Filling in current time for pub_date on item with no requested_datetime (%s)" % service_request_id)
ni.item_date = datetime.date(ni.pub_date.year, ni.pub_date.month, ni.pub_date.day)
if self.html_url_template:
ni.url = self.html_url_template.replace('{id}', service_request_id)
log.info('Assigning html url "%s" to %s' % (ni.url, service_request_id))
ni.save()
ni.attributes['service_request_id'] = service_request_id
# varchar fields
for fieldname in ('request_id', 'service_code', 'address_id',
'media_url', 'status_notes', 'service_notice'):
val = self._get_request_field(sreq, fieldname)
if val != '':
if len(val) < 4096:
ni.attributes[fieldname] = val
else:
log.info("truncating value for %s (%s)" % (fieldname, val))
ni.attributes[fieldname] = val[0:4096]
# text fields
for fieldname in ('service_notice'):
val = self._get_request_field(sreq, fieldname)
if val != '':
ni.attributes[fieldname] = val
# datetime fields
for fieldname in ('expected_datetime', 'requested_datetime'):
val = self._get_request_field(sreq, fieldname)
if val == '':
continue
# try to parse it
try:
ni.attributes[fieldname] = pyrfc3339.parse(val)
except ValueError:
# invalid date, just omit
log.info('Omitting invalid datetime field %s = %s' % (fieldname, val))
pass
# lookups
for fieldname in ('service_name', 'agency_responsible', 'status'):
val = self._get_request_field(sreq, fieldname)
if val == '':
ni.attributes[fieldname] = self._lookup_for(fieldname, 'Unknown')
ni.attributes[fieldname] = self._lookup_for(fieldname, val)
def update():
""" Download Calendar RSS feed and update database """
logger.info("Starting add_events")
url = """http://calendar.boston.com/search?acat=&cat=&commit=Search\
&new=n&rss=1&search=true&sort=0&srad=20&srss=50&ssrss=5&st=event\
&st_select=any&svt=text&swhat=&swhen=today&swhere=&trim=1"""
schema = 'events'
try:
schema = Schema.objects.get(slug=schema)
except Schema.DoesNotExist:
logger.error("Schema (%s): DoesNotExist" % schema)
sys.exit(1)
feed = feedparser.parse(url)
addcount = updatecount = 0
for entry in feed.entries:
title = convert_entities(entry.title).strip()
# Putting 'event' in the title is redundant, ticket #227
if title.lower().startswith('event: '):
title = title[7:]
try:
item = NewsItem.objects.get(title=title, schema__id=schema.id)
status = "updated"
except NewsItem.DoesNotExist:
item = NewsItem()
status = "added"
except NewsItem.MultipleObjectsReturned:
logger.warn(
"Multiple entries matched title %r, event titles are not unique?"
% title)
continue
try:
item.location_name = entry.get(
'xcal_x-calconnect-street') or entry.get(
'x-calconnect-street') or u''
item.schema = schema
item.title = title
item.description = convert_entities(entry.description)
item.url = entry.link
item.item_date = datetime.datetime(*entry.updated_parsed[:6])
item.pub_date = datetime.datetime(*entry.updated_parsed[:6])
item.location = Point(
(float(entry['geo_long']), float(entry['geo_lat'])))
if (item.location.x, item.location.y) == (0.0, 0.0):
logger.warn("Skipping %r, bad location 0,0" % item.title)
continue
if not item.location_name:
# Fall back to reverse-geocoding.
from ebpub.geocoder import reverse
try:
block, distance = reverse.reverse_geocode(item.location)
logger.info(" Reverse-geocoded point to %r" %
block.pretty_name)
item.location_name = block.pretty_name
except reverse.ReverseGeocodeError:
logger.debug(" Failed to reverse geocode %s for %r" %
(item.location.wkt, item.title))
item.location_name = u''
item.save()
if status == 'added':
addcount += 1
else:
updatecount += 1
logger.info("%s: %s" % (status, item.title))
except:
logger.exception("unexpected error:", sys.exc_info()[1])
logger.info("add_events finished: %d added, %d updated" %
(addcount, updatecount))
def update(self):
logger.info("Starting LocalNewsScraper update %s" % self.url)
try:
schema = Schema.objects.get(slug=self.schema_slug)
except Schema.DoesNotExist:
logger.error("Schema (%s): DoesNotExist" % self.schema_slug)
return 1
response, content = self.http.request(self.url)
if response.fromcache:
logger.info("Feed is unchanged since last update (cached)")
return
f = feedparser.parse(content)
addcount = updatecount = 0
for entry in f.entries:
title = convert_entities(entry.title)
description = convert_entities(entry.description)
if entry.id.startswith('http'):
item_url = entry.id
else:
item_url = entry.link
try:
item = NewsItem.objects.get(schema__id=schema.id,
title=title,
description=description)
#url=item_url)
status = 'updated'
except NewsItem.DoesNotExist:
item = NewsItem()
status = 'added'
except NewsItem.MultipleObjectsReturned:
# Seen some where we get the same story with multiple URLs. Why?
logger.warn(
"Multiple entries matched title %r and description %r. Expected unique!"
% (title, description))
continue
try:
item.title = title
item.schema = schema
item.description = description
item.url = item_url
# Support both georss and xcal for getting the location name.
# TODO: should also support ev:location per http://web.resource.org/rss/1.0/modules/event/
item.location_name = entry.get(
'xCal_x-calconnect-street') or entry.get(
'x-calconnect-street') or entry.get(
'georss_featurename') or entry.get('featurename')
item.item_date = datetime.datetime(*entry.updated_parsed[:6])
item.pub_date = datetime.datetime(*entry.updated_parsed[:6])
_short_title = item.title[:30] + '...'
# feedparser bug: depending on which parser it magically uses,
# we either get the xml namespace in the key name, or we don't.
point = entry.get('georss_point') or entry.get('point')
x, y = None, None
if point:
# GeoRSS puts latitude (Y) first.
y, x = point.split(' ')
else:
if item.location_name:
text = item.location_name
else:
# Geocode whatever we can find.
text = item.title + ' ' + item.description
logger.debug("...Falling back on geocoding from %r..." %
text[:50])
addrs = parse_addresses(text)
for addr, unused in addrs:
try:
result = SmartGeocoder().geocode(addr)
point = result['point']
logger.debug("internally geocoded %r" % addr)
x, y = point.x, point.y
if not item.location_name:
item.location_name = result['address']
break
except GeocodingException:
logger.debug("Geocoding exception on %r:" % text)
log_exception(level=logging.DEBUG)
continue
except:
logger.error(
'uncaught geocoder exception on %r\n' % addr)
log_exception()
if None in (x, y):
logger.debug(
"Skip, couldn't geocode any addresses in item '%s...'"
% _short_title)
continue
item.location = Point((float(x), float(y)))
if not intersects_metro_bbox(item.location):
reversed_loc = Point((float(y), float(x)))
if intersects_metro_bbox(reversed_loc):
logger.info(
"Got points in apparently reverse order, flipping them"
)
item.location = reversed_loc
else:
logger.info("Skipping %r as %s,%s is out of bounds" %
(_short_title, y, x))
continue
if not item.location_name:
# Fall back to reverse-geocoding.
from ebpub.geocoder import reverse
try:
block, distance = reverse.reverse_geocode(
item.location)
logger.debug(" Reverse-geocoded point to %r" %
block.pretty_name)
item.location_name = block.pretty_name
except reverse.ReverseGeocodeError:
logger.info(
" Skip, failed to reverse geocode %s for %r" %
(item.location.wkt, _short_title))
continue
item.save()
if status == 'added':
addcount += 1
else:
updatecount += 1
logger.info("%s: %s" % (status, _short_title))
except:
logger.error("Warning: couldn't save %r. Traceback:" %
_short_title)
log_exception()
logger.info("Finished LocalNewsScraper update: %d added, %d updated" %
(addcount, updatecount))
def update(self):
""" Download Calendar RSS feed and update database """
logger.info("Starting EventsCalendarScraper")
feed = feedparser.parse(self.url)
seencount = addcount = updatecount = 0
for entry in feed.entries:
def ns_get(element):
# work around feedparser unpredictability.
namespace, element = element.split(':')
result = entry.get('%s_%s' % (namespace, element))
if result is None:
result = entry.get(element)
return result
seencount += 1
title = convert_entities(entry.title)
try:
item = NewsItem.objects.get(title=title,
schema__id=self.schema.id)
status = "updated"
except NewsItem.DoesNotExist:
item = NewsItem()
status = "added"
except NewsItem.MultipleObjectsReturned:
logger.warn(
"Multiple entries matched title %r, event titles are not unique?"
% title)
continue
try:
item.location_name = '%s %s' % (
ns_get('xcal:x-calconnect-venue-name'),
ns_get('xcal:x-calconnect-street'))
item.location_name = item.location_name.strip()
item.schema = self.schema
item.title = title
item.description = convert_entities(entry.description)
item.url = entry.link
start_dt = ns_get('xcal:dtstart')
start_dt = dateutil.parser.parse(start_dt)
# Upstream bug: They provide a UTC offset of +0000 which
# means times in UTC, but they're actually times in
# US/Eastern, so do *not* fix the zone.
#start_dt = start_dt.astimezone(local_tz)
item.item_date = start_dt.date()
item.pub_date = datetime.datetime(*entry.updated_parsed[:6])
item.location = Point(
(float(ns_get('geo:long')), float(ns_get('geo:lat'))))
if (item.location.x, item.location.y) == (0.0, 0.0):
logger.warn("Skipping %r, bad location 0,0" % item.title)
continue
if not item.location_name:
# Fall back to reverse-geocoding.
from ebpub.geocoder import reverse
try:
block, distance = reverse.reverse_geocode(
item.location)
logger.info(" Reverse-geocoded point to %r" %
block.pretty_name)
item.location_name = block.pretty_name
except reverse.ReverseGeocodeError:
logger.debug(" Failed to reverse geocode %s for %r" %
(item.location.wkt, item.title))
item.location_name = u''
item.save()
item.attributes['start_time'] = start_dt.time()
end_dt = ns_get('xcal:dtend') or u''
if end_dt.strip():
end_dt = dateutil.parser.parse(end_dt.strip())
#end_dt = end_dt.astimezone(local_tz)
item.attributes['end_time'] = end_dt.time()
if status == 'added':
addcount += 1
else:
updatecount += 1
logger.info("%s: %s" % (status, item.title))
except Exception as e:
logger.exception("unexpected error: %s" % e)
logger.info(
"EventsCalendarScraper finished: %d added, %d updated of %s total"
% (addcount, updatecount, seencount))
# custom field crime type
crime_name = SchemaField()
crime_name.schema = crime_report
crime_name.pretty_name = "Crime Type"
crime_name.pretty_plural_name = "Crime Types"
crime_name.real_name = "varchar02"
crime_name.name = "crime_type"
crime_name.save()
# custom field crime code
crime_code = SchemaField()
crime_code.schema = crime_report
crime_code.pretty_name = "Crime Code"
crime_code.pretty_plural_name = "Crime Codes"
crime_code.real_name = "int01"
crime_code.name = "crime_code"
crime_code.save()
# create a Crime Report!
report = NewsItem()
report.schema = crime_report
report.title = "Hooligans causing disturbance downtown"
report.location_name = "123 Fakey St."
report.item_date = datetime.utcnow()
report.pub_date = datetime.utcnow()
report.description = "Blah Blah Blah"
report.save()
report.attributes['officer'] = "John Smith"
report.attributes['crime_type'] = "Disturbing The Peace"
report.attributes['crime_code'] = 187
def main(argv=None):
logger.info("Starting add_news")
if argv:
url = argv[0]
else:
url = 'http://search.boston.com/search/api?q=*&sort=-articleprintpublicationdate&subject=massachusetts&scope=bonzai'
schema_slug = 'local-news'
try:
schema = Schema.objects.get(slug=schema_slug)
except Schema.DoesNotExist:
logger.error("Schema (%s): DoesNotExist" % schema_slug)
sys.exit(1)
f = feedparser.parse(url)
addcount = updatecount = 0
for entry in f.entries:
title = convert_entities(entry.title)
description = convert_entities(entry.description)
if entry.id.startswith('http'):
item_url = entry.id
else:
item_url = entry.link
try:
item = NewsItem.objects.get(schema__id=schema.id,
title=title,
description=description)
#url=item_url)
status = 'updated'
except NewsItem.DoesNotExist:
item = NewsItem()
status = 'added'
except NewsItem.MultipleObjectsReturned:
# Seen some where we get the same story with multiple URLs. Why?
logger.warn(
"Multiple entries matched title %r and description %r. Expected unique!"
% (title, description))
continue
try:
item.title = title
item.schema = schema
item.description = description
item.url = item_url
item.location_name = entry.get('x-calconnect-street') or entry.get(
'georss_featurename')
item.item_date = datetime.datetime(*entry.updated_parsed[:6])
item.pub_date = datetime.datetime(*entry.updated_parsed[:6])
# feedparser bug: depending on which parser it magically uses,
# we either get the xml namespace in the key name, or we don't.
point = entry.get('georss_point') or entry.get('point')
if point:
x, y = point.split(' ')
else:
# Fall back on geocoding.
text = item.title + ' ' + item.description
try:
x, y = quick_dirty_fallback_geocode(text, parse=True)
except GeocodingException:
logger.debug("Geocoding exception on %r:" % text)
log_exception(level=logging.DEBUG)
continue
if None in (x, y):
logger.info("couldn't geocode '%s...'" % item.title[:30])
continue
item.location = Point((float(y), float(x)))
if item.location.x == 0.0 and item.location.y == 0.0:
# There's a lot of these. Maybe attempt to
# parse and geocode if we haven't already?
logger.info("Skipping %r as it has bad location 0,0" %
item.title)
continue
if not item.location_name:
# Fall back to reverse-geocoding.
from ebpub.geocoder import reverse
try:
block, distance = reverse.reverse_geocode(item.location)
logger.debug(" Reverse-geocoded point to %r" %
block.pretty_name)
item.location_name = block.pretty_name
item.block = block
except reverse.ReverseGeocodeError:
logger.debug(" Failed to reverse geocode %s for %r" %
(item.location.wkt, item.title))
item.location_name = u''
item.save()
if status == 'added':
addcount += 1
else:
updatecount += 1
logger.info("%s: %s" % (status, item.title))
except:
logger.error("Warning: couldn't save %r. Traceback:" % item.title)
log_exception()
logger.info("Finished add_news: %d added, %d updated" %
(addcount, updatecount))
