def get(self):
count = 0
chp_data_last_updated = CHPData.last_updated()
if chp_data_last_updated is not None:
query = CHPIncident.all(keys_only=True)
query.filter('updated <', chp_data_last_updated - timedelta(hours=1))
count = query.count()
db.delete(query)
self.response.write("Purged %d records." % count)
def get(self):
# We need to be careful here. It's possible this handler could run in the
# window between when the CHPData is updated and when the incidents get
# updated. So throw in 5 minutes of padding.
#
# See also: https://developers.google.com/appengine/docs/python/ndb/#writes
query = CHPIncident.query(CHPIncident.updated < CHPData.last_updated() - timedelta(minutes=5))
count = query.count()
keys = query.fetch(keys_only=True)
ndb.delete_multi(keys)
self.response.write("Purged %d records." % count)
def get_incidents(self):
"""Return CHP incidents given various request args.
"""
id = self.request.get("id")
center = self.request.get("center")
dispatch = self.request.get("dispatch")
area = self.request.get("area")
city = self.request.get("city")
since = self.request.get("since")
memcache_key = "incidents-%s-%s-%s-%s-%s-%s" % (id, center, dispatch, area, city, since)
memcache_expiry_time = 60
incidents = memcache.get(memcache_key)
if incidents is None:
if id == "":
query = CHPIncident.all()
query.order('-LogTime')
if center != "":
query.filter('CenterID =', center)
if dispatch != "":
query.filter('DispatchID =', dispatch)
if area != "":
query.filter('Area =', area)
if city != "":
query.filter('city =', city)
if since != "":
query.filter('LogTime >', datetime.fromtimestamp(float(since)))
incidents = query.fetch(10000)
else:
# Handle single incident requests, slightly different approach
# We want to use get_by_key_name() instead of filtering.
incidents = []
incident = CHPIncident.get_by_key_name(id)
if incident is not None:
incidents.append(incident)
try:
memcache.add(memcache_key, incidents, memcache_expiry_time)
except ValueError:
pass
if len(incidents) > 0:
self.incidents_last_mod = max(incidents, key=lambda incident: incident.updated).updated
else:
self.incidents_last_mod = CHPData.last_updated()
self.incidents = incidents
def update_chp_data():
"""Fetch the CHP data from the CHP and parse it. If it parses OK, defer
processing of each CHP Center.
"""
notice = "CHP Incidents loaded."
try:
result = urlfetch.fetch("http://media.chp.ca.gov/sa_xml/sa.xml", deadline=60)
except urlfetch.DownloadError as notice:
logging.warning(notice)
else:
if result.status_code == 200:
try:
chp_etree = ElementTree.XML(result.content)
except ElementTree.ParseError as e:
notice = "XML processing error. %s" % e.message
logging.warning(notice)
else:
# Store the raw CHP data. Mostly so we have a marker of the last
# successful update.
CHPData.save_chp_data(chp_etree);
# Now, finally, we process the CHP tree, breaking out each CHP
# center and deferring its processing
for chp_center in chp_etree:
deferred.defer(process_chp_center, chp_center, _queue="chpProcessQueue")
# Ping for the whole ATOM feed. We do it here because we only do it once.
if not debug:
deferred.defer(pubsubhubbub_publish.publish, 'http://pubsubhubbub.appspot.com', 'http://www.sactraffic.org/atom', _queue="pshPingQueue")
else:
notice = "CHP server returned " + str(result.status_code) + " status."
logging.warning(notice)
return notice
def get_incidents(self):
"""Return CHP incidents given various request args.
"""
center = self.request.get("center")
dispatch = self.request.get("dispatch")
area = self.request.get("area")
city = self.request.get("city")
# Change this for the West Sac News-Leger since no one appears home there
if "http://www.westsac.com/news-ledger" in self.request.headers['User-Agent']:
if dispatch == "":
dispatch = 'SACC'
memcache_key = "incidents-%s-%s-%s-%s" % (center, dispatch, area, city)
# Don't even try the cache if this looks like a PubSubHubBub request.
if "pubsubhubbub" in self.request.headers['User-Agent']:
incidents = None
else:
incidents = memcache.get(memcache_key)
if incidents is None:
query = CHPIncident.query().order(-CHPIncident.LogTime)
if city != "":
query = query.filter(CHPIncident.city == city)
elif area != "":
query = query.filter(CHPIncident.Area == area)
elif dispatch != "":
query = query.filter(CHPIncident.DispatchID == dispatch)
elif center != "":
query = query.filter(CHPIncident.CenterID == center)
incidents = query.fetch(10000)
try:
memcache.set(memcache_key, incidents, 300)
except:
pass
if len(incidents) > 0:
self.incidents_last_mod = max(incidents, key=lambda incident: incident.updated).updated
else:
self.incidents_last_mod = CHPData.last_updated()
self.incidents = incidents