def get_incidents(self): """Return CHP incidents given various request args. """ id = self.request.get("id") center = self.request.get("center") dispatch = self.request.get("dispatch") area = self.request.get("area") city = self.request.get("city") since = self.request.get("since") memcache_key = "incidents-%s-%s-%s-%s-%s-%s" % (id, center, dispatch, area, city, since) memcache_expiry_time = 60 incidents = memcache.get(memcache_key) if incidents is None: if id == "": query = CHPIncident.all() query.order('-LogTime') if center != "": query.filter('CenterID =', center) if dispatch != "": query.filter('DispatchID =', dispatch) if area != "": query.filter('Area =', area) if city != "": query.filter('city =', city) if since != "": query.filter('LogTime >', datetime.fromtimestamp(float(since))) incidents = query.fetch(10000) else: # Handle single incident requests, slightly different approach # We want to use get_by_key_name() instead of filtering. incidents = [] incident = CHPIncident.get_by_key_name(id) if incident is not None: incidents.append(incident) try: memcache.add(memcache_key, incidents, memcache_expiry_time) except ValueError: pass if len(incidents) > 0: self.incidents_last_mod = max(incidents, key=lambda incident: incident.updated).updated else: self.incidents_last_mod = CHPData.last_updated() self.incidents = incidents
def process_chp_center(chpCenter): """Process a CHP Center. """ incident_list = [] psh_pings = [] dash_re = re.compile(r'\s*-\s*') for chpDispatch in chpCenter: # For some reason, sometimes the Dispatch ID is blank # so skip these. if chpDispatch.attrib['ID'] == "": continue for chpLog in chpDispatch: # There are two different time formats in the CHP feed. Try the # "standard" format first, then fall back to the new SAHB format. try: log_time = datetime.strptime(chpLog.find('LogTime').text, '"%m/%d/%Y %I:%M:%S %p"').replace(tzinfo=tzinfo.Pacific()) except ValueError: log_time = datetime.strptime(chpLog.find('LogTime').text, '"%b %d %Y %I:%M%p"').replace(tzinfo=tzinfo.Pacific()) key_name = "%s.%s.%s.%d" % (chpCenter.attrib['ID'], chpDispatch.attrib['ID'], chpLog.attrib['ID'], time.mktime(log_time.timetuple())) incident = CHPIncident.get_by_key_name(key_name) if incident is None: incident = CHPIncident(key_name = key_name, CenterID = chpCenter.attrib['ID'], DispatchID = chpDispatch.attrib['ID'], LogID = chpLog.attrib['ID']) incident.LogTime = log_time (logtypeid, dash, logtype) = chpLog.find('LogType').text.strip('"').partition("-") if dash == '': # If 'dash' is missing then the hyphen was missing, if so we # use the whole thing as the LogType and forget the LogTypeID incident.LogType = deCopIfy(re.sub(dash_re, ' - ', logtypeid.strip())) else: incident.LogType = deCopIfy(re.sub(dash_re, ' - ', logtype.strip())) incident.LogTypeID = logtypeid.strip() incident.Location = deCopIfy(chpLog.find('Location').text.strip('"')) incident.Area = chpLog.find('Area').text.strip('"') incident.ThomasBrothers = chpLog.find('ThomasBrothers').text.strip('"') # Like the LogTime above, there are now two location formats. # This time we try the SAHB LATLON first, then fall back to TBXY. try: latlon = chpLog.find('LATLON').text.strip('"') if latlon != "0:0": incident.geolocation = db.GeoPt( lat = float(latlon.partition(":")[0]) / 1000000, lon = float(latlon.partition(":")[2]) / 1000000 * -1 ) except AttributeError: incident.TBXY = chpLog.find('TBXY').text.strip('"') incident.geolocation = geoConvertTBXY(incident.CenterID, incident.TBXY) # Special handling for the LogDetails LogDetails = { 'details': [], 'units': [] } logdetails_element = chpLog.find('LogDetails') for element in logdetails_element: try: detail_dict = { 'DetailTime': element.find('DetailTime').text.strip('"'), 'IncidentDetail': deCopIfy(element.find('IncidentDetail').text.strip('"^')).capitalize() } LogDetails[element.tag].append(detail_dict) except AttributeError: pass incident.LogDetails = pickle.dumps(LogDetails) # Set up the PSH pings. Note, we are NOT checking for actual # changes in the data, we are just assuming that the existance of # an incident in the CHP feed declares it as "updated" so we ping. psh_pings.append('http://www.sactraffic.org/atom?center=%s' % urllib.quote(incident.CenterID)) psh_pings.append('http://www.sactraffic.org/atom?dispatch=%s' % urllib.quote(incident.DispatchID)) psh_pings.append('http://www.sactraffic.org/atom?area=%s' % urllib.quote(incident.Area)) if incident.city is not None and incident.city != "": psh_pings.append('http://www.sactraffic.org/atom?city=%s' % urllib.quote(incident.city)) # Save this incident incident_list.append(incident) # Store the incidents in a batch db.put(incident_list) # Ping the PSH hub, use a set so we don't ping duplicates. ping_set = set(psh_pings) if not debug: deferred.defer(pubsubhubbub_publish.publish, 'http://pubsubhubbub.appspot.com', ping_set, _queue="pshPingQueue") else: logging.info("Skipping PSH pings for %s on the development server. %s" % (incident.CenterID, ping_set)) # Reverse geocode the incidents if we haven't already for incident in incident_list: if incident.city is None and incident.geolocation is not None: deferred.defer(reverse_geocode.load_city, incident, _queue="reverseGeocodeQueue") logging.info("Processed %d incidents in %s." % (len(incident_list), chpCenter.attrib['ID']))