def create_cell_measure(utcnow, entry): entry = normalized_cell_measure_dict(entry) if entry is None: return None report_id = entry.get('report_id') if report_id: report_id = uuid.UUID(hex=report_id).bytes return CellMeasure( report_id=report_id, created=utcnow, lat=entry['lat'], lon=entry['lon'], time=decode_datetime(entry.get('time', '')), accuracy=entry.get('accuracy', 0), altitude=entry.get('altitude', 0), altitude_accuracy=entry.get('altitude_accuracy', 0), radio=entry.get('radio', -1), mcc=entry.get('mcc', -1), mnc=entry.get('mnc', -1), lac=entry.get('lac', -1), cid=entry.get('cid', -1), psc=entry.get('psc', -1), asu=entry.get('asu', -1), signal=entry.get('signal', 0), ta=entry.get('ta', 0), heading=entry.get('heading', -1.0), speed=entry.get('speed', -1.0), )
def create_wifi_measure(utcnow, entry): entry = normalized_wifi_measure_dict(entry) if entry is None: return None return WifiMeasure( measure_id=entry.get('measure_id'), created=utcnow, lat=entry['lat'], lon=entry['lon'], time=decode_datetime(entry.get('time', '')), accuracy=entry.get('accuracy', 0), altitude=entry.get('altitude', 0), altitude_accuracy=entry.get('altitude_accuracy', 0), key=entry['key'], channel=entry.get('channel', 0), signal=entry.get('signal', 0), )
def create_cell_measure(utcnow, entry): entry = normalized_cell_measure_dict(entry) if entry is None: return None return CellMeasure( measure_id=entry.get('measure_id'), created=utcnow, lat=entry['lat'], lon=entry['lon'], time=decode_datetime(entry.get('time', '')), accuracy=entry.get('accuracy', 0), altitude=entry.get('altitude', 0), altitude_accuracy=entry.get('altitude_accuracy', 0), radio=entry.get('radio', -1), mcc=entry.get('mcc', -1), mnc=entry.get('mnc', -1), lac=entry.get('lac', -1), cid=entry.get('cid', -1), psc=entry.get('psc', -1), asu=entry.get('asu', -1), signal=entry.get('signal', 0), ta=entry.get('ta', 0), )
def create_wifi_measure(utcnow, entry): entry = normalized_wifi_measure_dict(entry) if entry is None: return None report_id = entry.get('report_id') if report_id: report_id = uuid.UUID(hex=report_id).bytes return WifiMeasure( report_id=report_id, created=utcnow, lat=entry['lat'], lon=entry['lon'], time=decode_datetime(entry.get('time', '')), accuracy=entry.get('accuracy', 0), altitude=entry.get('altitude', 0), altitude_accuracy=entry.get('altitude_accuracy', 0), key=entry['key'], channel=entry.get('channel', 0), signal=entry.get('signal', 0), snr=entry.get('signalToNoiseRatio', 0), heading=entry.get('heading', -1.0), speed=entry.get('speed', -1.0), )
def process_station_measures(session, entries, station_type, station_model, measure_model, blacklist_model, create_measure, create_key, join_key, userid=None, max_measures_per_station=11000, utcnow=None): all_measures = [] dropped_blacklisted = 0 dropped_malformed = 0 dropped_overflow = 0 stats_client = get_stats_client() new_stations = 0 if utcnow is None: utcnow = util.utcnow() elif isinstance(utcnow, basestring): utcnow = decode_datetime(utcnow) # Process entries and group by validated station key station_measures = defaultdict(list) for entry in entries: measure = create_measure(utcnow, entry) if not measure: dropped_malformed += 1 continue station_measures[create_key(measure)].append(measure) # Process measures one station at a time for key, measures in station_measures.items(): incomplete = False is_new_station = False # Figure out how much space is left for this station. free = available_station_space(session, key, station_model, join_key, max_measures_per_station) if free is None: is_new_station = True free = max_measures_per_station if is_new_station: # Drop measures for blacklisted stations. if blacklisted_station(session, key, blacklist_model, join_key, utcnow): dropped_blacklisted += len(measures) continue incomplete = incomplete_measure(key) if not incomplete: # We discovered an actual new complete station. new_stations += 1 # Accept measures up to input-throttling limit, then drop. num = 0 for measure in measures: if free <= 0: dropped_overflow += 1 continue all_measures.append(measure) free -= 1 num += 1 # Accept incomplete measures, just don't make stations for them. # (station creation is a side effect of count-updating) if not incomplete and num > 0: create_or_update_station(session, key, station_model, join_key, utcnow, num) # Credit the user with discovering any new stations. if userid is not None and new_stations > 0: process_score(userid, new_stations, session, key='new_' + station_type) if dropped_blacklisted != 0: stats_client.incr( "items.dropped.%s_ingress_blacklisted" % station_type, count=dropped_blacklisted) if dropped_malformed != 0: stats_client.incr( "items.dropped.%s_ingress_malformed" % station_type, count=dropped_malformed) if dropped_overflow != 0: stats_client.incr( "items.dropped.%s_ingress_overflow" % station_type, count=dropped_overflow) stats_client.incr( "items.inserted.%s_measures" % station_type, count=len(all_measures)) session.add_all(all_measures) return all_measures