def create_cell_measure(utcnow, entry):
entry = normalized_cell_measure_dict(entry)
if entry is None:
return None
report_id = entry.get('report_id')
if report_id:
report_id = uuid.UUID(hex=report_id).bytes
return CellMeasure(
report_id=report_id,
created=utcnow,
lat=entry['lat'],
lon=entry['lon'],
time=decode_datetime(entry.get('time', '')),
accuracy=entry.get('accuracy', 0),
altitude=entry.get('altitude', 0),
altitude_accuracy=entry.get('altitude_accuracy', 0),
radio=entry.get('radio', -1),
mcc=entry.get('mcc', -1),
mnc=entry.get('mnc', -1),
lac=entry.get('lac', -1),
cid=entry.get('cid', -1),
psc=entry.get('psc', -1),
asu=entry.get('asu', -1),
signal=entry.get('signal', 0),
ta=entry.get('ta', 0),
heading=entry.get('heading', -1.0),
speed=entry.get('speed', -1.0),
)
def create_wifi_measure(utcnow, entry):
entry = normalized_wifi_measure_dict(entry)
if entry is None:
return None
return WifiMeasure(
measure_id=entry.get('measure_id'),
created=utcnow,
lat=entry['lat'],
lon=entry['lon'],
time=decode_datetime(entry.get('time', '')),
accuracy=entry.get('accuracy', 0),
altitude=entry.get('altitude', 0),
altitude_accuracy=entry.get('altitude_accuracy', 0),
key=entry['key'],
channel=entry.get('channel', 0),
signal=entry.get('signal', 0),
)
def create_cell_measure(utcnow, entry):
entry = normalized_cell_measure_dict(entry)
if entry is None:
return None
return CellMeasure(
measure_id=entry.get('measure_id'),
created=utcnow,
lat=entry['lat'],
lon=entry['lon'],
time=decode_datetime(entry.get('time', '')),
accuracy=entry.get('accuracy', 0),
altitude=entry.get('altitude', 0),
altitude_accuracy=entry.get('altitude_accuracy', 0),
radio=entry.get('radio', -1),
mcc=entry.get('mcc', -1),
mnc=entry.get('mnc', -1),
lac=entry.get('lac', -1),
cid=entry.get('cid', -1),
psc=entry.get('psc', -1),
asu=entry.get('asu', -1),
signal=entry.get('signal', 0),
ta=entry.get('ta', 0),
)
def create_wifi_measure(utcnow, entry):
entry = normalized_wifi_measure_dict(entry)
if entry is None:
return None
report_id = entry.get('report_id')
if report_id:
report_id = uuid.UUID(hex=report_id).bytes
return WifiMeasure(
report_id=report_id,
created=utcnow,
lat=entry['lat'],
lon=entry['lon'],
time=decode_datetime(entry.get('time', '')),
accuracy=entry.get('accuracy', 0),
altitude=entry.get('altitude', 0),
altitude_accuracy=entry.get('altitude_accuracy', 0),
key=entry['key'],
channel=entry.get('channel', 0),
signal=entry.get('signal', 0),
snr=entry.get('signalToNoiseRatio', 0),
heading=entry.get('heading', -1.0),
speed=entry.get('speed', -1.0),
)
def process_station_measures(session, entries, station_type,
station_model, measure_model, blacklist_model,
create_measure, create_key, join_key,
userid=None, max_measures_per_station=11000,
utcnow=None):
all_measures = []
dropped_blacklisted = 0
dropped_malformed = 0
dropped_overflow = 0
stats_client = get_stats_client()
new_stations = 0
if utcnow is None:
utcnow = util.utcnow()
elif isinstance(utcnow, basestring):
utcnow = decode_datetime(utcnow)
# Process entries and group by validated station key
station_measures = defaultdict(list)
for entry in entries:
measure = create_measure(utcnow, entry)
if not measure:
dropped_malformed += 1
continue
station_measures[create_key(measure)].append(measure)
# Process measures one station at a time
for key, measures in station_measures.items():
incomplete = False
is_new_station = False
# Figure out how much space is left for this station.
free = available_station_space(session, key, station_model,
join_key, max_measures_per_station)
if free is None:
is_new_station = True
free = max_measures_per_station
if is_new_station:
# Drop measures for blacklisted stations.
if blacklisted_station(session, key, blacklist_model,
join_key, utcnow):
dropped_blacklisted += len(measures)
continue
incomplete = incomplete_measure(key)
if not incomplete:
# We discovered an actual new complete station.
new_stations += 1
# Accept measures up to input-throttling limit, then drop.
num = 0
for measure in measures:
if free <= 0:
dropped_overflow += 1
continue
all_measures.append(measure)
free -= 1
num += 1
# Accept incomplete measures, just don't make stations for them.
# (station creation is a side effect of count-updating)
if not incomplete and num > 0:
create_or_update_station(session, key, station_model,
join_key, utcnow, num)
# Credit the user with discovering any new stations.
if userid is not None and new_stations > 0:
process_score(userid, new_stations, session,
key='new_' + station_type)
if dropped_blacklisted != 0:
stats_client.incr(
"items.dropped.%s_ingress_blacklisted" % station_type,
count=dropped_blacklisted)
if dropped_malformed != 0:
stats_client.incr(
"items.dropped.%s_ingress_malformed" % station_type,
count=dropped_malformed)
if dropped_overflow != 0:
stats_client.incr(
"items.dropped.%s_ingress_overflow" % station_type,
count=dropped_overflow)
stats_client.incr(
"items.inserted.%s_measures" % station_type,
count=len(all_measures))
session.add_all(all_measures)
return all_measures
