def create_wifi_measure(measure_data, entry):
return WifiMeasure(
measure_id=measure_data['id'],
created=decode_datetime(measure_data.get('created', '')),
lat=measure_data['lat'],
lon=measure_data['lon'],
time=decode_datetime(measure_data.get('time', '')),
accuracy=measure_data.get('accuracy', 0),
altitude=measure_data.get('altitude', 0),
altitude_accuracy=measure_data.get('altitude_accuracy', 0),
id=entry.get('id', None),
key=entry['key'],
channel=entry.get('channel', 0),
signal=entry.get('signal', 0),
)
def create_wifi_measure(measure_data, entry):
return WifiMeasure(
measure_id=measure_data['id'],
created=decode_datetime(measure_data.get('created', '')),
lat=measure_data['lat'],
lon=measure_data['lon'],
time=decode_datetime(measure_data.get('time', '')),
accuracy=measure_data.get('accuracy', 0),
altitude=measure_data.get('altitude', 0),
altitude_accuracy=measure_data.get('altitude_accuracy', 0),
id=entry.get('id', None),
key=entry['key'],
channel=entry.get('channel', 0),
signal=entry.get('signal', 0),
)
def process_cell_measure(session, measure_data, entries, userid=None):
cell_count = defaultdict(int)
cell_measures = []
created = decode_datetime(measure_data.get('created', ''))
# process entries
for entry in entries:
cell_measure = create_cell_measure(measure_data, entry)
# use more specific cell type or
# fall back to less precise measure
if entry.get('radio'):
cell_measure.radio = RADIO_TYPE.get(entry['radio'], -1)
else:
cell_measure.radio = measure_data['radio']
cell_measures.append(cell_measure)
# group per unique cell
cell_count[CellKey(cell_measure.radio, cell_measure.mcc,
cell_measure.mnc, cell_measure.lac,
cell_measure.cid)] += 1
# update new/total measure counts
new_cells = 0
for cell_key, count in cell_count.items():
new_cells += update_cell_measure_count(
cell_key, count, created, session)
# update user score
if userid is not None and new_cells > 0:
process_score(userid, new_cells, session, key='new_cell')
session.add_all(cell_measures)
return cell_measures
def create_cell_measure(utcnow, entry):
# skip records with missing or invalid mcc or mnc
if 'mcc' not in entry or entry['mcc'] < 1 or entry['mcc'] > 999:
return
if 'mnc' not in entry or entry['mnc'] < 0 or entry['mnc'] > 32767:
return
# Skip CDMA towers missing lac or cid (no psc on CDMA exists to
# backfill using inference)
if entry.get('radio', -1) == 1 and \
(entry.get('lac', -1) < 0 or entry.get('cid', -1) < 0):
return
# some phones send maxint32 to signal "unknown"
# ignore anything above the maximum valid values
if 'lac' not in entry or entry['lac'] < 0 or entry['lac'] > 65535:
entry['lac'] = -1
if 'cid' not in entry or entry['cid'] < 0 or entry['cid'] > 268435455:
entry['cid'] = -1
if 'psc' not in entry or entry['psc'] < 0 or entry['psc'] > 512:
entry['psc'] = -1
# Treat the lac=0, cid=65535 combination as unspecified values
if entry['lac'] == 0 and entry['cid'] == 65535:
entry['lac'] = -1
entry['cid'] = -1
# Must have LAC+CID or PSC
if (entry['lac'] == -1 or entry['cid'] == -1) and entry['psc'] == -1:
return
# make sure fields stay within reasonable bounds
if 'asu' not in entry or entry['asu'] < 0 or entry['asu'] > 100:
entry['asu'] = -1
if 'signal' not in entry or entry['signal'] < -200 or entry['signal'] > -1:
entry['signal'] = 0
if 'ta' not in entry or entry['ta'] < 0 or entry['ta'] > 100:
entry['ta'] = 0
return CellMeasure(
measure_id=entry.get('measure_id'),
created=utcnow,
lat=entry['lat'],
lon=entry['lon'],
time=decode_datetime(entry.get('time', '')),
accuracy=entry.get('accuracy', 0),
altitude=entry.get('altitude', 0),
altitude_accuracy=entry.get('altitude_accuracy', 0),
radio=entry.get('radio', -1),
mcc=entry.get('mcc', -1),
mnc=entry.get('mnc', -1),
lac=entry.get('lac', -1),
cid=entry.get('cid', -1),
psc=entry.get('psc', -1),
asu=entry.get('asu', -1),
signal=entry.get('signal', 0),
ta=entry.get('ta', 0),
)
def create_cell_measure(measure_data, entry):
return CellMeasure(
measure_id=measure_data['id'],
created=decode_datetime(measure_data.get('created', '')),
lat=measure_data['lat'],
lon=measure_data['lon'],
time=decode_datetime(measure_data.get('time', '')),
accuracy=measure_data.get('accuracy', 0),
altitude=measure_data.get('altitude', 0),
altitude_accuracy=measure_data.get('altitude_accuracy', 0),
mcc=entry['mcc'],
mnc=entry['mnc'],
lac=entry.get('lac', 0),
cid=entry.get('cid', 0),
psc=entry.get('psc', -1),
asu=entry.get('asu', 0),
signal=entry.get('signal', 0),
ta=entry.get('ta', 0),
)
def create_cell_measure(measure_data, entry):
return CellMeasure(
measure_id=measure_data['id'],
created=decode_datetime(measure_data.get('created', '')),
lat=measure_data['lat'],
lon=measure_data['lon'],
time=decode_datetime(measure_data.get('time', '')),
accuracy=measure_data.get('accuracy', 0),
altitude=measure_data.get('altitude', 0),
altitude_accuracy=measure_data.get('altitude_accuracy', 0),
mcc=entry['mcc'],
mnc=entry['mnc'],
lac=entry.get('lac', 0),
cid=entry.get('cid', 0),
psc=entry.get('psc', 0),
asu=entry.get('asu', 0),
signal=entry.get('signal', 0),
ta=entry.get('ta', 0),
)
def create_wifi_measure(utcnow, entry):
return WifiMeasure(
measure_id=entry.get('measure_id'),
created=utcnow,
lat=entry['lat'],
lon=entry['lon'],
time=decode_datetime(entry.get('time', '')),
accuracy=entry.get('accuracy', 0),
altitude=entry.get('altitude', 0),
altitude_accuracy=entry.get('altitude_accuracy', 0),
key=entry['key'],
channel=entry.get('channel', 0),
signal=entry.get('signal', 0),
)
def create_wifi_measure(utcnow, entry):
return WifiMeasure(
measure_id=entry.get('measure_id'),
created=utcnow,
lat=entry['lat'],
lon=entry['lon'],
time=decode_datetime(entry.get('time', '')),
accuracy=entry.get('accuracy', 0),
altitude=entry.get('altitude', 0),
altitude_accuracy=entry.get('altitude_accuracy', 0),
key=entry['key'],
channel=entry.get('channel', 0),
signal=entry.get('signal', 0),
)
def process_wifi_measure(session, measure_data, entries, userid=None):
wifi_measures = []
wifi_count = defaultdict(int)
wifi_keys = set([e['key'] for e in entries])
created = decode_datetime(measure_data.get('created', ''))
# did we get measures for blacklisted wifis?
blacked = session.query(WifiBlacklist.key).filter(
WifiBlacklist.key.in_(wifi_keys)).all()
blacked = set([b[0] for b in blacked])
# process entries
for entry in entries:
wifi_key = entry['key']
# convert frequency into channel numbers and remove frequency
convert_frequency(entry)
wifi_measures.append(create_wifi_measure(measure_data, created, entry))
if wifi_key not in blacked:
# skip blacklisted wifi AP's
wifi_count[wifi_key] += 1
# update user score
if userid is not None:
# do we already know about any wifis?
white_keys = wifi_keys - blacked
if white_keys:
wifis = session.query(Wifi.key).filter(Wifi.key.in_(white_keys))
wifis = dict([(w[0], True) for w in wifis.all()])
else:
wifis = {}
# subtract known wifis from all unique wifis
new_wifis = len(wifi_count) - len(wifis)
if new_wifis > 0:
process_score(userid, new_wifis, session, key='new_wifi')
# update new/total measure counts
for wifi_key, num in wifi_count.items():
stmt = Wifi.__table__.insert(
on_duplicate='new_measures = new_measures + %s, '
'total_measures = total_measures + %s' % (num, num)
).values(
key=wifi_key, created=created,
new_measures=num, total_measures=num)
session.execute(stmt)
session.add_all(wifi_measures)
return wifi_measures
def process_wifi_measure(session, measure_data, entries, userid=None):
wifi_measures = []
wifi_keys = set([e['key'] for e in entries])
# did we get measures for blacklisted wifis?
blacked = session.query(WifiBlacklist.key).filter(
WifiBlacklist.key.in_(wifi_keys)).all()
blacked = set([b[0] for b in blacked])
# do we already know about these wifis?
wifis = session.query(Wifi.key).filter(Wifi.key.in_(wifi_keys))
wifis = dict([(w[0], True) for w in wifis.all()])
created = decode_datetime(measure_data.get('created', ''))
# TODO group by unique cell
for entry in entries:
wifi_key = entry['key']
# convert frequency into channel numbers and remove frequency
convert_frequency(entry)
wifi_measures.append(create_wifi_measure(measure_data, created, entry))
# update new/total measure counts
if wifi_key not in blacked:
# skip blacklisted wifi AP's
update_wifi_measure_count(
wifi_key, wifis, created, session, userid=userid)
session.add_all(wifi_measures)
return wifi_measures
