def create_wifi_measure(measure_data, entry): return WifiMeasure( measure_id=measure_data['id'], created=decode_datetime(measure_data.get('created', '')), lat=measure_data['lat'], lon=measure_data['lon'], time=decode_datetime(measure_data.get('time', '')), accuracy=measure_data.get('accuracy', 0), altitude=measure_data.get('altitude', 0), altitude_accuracy=measure_data.get('altitude_accuracy', 0), id=entry.get('id', None), key=entry['key'], channel=entry.get('channel', 0), signal=entry.get('signal', 0), )
def process_cell_measure(session, measure_data, entries, userid=None): cell_count = defaultdict(int) cell_measures = [] created = decode_datetime(measure_data.get('created', '')) # process entries for entry in entries: cell_measure = create_cell_measure(measure_data, entry) # use more specific cell type or # fall back to less precise measure if entry.get('radio'): cell_measure.radio = RADIO_TYPE.get(entry['radio'], -1) else: cell_measure.radio = measure_data['radio'] cell_measures.append(cell_measure) # group per unique cell cell_count[CellKey(cell_measure.radio, cell_measure.mcc, cell_measure.mnc, cell_measure.lac, cell_measure.cid)] += 1 # update new/total measure counts new_cells = 0 for cell_key, count in cell_count.items(): new_cells += update_cell_measure_count( cell_key, count, created, session) # update user score if userid is not None and new_cells > 0: process_score(userid, new_cells, session, key='new_cell') session.add_all(cell_measures) return cell_measures
def create_cell_measure(utcnow, entry): # skip records with missing or invalid mcc or mnc if 'mcc' not in entry or entry['mcc'] < 1 or entry['mcc'] > 999: return if 'mnc' not in entry or entry['mnc'] < 0 or entry['mnc'] > 32767: return # Skip CDMA towers missing lac or cid (no psc on CDMA exists to # backfill using inference) if entry.get('radio', -1) == 1 and \ (entry.get('lac', -1) < 0 or entry.get('cid', -1) < 0): return # some phones send maxint32 to signal "unknown" # ignore anything above the maximum valid values if 'lac' not in entry or entry['lac'] < 0 or entry['lac'] > 65535: entry['lac'] = -1 if 'cid' not in entry or entry['cid'] < 0 or entry['cid'] > 268435455: entry['cid'] = -1 if 'psc' not in entry or entry['psc'] < 0 or entry['psc'] > 512: entry['psc'] = -1 # Treat the lac=0, cid=65535 combination as unspecified values if entry['lac'] == 0 and entry['cid'] == 65535: entry['lac'] = -1 entry['cid'] = -1 # Must have LAC+CID or PSC if (entry['lac'] == -1 or entry['cid'] == -1) and entry['psc'] == -1: return # make sure fields stay within reasonable bounds if 'asu' not in entry or entry['asu'] < 0 or entry['asu'] > 100: entry['asu'] = -1 if 'signal' not in entry or entry['signal'] < -200 or entry['signal'] > -1: entry['signal'] = 0 if 'ta' not in entry or entry['ta'] < 0 or entry['ta'] > 100: entry['ta'] = 0 return CellMeasure( measure_id=entry.get('measure_id'), created=utcnow, lat=entry['lat'], lon=entry['lon'], time=decode_datetime(entry.get('time', '')), accuracy=entry.get('accuracy', 0), altitude=entry.get('altitude', 0), altitude_accuracy=entry.get('altitude_accuracy', 0), radio=entry.get('radio', -1), mcc=entry.get('mcc', -1), mnc=entry.get('mnc', -1), lac=entry.get('lac', -1), cid=entry.get('cid', -1), psc=entry.get('psc', -1), asu=entry.get('asu', -1), signal=entry.get('signal', 0), ta=entry.get('ta', 0), )
def create_cell_measure(measure_data, entry): return CellMeasure( measure_id=measure_data['id'], created=decode_datetime(measure_data.get('created', '')), lat=measure_data['lat'], lon=measure_data['lon'], time=decode_datetime(measure_data.get('time', '')), accuracy=measure_data.get('accuracy', 0), altitude=measure_data.get('altitude', 0), altitude_accuracy=measure_data.get('altitude_accuracy', 0), mcc=entry['mcc'], mnc=entry['mnc'], lac=entry.get('lac', 0), cid=entry.get('cid', 0), psc=entry.get('psc', -1), asu=entry.get('asu', 0), signal=entry.get('signal', 0), ta=entry.get('ta', 0), )
def create_cell_measure(measure_data, entry): return CellMeasure( measure_id=measure_data['id'], created=decode_datetime(measure_data.get('created', '')), lat=measure_data['lat'], lon=measure_data['lon'], time=decode_datetime(measure_data.get('time', '')), accuracy=measure_data.get('accuracy', 0), altitude=measure_data.get('altitude', 0), altitude_accuracy=measure_data.get('altitude_accuracy', 0), mcc=entry['mcc'], mnc=entry['mnc'], lac=entry.get('lac', 0), cid=entry.get('cid', 0), psc=entry.get('psc', 0), asu=entry.get('asu', 0), signal=entry.get('signal', 0), ta=entry.get('ta', 0), )
def create_wifi_measure(utcnow, entry): return WifiMeasure( measure_id=entry.get('measure_id'), created=utcnow, lat=entry['lat'], lon=entry['lon'], time=decode_datetime(entry.get('time', '')), accuracy=entry.get('accuracy', 0), altitude=entry.get('altitude', 0), altitude_accuracy=entry.get('altitude_accuracy', 0), key=entry['key'], channel=entry.get('channel', 0), signal=entry.get('signal', 0), )
def process_wifi_measure(session, measure_data, entries, userid=None): wifi_measures = [] wifi_count = defaultdict(int) wifi_keys = set([e['key'] for e in entries]) created = decode_datetime(measure_data.get('created', '')) # did we get measures for blacklisted wifis? blacked = session.query(WifiBlacklist.key).filter( WifiBlacklist.key.in_(wifi_keys)).all() blacked = set([b[0] for b in blacked]) # process entries for entry in entries: wifi_key = entry['key'] # convert frequency into channel numbers and remove frequency convert_frequency(entry) wifi_measures.append(create_wifi_measure(measure_data, created, entry)) if wifi_key not in blacked: # skip blacklisted wifi AP's wifi_count[wifi_key] += 1 # update user score if userid is not None: # do we already know about any wifis? white_keys = wifi_keys - blacked if white_keys: wifis = session.query(Wifi.key).filter(Wifi.key.in_(white_keys)) wifis = dict([(w[0], True) for w in wifis.all()]) else: wifis = {} # subtract known wifis from all unique wifis new_wifis = len(wifi_count) - len(wifis) if new_wifis > 0: process_score(userid, new_wifis, session, key='new_wifi') # update new/total measure counts for wifi_key, num in wifi_count.items(): stmt = Wifi.__table__.insert( on_duplicate='new_measures = new_measures + %s, ' 'total_measures = total_measures + %s' % (num, num) ).values( key=wifi_key, created=created, new_measures=num, total_measures=num) session.execute(stmt) session.add_all(wifi_measures) return wifi_measures
def process_wifi_measure(session, measure_data, entries, userid=None): wifi_measures = [] wifi_keys = set([e['key'] for e in entries]) # did we get measures for blacklisted wifis? blacked = session.query(WifiBlacklist.key).filter( WifiBlacklist.key.in_(wifi_keys)).all() blacked = set([b[0] for b in blacked]) # do we already know about these wifis? wifis = session.query(Wifi.key).filter(Wifi.key.in_(wifi_keys)) wifis = dict([(w[0], True) for w in wifis.all()]) created = decode_datetime(measure_data.get('created', '')) # TODO group by unique cell for entry in entries: wifi_key = entry['key'] # convert frequency into channel numbers and remove frequency convert_frequency(entry) wifi_measures.append(create_wifi_measure(measure_data, created, entry)) # update new/total measure counts if wifi_key not in blacked: # skip blacklisted wifi AP's update_wifi_measure_count( wifi_key, wifis, created, session, userid=userid) session.add_all(wifi_measures) return wifi_measures