def test_geoip_match(self):
app = self.app
session = self.db_master_session
cell = Cell()
cell.lat = PARIS_LAT
cell.lon = PARIS_LON
cell.radio = 0
cell.mcc = FRANCE_MCC
cell.mnc = 1
cell.lac = 2
cell.cid = 1234
cell.total_measures = 1
cell.new_measures = 0
session.add(cell)
session.commit()
res = app.post_json('/v1/geosubmit?key=test',
{'items': [{"latitude": PARIS_LAT + 0.1,
"longitude": PARIS_LON + 0.1,
"accuracy": 12.4,
"radioType": "gsm",
"cellTowers": [{
"cellId": 1234,
"locationAreaCode": 2,
"mobileCountryCode": FRANCE_MCC,
"mobileNetworkCode": 1,
}]},
{"latitude": PARIS_LAT - 0.1,
"longitude": PARIS_LON - 0.1,
"accuracy": 22.4,
"radioType": "gsm",
"cellTowers": [{
"cellId": 2234,
"locationAreaCode": 22,
"mobileCountryCode": FRANCE_MCC,
"mobileNetworkCode": 2,
}]}]},
extra_environ={'HTTP_X_FORWARDED_FOR': PARIS_IP},
status=200)
# check that we get an empty response
self.assertEqual(res.content_type, 'application/json')
self.assertEqual(res.json, {})
# check that two new CellMeasure records are created
self.assertEquals(2, session.query(CellMeasure).count())
cm1 = session.query(CellMeasure).filter(
CellMeasure.cid == 1234).count()
cm2 = session.query(CellMeasure).filter(
CellMeasure.cid == 2234).count()
self.assertEquals(1, cm1)
self.assertEquals(1, cm2)
def test_ok_cell(self):
app = self.app
session = self.db_slave_session
cell = Cell()
cell.lat = 123456781
cell.lon = 234567892
cell.radio = 0
cell.mcc = 123
cell.mnc = 1
cell.lac = 2
cell.cid = 1234
session.add(cell)
session.commit()
res = app.post_json(
'/v1/geolocate?key=test', {
"radioType": "gsm",
"cellTowers": [
{"mobileCountryCode": 123, "mobileNetworkCode": 1,
"locationAreaCode": 2, "cellId": 1234},
]},
status=200)
self.assertEqual(res.content_type, 'application/json')
self.assertEqual(res.body, '{"location": {"lat": 12.3456781, '
'"lng": 23.4567892}, "accuracy": 35000.0}')
def test_ok_cell(self):
app = self.app
session = self.get_session()
cell = Cell()
cell.lat = PARIS_LAT
cell.lon = PARIS_LON
cell.radio = RADIO_TYPE['gsm']
cell.mcc = FRANCE_MCC
cell.mnc = 1
cell.lac = 2
cell.cid = 1234
session.add(cell)
session.commit()
res = app.post_json(
'%s?key=test' % self.url, {
"radioType": "gsm",
"cellTowers": [
{"mobileCountryCode": FRANCE_MCC,
"mobileNetworkCode": 1,
"locationAreaCode": 2,
"cellId": 1234},
]},
status=200)
self.check_stats(
counter=[self.metric_url + '.200', self.metric + '.api_key.test']
)
self.assertEqual(res.content_type, 'application/json')
self.assertEqual(res.json, {"location": {"lat": PARIS_LAT,
"lng": PARIS_LON},
"accuracy": CELL_MIN_ACCURACY})
def test_ok_cell_radio_in_celltowers(self):
app = self.app
session = self.db_slave_session
cell = Cell()
cell.lat = 123456781
cell.lon = 234567892
cell.radio = 0
cell.mcc = 123
cell.mnc = 1
cell.lac = 2
cell.cid = 1234
session.add(cell)
session.commit()
res = app.post_json(
'/v1/geolocate?key=test', {
"cellTowers": [
{"radio": "gsm",
"mobileCountryCode": 123,
"mobileNetworkCode": 1,
"locationAreaCode": 2,
"cellId": 1234},
]},
status=200)
self.check_expected_heka_messages(
counter=['http.request', 'geolocate.api_key.test']
)
self.assertEqual(res.content_type, 'application/json')
self.assertEqual(res.json, {"location": {"lat": 12.3456781,
"lng": 23.4567892},
"accuracy": CELL_MIN_ACCURACY})
def test_ok_cell(self):
app = self.app
session = self.db_slave_session
cell = Cell()
cell.lat = 123456781
cell.lon = 234567892
cell.radio = 0
cell.mcc = 123
cell.mnc = 1
cell.lac = 2
cell.cid = 1234
session.add(cell)
session.commit()
res = app.post_json(
'/v1/geolocate?key=test', {
"radioType": "gsm",
"cellTowers": [
{"mobileCountryCode": 123, "mobileNetworkCode": 1,
"locationAreaCode": 2, "cellId": 1234},
]},
status=200)
find_msg = self.find_heka_messages
self.assertEquals(
len(find_msg('counter', 'http.request')), 1)
self.assertEqual(1, len(find_msg('counter', 'geolocate.api_key.test')))
self.assertEqual(res.content_type, 'application/json')
self.assertEqual(res.json, {"location": {"lat": 12.3456781,
"lng": 23.4567892},
"accuracy": 35000.0})
def test_ok_cell_radio_in_celltowers_dupes(self):
# This test covered a bug related to FxOS calling the
# geolocate API incorrectly.
app = self.app
session = self.get_session()
cell = Cell()
cell.lat = PARIS_LAT
cell.lon = PARIS_LON
cell.radio = 0
cell.mcc = FRANCE_MCC
cell.mnc = 1
cell.lac = 2
cell.cid = 1234
session.add(cell)
session.commit()
res = app.post_json(
'%s?key=test' % self.url, {
"cellTowers": [
{"radio": "gsm",
"mobileCountryCode": FRANCE_MCC,
"mobileNetworkCode": 1,
"locationAreaCode": 2,
"cellId": 1234},
{"radio": "gsm",
"mobileCountryCode": FRANCE_MCC,
"mobileNetworkCode": 1,
"locationAreaCode": 2,
"cellId": 1234},
]},
status=200)
self.assertEqual(res.content_type, 'application/json')
self.assertEqual(res.json, {"location": {"lat": PARIS_LAT,
"lng": PARIS_LON},
"accuracy": CELL_MIN_ACCURACY})
def test_removal_updates_lac(self):
session = self.session
keys = dict(radio=Radio.cdma, mcc=1, mnc=1, lac=1)
# setup: build LAC as above
self.add_line_of_cells_and_scan_lac()
# confirm we got one
lac = session.query(CellArea).filter(CellArea.lac == 1).first()
self.assertEqual(lac.lat, 4.5)
self.assertEqual(lac.lon, 4.5)
self.assertEqual(lac.range, 723001)
# Remove cells one by one checking that the LAC
# changes shape along the way.
steps = [
((5.0, 5.0), 644242),
((5.5, 5.5), 565475),
((6.0, 6.0), 486721),
((6.5, 6.5), 408000),
((7.0, 7.0), 329334),
((7.5, 7.5), 250743),
((8.0, 8.0), 172249),
((8.5, 8.5), 93871),
((9.0, 9.0), 15630),
]
for i in range(9):
session.expire(lac)
k = Cell.to_hashkey(cid=i, **keys)
result = remove_cell.delay([k])
self.assertEqual(1, result.get())
result = scan_areas.delay()
self.assertEqual(1, result.get())
lac = session.query(CellArea).filter(CellArea.lac == 1).first()
self.assertEqual(lac.lat, steps[i][0][0])
self.assertEqual(lac.lon, steps[i][0][1])
self.assertEqual(lac.range, steps[i][1])
# Remove final cell, check LAC is gone
k = Cell.to_hashkey(cid=9, **keys)
result = remove_cell.delay([k])
self.assertEqual(1, result.get())
result = scan_areas.delay()
self.assertEqual(1, result.get())
lac = session.query(CellArea).filter(CellArea.lac == 1).first()
self.assertEqual(lac, None)
def test_local_export(self):
session = self.db_master_session
cell_fixture_fields = ('radio', 'cid', 'lat', 'lon', 'mnc', 'mcc',
'lac')
cell_key = {'radio': RADIO_TYPE['gsm'], 'mcc': 1, 'mnc': 2, 'lac': 4}
cells = set()
for cid in range(190, 200):
cell = dict(cid=cid, lat=1.0, lon=2.0, **cell_key)
session.add(Cell(**cell))
cell['radio'] = 'GSM'
cell_strings = [(field, str(value))
for (field, value) in cell.items()]
cell_tuple = tuple(sorted(cell_strings))
cells.add(cell_tuple)
# add one incomplete / unprocessed cell
session.add(Cell(cid=210, lat=None, lon=None, **cell_key))
session.commit()
with selfdestruct_tempdir() as temp_dir:
path = os.path.join(temp_dir, 'export.csv.gz')
cond = Cell.__table__.c.lat.isnot(None)
write_stations_to_csv(session, Cell.__table__, CELL_COLUMNS, cond,
path, make_cell_export_dict, CELL_FIELDS)
with GzipFile(path, 'rb') as gzip_file:
reader = csv.DictReader(gzip_file, CELL_FIELDS)
header = reader.next()
self.assertTrue('area' in header.values())
self.assertEqual(header, CELL_HEADER_DICT)
exported_cells = set()
for exported_cell in reader:
exported_cell_filtered = [
(field, value)
for (field, value) in exported_cell.items()
if field in cell_fixture_fields
]
exported_cell = tuple(sorted(exported_cell_filtered))
exported_cells.add(exported_cell)
self.assertEqual(cells, exported_cells)
def test_insert_measures_invalid_lac(self):
session = self.db_master_session
time = util.utcnow() - timedelta(days=1)
session.add(
Cell(radio=RADIO_TYPE['gsm'],
mcc=FRANCE_MCC,
mnc=2,
lac=3,
cid=4,
new_measures=2,
total_measures=5))
session.add(Score(userid=1, key=SCORE_TYPE['new_cell'], value=7))
session.flush()
measure = dict(created=encode_datetime(time),
lat=PARIS_LAT,
lon=PARIS_LON,
time=encode_datetime(time),
accuracy=0,
altitude=0,
altitude_accuracy=0,
radio=RADIO_TYPE['gsm'])
entries = [
{
"mcc": FRANCE_MCC,
"mnc": 2,
"lac": 3147483647,
"cid": 2147483647,
"psc": 5,
"asu": 8
},
{
"mcc": FRANCE_MCC,
"mnc": 2,
"lac": -1,
"cid": -1,
"psc": 5,
"asu": 8
},
]
for e in entries:
e.update(measure)
result = insert_measures_cell.delay(entries, userid=1)
self.assertEqual(result.get(), 2)
measures = session.query(CellMeasure).all()
self.assertEqual(len(measures), 2)
self.assertEqual(set([m.lac for m in measures]), set([-1]))
self.assertEqual(set([m.cid for m in measures]), set([-1]))
# Nothing should change in the initially created Cell record
cells = session.query(Cell).all()
self.assertEqual(len(cells), 1)
self.assertEqual(set([c.new_measures for c in cells]), set([2]))
self.assertEqual(set([c.total_measures for c in cells]), set([5]))
def remove_cell(self, cell_keys):
cells_removed = 0
redis_client = self.app.redis_client
with self.db_session() as session:
changed_lacs = set()
for k in cell_keys:
key = Cell.to_hashkey(k)
query = session.query(Cell).filter(*Cell.joinkey(key))
cells_removed += query.delete()
changed_lacs.add(CellArea.to_hashkey(key))
if changed_lacs:
session.on_post_commit(enqueue_lacs, redis_client, changed_lacs,
UPDATE_KEY['cell_lac'])
session.commit()
return cells_removed
def test_cell_location_update(self):
from ichnaea.tasks import cell_location_update
now = datetime.utcnow()
before = now - timedelta(days=1)
session = self.db_master_session
k1 = dict(radio=1, mcc=1, mnc=2, lac=3, cid=4)
k2 = dict(radio=1, mcc=1, mnc=2, lac=6, cid=8)
k3 = dict(radio=1, mcc=1, mnc=2, lac=-1, cid=-1)
data = [
Cell(new_measures=3, total_measures=5, **k1),
CellMeasure(lat=10000000, lon=10000000, **k1),
CellMeasure(lat=10020000, lon=10030000, **k1),
CellMeasure(lat=10040000, lon=10060000, **k1),
# The lac, cid are invalid and should be skipped
CellMeasure(lat=15000000, lon=15000000, **k3),
CellMeasure(lat=15020000, lon=15030000, **k3),
Cell(lat=20000000,
lon=20000000,
new_measures=2,
total_measures=4,
**k2),
# the lat/lon is bogus and mismatches the line above on purpose
# to make sure old measures are skipped
CellMeasure(lat=-10000000, lon=-10000000, created=before, **k2),
CellMeasure(lat=-10000000, lon=-10000000, created=before, **k2),
CellMeasure(lat=20020000, lon=20040000, **k2),
CellMeasure(lat=20020000, lon=20040000, **k2),
]
session.add_all(data)
session.commit()
result = cell_location_update.delay(min_new=1)
self.assertEqual(result.get(), (2, 0))
cells = session.query(Cell).filter(Cell.cid != CELLID_LAC).all()
self.assertEqual(len(cells), 2)
self.assertEqual([c.new_measures for c in cells], [0, 0])
for cell in cells:
if cell.cid == 4:
self.assertEqual(cell.lat, 10020000)
self.assertEqual(cell.lon, 10030000)
elif cell.cid == 8:
self.assertEqual(cell.lat, 20010000)
self.assertEqual(cell.lon, 20020000)
def test_location_update_cell(self):
now = util.utcnow()
before = now - timedelta(days=1)
session = self.db_master_session
k1 = dict(radio=1, mcc=1, mnc=2, lac=3, cid=4)
k2 = dict(radio=1, mcc=1, mnc=2, lac=6, cid=8)
k3 = dict(radio=1, mcc=1, mnc=2, lac=-1, cid=-1)
data = [
Cell(new_measures=3, total_measures=5, **k1),
CellMeasure(lat=1.0, lon=1.0, **k1),
CellMeasure(lat=1.002, lon=1.003, **k1),
CellMeasure(lat=1.004, lon=1.006, **k1),
# The lac, cid are invalid and should be skipped
CellMeasure(lat=1.5, lon=1.5, **k3),
CellMeasure(lat=1.502, lon=1.503, **k3),
Cell(lat=2.0, lon=2.0, new_measures=2, total_measures=4, **k2),
# the lat/lon is bogus and mismatches the line above on purpose
# to make sure old measures are skipped
CellMeasure(lat=-1.0, lon=-1.0, created=before, **k2),
CellMeasure(lat=-1.0, lon=-1.0, created=before, **k2),
CellMeasure(lat=2.002, lon=2.004, **k2),
CellMeasure(lat=2.002, lon=2.004, **k2),
]
session.add_all(data)
session.commit()
result = location_update_cell.delay(min_new=1)
self.assertEqual(result.get(), (2, 0))
self.check_stats(
total=2,
timer=['task.data.location_update_cell'],
gauge=['task.data.location_update_cell.new_measures_1_100'],
)
cells = session.query(Cell).filter(Cell.cid != CELLID_LAC).all()
self.assertEqual(len(cells), 2)
self.assertEqual([c.new_measures for c in cells], [0, 0])
for cell in cells:
if cell.cid == 4:
self.assertEqual(cell.lat, 1.002)
self.assertEqual(cell.lon, 1.003)
elif cell.cid == 8:
self.assertEqual(cell.lat, 2.001)
self.assertEqual(cell.lon, 2.002)
def test_inconsistent_cell_radio_in_towers(self):
app = self.app
session = self.db_slave_session
cells = [
Cell(lat=PARIS_LAT,
lon=PARIS_LON,
radio=RADIO_TYPE['gsm'],
mcc=FRANCE_MCC, mnc=1, lac=2, cid=3,
range=10000),
Cell(lat=PARIS_LAT + 0.002,
lon=PARIS_LON + 0.004,
radio=RADIO_TYPE['umts'],
mcc=FRANCE_MCC, mnc=2, lac=3, cid=4,
range=2000),
]
session.add_all(cells)
session.commit()
res = app.post_json(
'%s?key=test' % self.url, {
"radioType": "cdma",
"cellTowers": [
{"radio": "gsm",
"mobileCountryCode": FRANCE_MCC,
"mobileNetworkCode": 1,
"locationAreaCode": 2,
"cellId": 3},
{"radio": "wcdma",
"mobileCountryCode": FRANCE_MCC,
"mobileNetworkCode": 2,
"locationAreaCode": 3,
"cellId": 4},
]},
status=200)
self.check_stats(
counter=[self.metric_url + '.200', self.metric + '.api_key.test']
)
self.assertEqual(res.content_type, 'application/json')
location = res.json['location']
self.assertAlmostEquals(location['lat'], PARIS_LAT + 0.002)
self.assertAlmostEquals(location['lng'], PARIS_LON + 0.004)
self.assertEqual(res.json['accuracy'], CELL_MIN_ACCURACY)
def test_cell_multiple_lac_hit(self):
session = self.db_slave_session
lat = PARIS_LAT
lon = PARIS_LON
gsm = RADIO_TYPE['gsm']
key = dict(mcc=FRANCE_MCC, mnc=2, lac=3)
key2 = dict(mcc=FRANCE_MCC, mnc=2, lac=4)
expected_lac = CellArea(
lat=lat + 0.2, lon=lon + 0.2, radio=gsm,
range=20000, **key)
data = [
Cell(lat=lat + 0.02, lon=lon + 0.02, radio=gsm,
cid=4, range=2000, **key2),
Cell(lat=lat + 0.04, lon=lon + 0.04, radio=gsm,
cid=5, range=3000, **key2),
Cell(lat=lat + 0.2, lon=lon + 0.4, radio=gsm,
cid=5, range=1000, **key),
CellArea(lat=lat, lon=lon, radio=gsm,
range=30000, **key2),
expected_lac,
]
session.add_all(data)
session.flush()
# We have two lacs, both with two cells, but only know about
# one cell in one of them and two in the other.
# The lac with two known cells wins and we use both their
# positions to calculate the final result.
result = self._make_query(data={
"cell": [
dict(radio="gsm", cid=4, **key),
dict(radio="gsm", cid=9, **key),
dict(radio="gsm", cid=4, **key2),
dict(radio="gsm", cid=5, **key2),
]
})
self.assertEqual(result,
{'lat': expected_lac.lat,
'lon': expected_lac.lon,
'accuracy': expected_lac.range})
def test_ok_cell(self):
app = self.app
session = self.db_slave_session
key = dict(mcc=FRANCE_MCC, mnc=2, lac=3)
lat = PARIS_LAT
lon = PARIS_LON
data = [
Cell(lat=lat, lon=lon, range=1000, radio=2, cid=4, **key),
Cell(lat=lat + 0.002,
lon=lon + 0.004,
range=1000,
radio=2,
cid=5,
**key),
]
session.add_all(data)
session.commit()
res = app.post_json('/v1/search?key=test', {
"radio":
"gsm",
"cell": [
dict(radio="umts", cid=4, **key),
dict(radio="umts", cid=5, **key),
]
},
status=200)
self.assertEqual(res.content_type, 'application/json')
self.assertEqual(
res.json, {
"status": "ok",
"lat": PARIS_LAT + 0.001,
"lon": PARIS_LON + 0.002,
"accuracy": CELL_MIN_ACCURACY
})
self.check_stats(
timer=[('request.v1.search', 1)],
counter=[('search.api_key.test', 1), ('search.cell_hit', 1),
('request.v1.search.200', 1),
('search.api_log.test.cell_hit', 1)],
)
def test_unique_cell_histogram(self):
session = self.db_master_session
today = util.utcnow().date()
one_day = (today - timedelta(1))
two_days = (today - timedelta(2))
long_ago = (today - timedelta(3))
cells = [
Cell(created=long_ago, radio=0, mcc=1, mnc=2, lac=3, cid=4),
Cell(created=two_days, radio=2, mcc=1, mnc=2, lac=3, cid=4),
Cell(created=two_days, radio=2, mcc=1, mnc=2, lac=3, cid=5),
Cell(created=one_day, radio=0, mcc=2, mnc=2, lac=3, cid=5),
Cell(created=today, radio=0, mcc=1, mnc=3, lac=3, cid=4),
]
session.add_all(cells)
session.commit()
result = unique_cell_histogram.delay(ago=3)
self.assertEqual(result.get(), 1)
stats = session.query(Stat).order_by(Stat.time).all()
self.assertEqual(len(stats), 1)
self.assertEqual(stats[0].key, STAT_TYPE['unique_cell'])
self.assertEqual(stats[0].time, long_ago)
self.assertEqual(stats[0].value, 1)
# fill up newer dates
unique_cell_histogram.delay(ago=2).get()
unique_cell_histogram.delay(ago=1).get()
unique_cell_histogram.delay(ago=0).get()
# test duplicate execution
unique_cell_histogram.delay(ago=1).get()
stats = session.query(Stat).order_by(Stat.time).all()
self.assertEqual(len(stats), 4)
self.assertEqual(stats[0].time, long_ago)
self.assertEqual(stats[0].value, 1)
self.assertEqual(stats[1].time, two_days)
self.assertEqual(stats[1].value, 3)
self.assertEqual(stats[2].time, one_day)
self.assertEqual(stats[2].value, 4)
self.assertEqual(stats[3].time, today)
self.assertEqual(stats[3].value, 5)
def test_ok_cell(self):
app = self.app
session = self.db_master_session
cell = Cell()
cell.lat = PARIS_LAT + 0.1
cell.lon = PARIS_LON + 0.1
cell.radio = 0
cell.mcc = FRANCE_MCC
cell.mnc = 1
cell.lac = 2
cell.cid = 1234
cell.total_measures = 1
cell.new_measures = 0
session.add(cell)
session.commit()
res = app.post_json('/v1/geosubmit?key=test', {
"latitude": PARIS_LAT,
"longitude": PARIS_LON,
"accuracy": 12.4,
"radioType": "gsm",
"cellTowers": [{
"cellId": 1234,
"locationAreaCode": 2,
"mobileCountryCode": FRANCE_MCC,
"mobileNetworkCode": 1,
}]},
status=200)
# check that we get back a location
self.assertEqual(res.content_type, 'application/json')
self.assertEqual(res.json, {"location": {"lat": PARIS_LAT,
"lng": PARIS_LON},
"accuracy": 12.4})
cell = session.query(Cell).first()
self.assertEqual(2, cell.total_measures)
self.assertEqual(1, cell.new_measures)
# check that one new CellMeasure record is created
self.assertEquals(1, session.query(CellMeasure).count())
def test_lac_miss(self):
session = self.db_slave_session
key = dict(mcc=FRANCE_MCC, mnc=2, lac=3)
lat = PARIS_LAT
lon = PARIS_LON
gsm = RADIO_TYPE['gsm']
data = [
Cell(lat=lat, lon=lon, radio=gsm, cid=4, **key),
Cell(lat=lat + 0.002, lon=lon + 0.004, radio=gsm, cid=5, **key),
Cell(lat=1.006, lon=1.006, radio=gsm, cid=6, **key),
CellArea(lat=1.0026666, lon=1.0033333, radio=gsm,
range=50000, **key),
]
session.add_all(data)
session.flush()
result = self._make_query(
data={"cell": [dict(radio="gsm", mcc=FRANCE_MCC,
mnc=2, lac=4, cid=5)]})
self.assertTrue(result is None)
def test_cell_multiple_lac_hit(self):
session = self.db_slave_session
lat = PARIS_LAT
lon = PARIS_LON
gsm = RADIO_TYPE['gsm']
key = dict(mcc=FRANCE_MCC, mnc=2, lac=3)
key2 = dict(mcc=FRANCE_MCC, mnc=2, lac=4)
data = [
Cell(lat=lat + 0.2, lon=lon + 0.2, radio=gsm,
cid=CELLID_LAC, range=20000, **key),
Cell(lat=lat + 0.2, lon=lon + 0.4, radio=gsm,
cid=5, range=1000, **key),
Cell(lat=lat, lon=lon, radio=gsm,
cid=CELLID_LAC, range=30000, **key2),
Cell(lat=lat + 0.02, lon=lon + 0.02, radio=gsm,
cid=4, range=2000, **key2),
Cell(lat=lat + 0.04, lon=lon + 0.04, radio=gsm,
cid=5, range=3000, **key2),
]
session.add_all(data)
session.flush()
# We have two lacs, both with two cells, but only know about
# one cell in one of them and two in the other.
# The lac with two known cells wins and we use both their
# positions to calculate the final result.
result = locate.search_all_sources(
session, 'm',
{"cell": [
dict(radio="gsm", cid=4, **key),
dict(radio="gsm", cid=9, **key),
dict(radio="gsm", cid=4, **key2),
dict(radio="gsm", cid=5, **key2),
]})
self.assertEqual(result,
{'lat': PARIS_LAT + 0.03,
'lon': PARIS_LON + 0.03,
'accuracy': CELL_MIN_ACCURACY})
def test_scan_areas_race_with_location_update(self):
session = self.session
# First batch of cell observations for CID 1
keys = dict(radio=Radio.cdma, mcc=1, mnc=1, lac=1, cid=1)
cell = Cell(new_measures=4, total_measures=1, **keys)
observations = [
CellObservation(lat=1.0, lon=1.0, **keys),
CellObservation(lat=1.0, lon=1.0, **keys),
CellObservation(lat=1.0, lon=1.0, **keys),
CellObservation(lat=1.0, lon=1.0, **keys),
]
session.add(cell)
session.add_all(observations)
session.commit()
# Periodic location_update_cell runs and updates CID 1
# to have a location, inserts LAC 1 with new_measures=1
# which will be picked up by the next scan_lac.
result = location_update_cell.delay(min_new=1)
self.assertEqual(result.get(), (1, 0))
# Second batch of cell observations for CID 2
keys['cid'] = 2
cell = Cell(new_measures=4, total_measures=1, **keys)
observations = [
CellObservation(lat=1.0, lon=1.0, **keys),
CellObservation(lat=1.0, lon=1.0, **keys),
CellObservation(lat=1.0, lon=1.0, **keys),
CellObservation(lat=1.0, lon=1.0, **keys),
]
session.add(cell)
session.add_all(observations)
session.commit()
# Periodic LAC scan runs, picking up LAC 1; this could
# accidentally pick up CID 2, but it should not since it
# has not had its location updated yet. If there's no
# exception here, CID 2 is properly ignored.
scan_areas.delay()
def test_cell_ignore_invalid_lac_cid(self):
schema = ValidCellKeySchema()
session = self.db_slave_session
lat = PARIS_LAT
lon = PARIS_LON
gsm = RADIO_TYPE['gsm']
lte = RADIO_TYPE['lte']
key = dict(mcc=FRANCE_MCC, mnc=2, lac=3)
ignored_key = dict(
mcc=FRANCE_MCC, mnc=2,
lac=schema.fields['lac'].missing,
cid=schema.fields['cid'].missing)
data = [
Cell(lat=lat, lon=lon, range=1000,
radio=gsm, cid=4, **key),
Cell(lat=lat + 0.002, lon=lon + 0.004, range=1000,
radio=gsm, cid=5, **key),
Cell(lat=lat, lon=lon, range=1000,
radio=gsm, **ignored_key),
Cell(lat=lat + 0.002, lon=lon + 0.004, range=1000,
radio=lte, **ignored_key),
]
session.add_all(data)
session.flush()
result = self._make_query(data={
"cell": [
dict(radio="gsm", cid=4, **key),
dict(radio="gsm", cid=5, **key),
dict(radio="gsm", cid=5, mcc=FRANCE_MCC, mnc=2, lac=-1),
dict(radio="gsm", cid=-1, mcc=FRANCE_MCC, mnc=2, lac=3),
]
})
self.assertEqual(result,
{'lat': PARIS_LAT + 0.001,
'lon': PARIS_LON + 0.002,
'accuracy': CELL_MIN_ACCURACY})
def test_cell_disagrees_with_lac(self):
# This test checks that when a cell is at a lat/lon that
# is not in the LAC associated with it, we drop back
# to the LAC. This likely represents some kind of internal
# database consistency error, but it might also just be a
# new cell that hasn't been integrated yet or something.
session = self.db_slave_session
key = dict(mcc=BRAZIL_MCC, mnc=VIVO_MNC, lac=12345)
data = [
Cell(lat=PORTO_ALEGRE_LAT,
lon=PORTO_ALEGRE_LON,
radio=RADIO_TYPE['gsm'], cid=6789, **key),
Cell(lat=SAO_PAULO_LAT,
lon=SAO_PAULO_LON,
radio=RADIO_TYPE['gsm'], cid=CELLID_LAC, **key),
]
session.add_all(data)
session.flush()
result = locate.search_all_sources(
session, 'm',
{"cell": [dict(radio="gsm", cid=6789, **key)]},
)
self.assertEqual(result,
{'lat': SAO_PAULO_LAT,
'lon': SAO_PAULO_LON,
'accuracy': LAC_MIN_ACCURACY})
self.check_stats(
counter=[
('m.anomaly.cell_cell_lac_mismatch', 1),
('m.country_from_mcc', 1),
('m.cell_lac_found', 1),
('m.cell_found', 1),
('m.cell_lac_hit', 1),
]
)
def test_lac_miss(self):
app = self.app
session = self.db_slave_session
key = dict(mcc=1, mnc=2, lac=3)
data = [
Cell(lat=10000000, lon=10000000, radio=2, cid=4, **key),
Cell(lat=10020000, lon=10040000, radio=2, cid=5, **key),
Cell(lat=10060000, lon=10060000, radio=2, cid=6, **key),
Cell(lat=10026666, lon=10033333, radio=2, cid=CELLID_LAC,
range=50000, **key),
]
session.add_all(data)
session.commit()
res = app.post_json(
'/v1/search?key=test',
{"radio": "gsm", "cell": [
dict(radio="umts", mcc=1, mnc=2, lac=4, cid=5),
]},
status=200)
self.assertEqual(res.content_type, 'application/json')
self.assertEqual(res.json, {"status": "not_found"})
def test_cell_multiple_lac_lower_range_wins(self):
session = self.db_slave_session
lat = PARIS_LAT
lon = PARIS_LON
gsm = RADIO_TYPE['gsm']
key = dict(mcc=FRANCE_MCC, mnc=2, lac=3)
key2 = dict(mcc=FRANCE_MCC, mnc=2, lac=4)
expected_lac = CellArea(
lat=lat + 0.2, lon=lon + 0.2, radio=gsm,
range=10000, **key)
data = [
Cell(lat=lat + 0.02, lon=lon + 0.02, radio=gsm,
cid=4, range=2000, **key2),
Cell(lat=lat + 0.2, lon=lon + 0.4, radio=gsm,
cid=4, range=4000, **key),
CellArea(lat=lat, lon=lon, radio=gsm,
range=20000, **key2),
expected_lac,
]
session.add_all(data)
session.flush()
# We have two lacs with each one known cell.
# The lac with the smallest cell wins.
result = self._make_query(data={
"cell": [
dict(radio="gsm", cid=4, **key),
dict(radio="gsm", cid=4, **key2),
]
})
self.assertEqual(result,
{'lat': expected_lac.lat,
'lon': expected_lac.lon,
'accuracy': LAC_MIN_ACCURACY})
def test_scan_lacs_remove(self):
session = self.db_master_session
redis_client = self.redis_client
# create an orphaned lac entry
key = dict(radio=1, mcc=1, mnc=1, lac=1, cid=CELLID_LAC)
session.add(Cell(**key))
session.flush()
enqueue_lacs(session, redis_client, [CellKey(**key)])
# after scanning the orphaned record gets removed
self.assertEqual(scan_lacs.delay().get(), 1)
lacs = session.query(Cell).filter(Cell.cid == CELLID_LAC).all()
self.assertEqual(lacs, [])
def __call__(self, cell_keys):
cells_removed = 0
changed_areas = set()
area_queue = self.task.app.data_queues['update_cellarea']
for key in cell_keys:
query = Cell.querykey(self.session, key)
cells_removed += query.delete()
changed_areas.add(CellArea.to_hashkey(key))
if changed_areas:
area_queue.enqueue(changed_areas, pipe=self.pipe)
return cells_removed
def test_daily_export(self):
session = self.db_master_session
gsm = RADIO_TYPE['gsm']
k = dict(radio=gsm, mcc=1, mnc=2, lac=4, lat=1.0, lon=2.0)
for i in range(190, 200):
session.add(Cell(cid=i, **k))
session.commit()
with mock_s3() as mock_key:
export_modified_cells(bucket="localhost.bucket", hourly=False)
pat = r"MLS-full-cell-export-\d+-\d+-\d+T000000\.csv\.gz"
self.assertRegexpMatches(mock_key.key, pat)
method = mock_key.set_contents_from_filename
self.assertRegexpMatches(method.call_args[0][0], pat)
def remove(self, cell_keys):
cells_removed = 0
changed_areas = set()
for key in cell_keys:
query = Cell.querykey(self.session, key)
cells_removed += query.delete()
changed_areas.add(CellArea.to_hashkey(key))
if changed_areas:
self.session.on_post_commit(enqueue_areas, self.redis_client,
changed_areas, UPDATE_KEY['cell_lac'])
return cells_removed
def test_cell_multiple_radio_lac_hit_with_min_lac_accuracy(self):
session = self.db_slave_session
lat = PARIS_LAT
lon = PARIS_LON
gsm = RADIO_TYPE['gsm']
lte = RADIO_TYPE['lte']
key = dict(mcc=FRANCE_MCC, mnc=3, lac=4)
key2 = dict(mcc=FRANCE_MCC, mnc=2, lac=3)
expected_lac = CellArea(
lat=lat + 0.2, lon=lon + 0.2, radio=gsm,
range=3000, **key)
data = [
Cell(lat=lat + 0.01, lon=lon + 0.02, radio=lte,
cid=4, range=2000, **key2),
Cell(lat=lat + 0.2, lon=lon + 0.4, radio=gsm,
cid=5, range=500, **key),
CellArea(lat=lat, lon=lon, radio=lte,
range=10000, **key2),
expected_lac,
]
session.add_all(data)
session.flush()
# GSM lac-only hit (cid 9 instead of 5) and a LTE cell hit
result = self._make_query(data={
"cell": [
dict(radio="gsm", cid=9, **key),
dict(radio="lte", cid=4, **key2),
]
})
self.assertEqual(result,
{'lat': expected_lac.lat,
'lon': expected_lac.lon,
'accuracy': LAC_MIN_ACCURACY})
def test_ok_cell(self):
app = self.app
session = self.db_slave_session
cell = Cell()
cell.lat = PARIS_LAT
cell.lon = PARIS_LON
cell.radio = RADIO_TYPE['gsm']
cell.mcc = FRANCE_MCC
cell.mnc = 1
cell.lac = 2
cell.cid = 1234
cell.range = 1000
session.add(cell)
session.commit()
res = app.post_json(
'%s?key=test' % self.url, {
"radioType": "gsm",
"cellTowers": [
{"mobileCountryCode": FRANCE_MCC,
"mobileNetworkCode": 1,
"locationAreaCode": 2,
"cellId": 1234},
]},
status=200)
self.check_stats(
counter=[self.metric_url + '.200',
self.metric + '.api_key.test',
self.metric + '.api_log.test.cell_hit']
)
self.assertEqual(res.content_type, 'application/json')
self.assertEqual(res.json, {"location": {"lat": PARIS_LAT,
"lng": PARIS_LON},
"accuracy": CELL_MIN_ACCURACY})
def test_ok_cell_radio_in_celltowers(self):
# This test covers a bug related to FxOS calling the
# geolocate API incorrectly.
app = self.app
session = self.db_slave_session
cell = Cell()
cell.lat = PARIS_LAT
cell.lon = PARIS_LON
cell.radio = 0
cell.mcc = FRANCE_MCC
cell.mnc = 1
cell.lac = 2
cell.cid = 1234
cell.range = 1000
session.add(cell)
session.commit()
res = app.post_json(
'%s?key=test' % self.url, {
"cellTowers": [
{"radio": "gsm",
"mobileCountryCode": FRANCE_MCC,
"mobileNetworkCode": 1,
"locationAreaCode": 2,
"cellId": 1234},
]},
status=200)
self.check_stats(
counter=[self.metric_url + '.200', self.metric + '.api_key.test']
)
self.assertEqual(res.content_type, 'application/json')
self.assertEqual(res.json, {"location": {"lat": PARIS_LAT,
"lng": PARIS_LON},
"accuracy": CELL_MIN_ACCURACY})
def test_cell_agrees_with_lac(self):
# This test checks that when a cell is at a lat/lon that
# is inside its enclosing LAC, we accept it and tighten
# our accuracy accordingly.
session = self.db_slave_session
key = dict(mcc=BRAZIL_MCC, mnc=VIVO_MNC, lac=12345)
data = [
Cell(lat=SAO_PAULO_LAT + 0.002,
lon=SAO_PAULO_LON + 0.002,
radio=RADIO_TYPE['gsm'], cid=6789, **key),
Cell(lat=SAO_PAULO_LAT,
lon=SAO_PAULO_LON,
radio=RADIO_TYPE['gsm'], cid=CELLID_LAC, **key),
]
session.add_all(data)
session.flush()
result = locate.search_all_sources(
session, 'm',
{"cell": [dict(radio="gsm", cid=6789, **key)]},
)
self.assertEqual(result,
{'lat': SAO_PAULO_LAT + 0.002,
'lon': SAO_PAULO_LON + 0.002,
'accuracy': CELL_MIN_ACCURACY})
self.check_stats(
counter=[
('m.country_from_mcc', 1),
('m.cell_lac_found', 1),
('m.cell_found', 1),
('m.cell_hit', 1),
]
)
def test_inconsistent_cell_radio_in_towers(self):
app = self.app
session = self.db_slave_session
cell = Cell()
cell.lat = 123456781
cell.lon = 234567892
cell.radio = 0
cell.mcc = 123
cell.mnc = 1
cell.lac = 2
cell.cid = 1234
session.add(cell)
session.commit()
res = app.post_json(
'/v1/geolocate?key=test', {
"cellTowers": [
{"radio": "gsm",
"mobileCountryCode": 123,
"mobileNetworkCode": 1,
"locationAreaCode": 2,
"cellId": 1234},
{"radio": "cdma",
"mobileCountryCode": 123,
"mobileNetworkCode": 1,
"locationAreaCode": 2,
"cellId": 1234},
]},
status=400)
self.check_expected_heka_messages(
counter=['http.request', 'geolocate.api_key.test']
)
self.assertEqual(res.content_type, 'application/json')
self.assertEqual(
res.json, {"error": {
"errors": [{
"domain": "global",
"reason": "parseError",
"message": "Parse Error",
}],
"code": 400,
"message": "Parse Error"
}}
)
def test_ok_cell(self):
app = self.app
session = self.db_slave_session
key = dict(mcc=1, mnc=2, lac=3)
data = [
Cell(lat=10000000, lon=10000000, radio=2, cid=4, **key),
Cell(lat=10020000, lon=10040000, radio=2, cid=5, **key),
]
session.add_all(data)
session.commit()
res = app.post_json('/v1/search', {
"radio":
"gsm",
"cell": [
dict(radio="umts", cid=4, **key),
dict(radio="umts", cid=5, **key),
]
},
status=200)
self.assertEqual(res.content_type, 'application/json')
self.assertEqual(
res.body, '{"status": "ok", "lat": 1.0010000, '
'"lon": 1.0020000, "accuracy": 35000}')
def test_cell_hit_ignores_lac(self):
session = self.db_slave_session
lat = PARIS_LAT
lon = PARIS_LON
key = dict(mcc=FRANCE_MCC, mnc=2, lac=3)
data = [
Cell(lat=lat, lon=lon, radio=2, cid=4, **key),
Cell(lat=lat + 0.002, lon=lon + 0.004, radio=2, cid=5, **key),
Cell(lat=lat + 0.006, lon=lon + 0.006, radio=2, cid=6, **key),
Cell(lat=lat + 0.0026666,
lon=lon + 0.0033333, radio=2, cid=CELLID_LAC,
range=50000, **key),
]
session.add_all(data)
session.flush()
result = locate.search_all_sources(
session, 'm',
{"cell": [dict(radio="umts", cid=5, **key)]})
self.assertEqual(result,
{'lat': PARIS_LAT + 0.002,
'lon': PARIS_LON + 0.004,
'accuracy': CELL_MIN_ACCURACY})
def test_geoip_mcc_mismatch(self):
session = self.db_slave_session
gsm = RADIO_TYPE['gsm']
key = {'mcc': USA_MCC, 'mnc': 1, 'lac': 1, 'cid': 1}
key2 = {'mcc': USA_MCC, 'mnc': 1, 'lac': 1, 'cid': CELLID_LAC}
session.add(Cell(radio=gsm, lat=FREMONT_LAT, lon=FREMONT_LON, **key))
session.add(Cell(radio=gsm, lat=FREMONT_LAT, lon=FREMONT_LON, **key2))
session.flush()
result = locate.search_all_sources(
session, 'm',
{'cell': [dict(radio='gsm', **key)]},
client_addr=GB_IP, geoip_db=self.geoip_db)
self.assertEqual(result,
{'lat': FREMONT_LAT,
'lon': FREMONT_LON,
'accuracy': CELL_MIN_ACCURACY})
self.check_stats(
counter=[
'm.anomaly.geoip_mcc_mismatch',
],
)
def test_wifi_not_found_cell_fallback(self):
app = self.app
session = self.db_slave_session
key = dict(mcc=1, mnc=2, lac=3)
data = [
Wifi(key="abcd", lat=30000000, lon=30000000),
Cell(lat=10000000, lon=10000000, radio=2, cid=4, **key),
Cell(lat=10020000, lon=10040000, radio=2, cid=5, **key),
]
session.add_all(data)
session.commit()
res = app.post_json('/v1/search?key=test', {
"radio":
"gsm",
"cell": [
dict(radio="umts", cid=4, **key),
dict(radio="umts", cid=5, **key),
],
"wifi": [
{
"key": "abcd"
},
{
"key": "cdef"
},
]
},
status=200)
self.assertEqual(res.content_type, 'application/json')
self.assertEqual(res.json, {
"status": "ok",
"lat": 1.0010000,
"lon": 1.0020000,
"accuracy": 35000
})
def remove(self, cell_keys):
cells_removed = 0
changed_areas = set()
for key in cell_keys:
query = Cell.querykey(self.session, key)
cells_removed += query.delete()
changed_areas.add(CellArea.to_hashkey(key))
if changed_areas:
redis_key = self.task.app.data_queues['cell_area_update']
self.session.on_post_commit(enqueue_areas, self.redis_client,
changed_areas, redis_key)
return cells_removed
def test_ok_cell(self):
app = self.app
session = self.db_slave_session
key = dict(mcc=1, mnc=2, lac=3)
data = [
Cell(lat=10000000, lon=10000000, radio=2, cid=4, **key),
Cell(lat=10020000, lon=10040000, radio=2, cid=5, **key),
]
session.add_all(data)
session.commit()
res = app.post_json('/v1/search?key=test', {
"radio":
"gsm",
"cell": [
dict(radio="umts", cid=4, **key),
dict(radio="umts", cid=5, **key),
]
},
status=200)
self.assertEqual(res.content_type, 'application/json')
self.assertEqual(res.json, {
"status": "ok",
"lat": 1.0010000,
"lon": 1.0020000,
"accuracy": 35000
})
self.check_expected_heka_messages(total=4,
timer=[('http.request', {
'url_path': '/v1/search'
})],
counter=[('search.api_key', 1),
('search.cell_hit', 1),
('http.request', 1)])
def process_reports(self, reports, userid=None):
malformed_reports = 0
positions = set()
observations = {'cell': [], 'wifi': []}
obs_count = {
'cell': {'upload': 0, 'drop': 0},
'wifi': {'upload': 0, 'drop': 0},
}
new_station_count = {'cell': 0, 'wifi': 0}
for report in reports:
cell, wifi, malformed_obs = self.process_report(report)
if cell:
observations['cell'].extend(cell)
obs_count['cell']['upload'] += len(cell)
if wifi:
observations['wifi'].extend(wifi)
obs_count['wifi']['upload'] += len(wifi)
if (cell or wifi):
positions.add((report['lat'], report['lon']))
else:
malformed_reports += 1
for name in ('cell', 'wifi'):
obs_count[name]['drop'] += malformed_obs[name]
# group by unique station key
for name in ('cell', 'wifi'):
station_keys = set()
for obs in observations[name]:
if name == 'cell':
station_keys.add(Cell.to_hashkey(obs))
elif name == 'wifi':
station_keys.add(obs.mac)
# determine scores for stations
new_station_count[name] += self.new_stations(name, station_keys)
for name, queue in (('cell', self.cell_queue),
('wifi', self.wifi_queue)):
if observations[name]:
queue.enqueue(list(observations[name]), pipe=self.pipe)
self.process_mapstat(positions)
self.process_score(userid, positions, new_station_count)
self.emit_stats(
len(reports),
malformed_reports,
obs_count,
)
def test_ok_cell(self):
app = self.app
session = self.db_slave_session
cell = Cell()
cell.lat = 123456781
cell.lon = 234567892
cell.radio = 0
cell.mcc = 123
cell.mnc = 1
cell.lac = 2
cell.cid = 1234
session.add(cell)
session.commit()
res = app.post_json('/v1/geolocate?key=test', {
"radioType":
"gsm",
"cellTowers": [
{
"mobileCountryCode": 123,
"mobileNetworkCode": 1,
"locationAreaCode": 2,
"cellId": 1234
},
]
},
status=200)
self.check_expected_heka_messages(
counter=['http.request', 'geolocate.api_key.test'])
self.assertEqual(res.content_type, 'application/json')
self.assertEqual(
res.json, {
"location": {
"lat": 12.3456781,
"lng": 23.4567892
},
"accuracy": 35000.0
})
def remove(self, cell_keys):
cells_removed = 0
changed_areas = set()
for key in cell_keys:
query = Cell.querykey(self.session, key)
cells_removed += query.delete()
changed_areas.add(CellArea.to_hashkey(key))
if changed_areas:
redis_key = self.task.app.data_queues['cell_area_update']
self.session.on_post_commit(
enqueue_areas,
self.redis_client,
changed_areas,
redis_key)
return cells_removed
def new_stations(self, name, station_keys):
if len(station_keys) == 0:
return 0
# assume all stations are unknown
unknown_keys = set(station_keys)
if name == 'wifi':
# there is only one combined table structure
shards = defaultdict(list)
for mac in unknown_keys:
shards[WifiShard.shard_model(mac)].append(mac)
for shard, macs in shards.items():
query = (self.session.query(shard.mac)
.filter(shard.mac.in_(macs)))
unknown_keys -= set([r.mac for r in query.all()])
elif name == 'cell':
# first check the station table, which is more likely to contain
# stations
station_iter = Cell.iterkeys(
self.session,
list(unknown_keys),
# only load the columns required for the hashkey
extra=lambda query: query.options(
load_only(*tuple(Cell._hashkey_cls._fields))))
# subtract all stations which are found in the station table
unknown_keys -= set([sta.hashkey() for sta in station_iter])
if len(unknown_keys) == 0: # pragma: no cover
return 0
# Only check the blocklist table for the still unknown keys.
# There is no need to check for the already found keys again.
block_iter = CellBlocklist.iterkeys(
self.session,
list(unknown_keys),
# only load the columns required for the hashkey
extra=lambda query: query.options(
load_only(*tuple(CellBlocklist._hashkey_cls._fields))))
# subtract all stations which are found in the blocklist table
unknown_keys -= set([block.hashkey() for block in block_iter])
return len(unknown_keys)
def test_ok_cell(self):
app = self.app
session = self.db_slave_session
cell = Cell()
cell.lat = 123456781
cell.lon = 234567892
cell.radio = 2
cell.mcc = 123
cell.mnc = 1
cell.lac = 2
cell.cid = 1234
session.add(cell)
session.commit()
res = app.post_json('/v1/search',
{"radio": "gsm",
"cell": [{"radio": "umts", "mcc": 123, "mnc": 1,
"lac": 2, "cid": 1234}]},
status=200)
self.assertEqual(res.content_type, 'application/json')
self.assertEqual(res.body, '{"status": "ok", "lat": 12.3456781, '
'"lon": 23.4567892, "accuracy": 35000}')
def update_lac(self, radio, mcc, mnc, lac):
with self.db_session() as session:
# Select all the cells in this LAC that aren't the virtual
# cell itself, and derive a bounding box for them.
q = session.query(Cell).filter(
Cell.radio == radio).filter(
Cell.mcc == mcc).filter(
Cell.mnc == mnc).filter(
Cell.lac == lac).filter(
Cell.cid != CELLID_LAC).filter(
Cell.new_measures == 0).filter(
Cell.lat.isnot(None)).filter(
Cell.lon.isnot(None))
cells = q.all()
if len(cells) == 0:
return
points = [(to_degrees(c.lat),
to_degrees(c.lon)) for c in cells]
min_lat = to_degrees(min([c.min_lat for c in cells]))
min_lon = to_degrees(min([c.min_lon for c in cells]))
max_lat = to_degrees(max([c.max_lat for c in cells]))
max_lon = to_degrees(max([c.max_lon for c in cells]))
bbox_points = [(min_lat, min_lon),
(min_lat, max_lon),
(max_lat, min_lon),
(max_lat, max_lon)]
ctr = centroid(points)
rng = range_to_points(ctr, bbox_points)
# switch units back to DB preferred centimicrodegres angle
# and meters distance.
ctr_lat = from_degrees(ctr[0])
ctr_lon = from_degrees(ctr[1])
rng = int(round(rng * 1000.0))
# Now create or update the LAC virtual cell
q = session.query(Cell).filter(
Cell.radio == radio).filter(
Cell.mcc == mcc).filter(
Cell.mnc == mnc).filter(
Cell.lac == lac).filter(
Cell.cid == CELLID_LAC)
lac = q.first()
if lac is None:
lac = Cell(radio=radio,
mcc=mcc,
mnc=mnc,
lac=lac,
cid=CELLID_LAC,
lat=ctr_lat,
lon=ctr_lon,
range=rng)
else:
lac.new_measures = 0
lac.lat = ctr_lat
lac.lon = ctr_lon
lac.range = rng
session.commit()
def update_lac(self, radio, mcc, mnc, lac):
try:
utcnow = util.utcnow()
with self.db_session() as session:
# Select all the cells in this LAC that aren't the virtual
# cell itself, and derive a bounding box for them.
cell_query = session.query(Cell).filter(
Cell.radio == radio).filter(
Cell.mcc == mcc).filter(
Cell.mnc == mnc).filter(
Cell.lac == lac).filter(
Cell.cid != CELLID_LAC).filter(
Cell.lat.isnot(None)).filter(
Cell.lon.isnot(None))
cells = cell_query.all()
lac_query = session.query(Cell).filter(
Cell.radio == radio).filter(
Cell.mcc == mcc).filter(
Cell.mnc == mnc).filter(
Cell.lac == lac).filter(
Cell.cid == CELLID_LAC)
if len(cells) == 0:
# If there are no more underlying cells, delete the lac entry
lac_query.delete()
else:
# Otherwise update the lac entry based on all the cells
lac_obj = lac_query.first()
points = [(c.lat, c.lon) for c in cells]
min_lat = min([c.min_lat for c in cells])
min_lon = min([c.min_lon for c in cells])
max_lat = max([c.max_lat for c in cells])
max_lon = max([c.max_lon for c in cells])
bbox_points = [(min_lat, min_lon),
(min_lat, max_lon),
(max_lat, min_lon),
(max_lat, max_lon)]
ctr = centroid(points)
rng = range_to_points(ctr, bbox_points)
# Switch units back to DB preferred centimicrodegres angle
# and meters distance.
ctr_lat = ctr[0]
ctr_lon = ctr[1]
rng = int(round(rng * 1000.0))
# Now create or update the LAC virtual cell
if lac_obj is None:
lac_obj = Cell(radio=radio,
mcc=mcc,
mnc=mnc,
lac=lac,
cid=CELLID_LAC,
lat=ctr_lat,
lon=ctr_lon,
created=utcnow,
modified=utcnow,
range=rng,
total_measures=len(cells))
session.add(lac_obj)
else:
lac_obj.total_measures = len(cells)
lac_obj.lat = ctr_lat
lac_obj.lon = ctr_lon
lac_obj.modified = utcnow
lac_obj.range = rng
session.commit()
except Exception as exc: # pragma: no cover
self.heka_client.raven('error')
raise self.retry(exc=exc)
def __call__(self, batch=10):
all_observations = self.data_queue.dequeue(batch=batch)
drop_counter = defaultdict(int)
added = 0
new_stations = 0
station_obs = defaultdict(list)
for obs in all_observations:
station_obs[Cell.to_hashkey(obs)].append(obs)
if not station_obs:
return (0, 0)
stations = {}
for station in Cell.iterkeys(self.session, list(station_obs.keys())):
stations[station.hashkey()] = station
blocklist = self.blocklisted_stations(station_obs.keys())
new_station_values = []
changed_station_values = []
moving_stations = set()
for station_key, observations in station_obs.items():
blocked, first_blocked, block = blocklist.get(
station_key, (False, None, None))
if not any(observations):
continue
if blocked:
# Drop observations for blocklisted stations.
drop_counter['blocklisted'] += len(observations)
continue
station = stations.get(station_key, None)
if station is None and not first_blocked:
# We discovered an actual new never before seen station.
new_stations += 1
moving, new_values, changed_values = self.new_station_values(
station, station_key, first_blocked, observations)
if moving:
moving_stations.add((station_key, block))
else:
added += len(observations)
if new_values:
new_station_values.append(new_values)
if changed_values:
changed_station_values.append(changed_values)
# track potential updates to dependent areas
self.add_area_update(station_key)
if new_station_values:
# do a batch insert of new stations
stmt = Cell.__table__.insert(
mysql_on_duplicate='total_measures = total_measures' # no-op
)
# but limit the batch depending on each model
ins_batch = Cell._insert_batch
for i in range(0, len(new_station_values), ins_batch):
batch_values = new_station_values[i:i + ins_batch]
self.session.execute(stmt.values(batch_values))
if changed_station_values:
# do a batch update of changed stations
ins_batch = Cell._insert_batch
for i in range(0, len(changed_station_values), ins_batch):
batch_values = changed_station_values[i:i + ins_batch]
self.session.bulk_update_mappings(Cell, batch_values)
if self.updated_areas:
self.queue_area_updates()
if moving_stations:
self.blocklist_stations(moving_stations)
self.emit_stats(added, drop_counter)
self.emit_statcounters(added, new_stations)
if self.data_queue.enough_data(batch=batch): # pragma: no cover
self.update_task.apply_async(
kwargs={'batch': batch},
countdown=2,
expires=10)
return (len(stations) + len(new_station_values), len(moving_stations))