def add_line_of_cells_and_scan_lac(self):
session = self.session
big = 1.0
small = big / 10
keys = dict(radio=Radio.cdma, mcc=1, mnc=1, lac=1)
observations = [
CellObservation(lat=ctr + xd, lon=ctr + yd, cid=cell, **keys)
for cell in range(10)
for ctr in [cell * big]
for (xd, yd) in [(small, small),
(small, -small),
(-small, small),
(-small, -small)]
]
session.add_all(observations)
cells = [
Cell(lat=ctr, lon=ctr, cid=cell,
new_measures=4, total_measures=1, **keys)
for cell in range(10)
for ctr in [cell * big]
]
session.add_all(cells)
session.commit()
result = location_update_cell.delay(min_new=0,
max_new=9999,
batch=len(observations))
self.assertEqual(result.get(), (len(cells), 0))
scan_areas.delay()
def test_max_min_range_update(self):
session = self.db_master_session
k1 = dict(radio=1, mcc=1, mnc=2, lac=3, cid=4)
data = [
Cell(lat=1.001, lon=-1.001,
max_lat=1.002, min_lat=1.0,
max_lon=-1.0, min_lon=-1.002,
new_measures=2, total_measures=4, **k1),
CellMeasure(lat=1.001, lon=-1.003, **k1),
CellMeasure(lat=1.005, lon=-1.007, **k1),
]
session.add_all(data)
session.commit()
result = location_update_cell.delay(min_new=1)
self.assertEqual(result.get(), (1, 0))
cells = session.query(Cell).filter(Cell.cid != CELLID_LAC).all()
self.assertEqual(len(cells), 1)
cell = cells[0]
self.assertEqual(cell.lat, 1.002)
self.assertEqual(cell.max_lat, 1.005)
self.assertEqual(cell.min_lat, 1.0)
self.assertEqual(cell.lon, -1.003)
self.assertEqual(cell.max_lon, -1.0)
self.assertEqual(cell.min_lon, -1.007)
# independent calculation: the cell bounding box is
# (1.000, -1.007) to (1.005, -1.000), with centroid
# at (1.002, -1.003); worst distance from centroid
# to any corner is 556m
self.assertEqual(cell.range, 556)
def add_line_of_cells_and_scan_lac(self):
session = self.session
big = 1.0
small = big / 10
keys = dict(radio=Radio.cdma, mcc=1, mnc=1, lac=1)
observations = [
CellObservation(lat=ctr + xd, lon=ctr + yd, cid=cell, **keys)
for cell in range(10)
for ctr in [cell * big]
for (xd, yd) in [(small, small),
(small, -small),
(-small, small),
(-small, -small)]
]
session.add_all(observations)
cells = [
Cell(lat=ctr, lon=ctr, cid=cell,
new_measures=4, total_measures=1, **keys)
for cell in range(10)
for ctr in [cell * big]
]
session.add_all(cells)
session.commit()
result = location_update_cell.delay(min_new=0,
max_new=9999,
batch=len(observations))
self.assertEqual(result.get(), (len(cells), 0))
scan_areas.delay()
def test_location_update_cell(self):
now = util.utcnow()
before = now - timedelta(hours=1)
schema = ValidCellKeySchema()
obs_factory = CellObservationFactory
invalid_key = dict(lac=schema.fields['lac'].missing,
cid=schema.fields['cid'].missing)
cell1 = CellFactory(new_measures=3, total_measures=5)
lat1, lon1 = (cell1.lat, cell1.lon)
key1 = dict(lac=cell1.lac, cid=cell1.cid)
obs_factory(lat=lat1, lon=lon1, created=now, **key1)
obs_factory(lat=lat1 + 0.004, lon=lon1 + 0.006, created=now, **key1)
obs_factory(lat=lat1 + 0.006, lon=lon1 + 0.009, created=now, **key1)
# The lac, cid are invalid and should be skipped
obs_factory.create_batch(2, created=now, **invalid_key)
cell2 = CellFactory(lat=lat1 + 1.0, lon=lon1 + 1.0,
new_measures=2, total_measures=4)
lat2, lon2 = (cell2.lat, cell2.lon)
key2 = dict(lac=cell2.lac, cid=cell2.cid)
# the lat/lon is bogus and mismatches the line above on purpose
# to make sure old observations are skipped
obs_factory(lat=lat2 - 2.0, lon=lon2 - 2.0, created=before, **key2)
obs_factory(lat=lat2 - 2.0, lon=lon2 - 2.0, created=before, **key2)
obs_factory(lat=lat2 + 0.001, lon=lon2 + 0.002, created=now, **key2)
obs_factory(lat=lat2 + 0.003, lon=lon2 + 0.006, created=now, **key2)
cell3 = CellFactory(new_measures=10, total_measures=100000)
lat3, lon3 = (cell3.lat, cell3.lon)
obs_factory.create_batch(
10, lat=lat3 + 1.0, lon=lon3 + 1.0,
**dict(lac=cell3.lac, cid=cell3.cid))
self.session.commit()
result = location_update_cell.delay(min_new=1)
self.assertEqual(result.get(), (3, 0))
self.check_stats(
total=2,
timer=['task.data.location_update_cell'],
gauge=['task.data.location_update_cell.new_measures_1_100'],
)
cells = self.session.query(Cell).all()
self.assertEqual(len(cells), 3)
self.assertEqual(set([c.new_measures for c in cells]), set([0]))
for cell in cells:
if cell.hashkey() == cell1.hashkey():
self.assertEqual(cell.lat, lat1 + 0.002)
self.assertEqual(cell.lon, lon1 + 0.003)
if cell.hashkey() == cell2.hashkey():
self.assertEqual(cell.lat, lat2 + 0.001)
self.assertEqual(cell.lon, lon2 + 0.002)
if cell.hashkey() == cell3.hashkey():
expected_lat = ((lat3 * 1000) + (lat3 + 1.0) * 10) / 1010
expected_lon = ((lon3 * 1000) + (lon3 + 1.0) * 10) / 1010
self.assertAlmostEqual(cell.lat, expected_lat, 7)
self.assertAlmostEqual(cell.lon, expected_lon, 7)
def test_update_cell(self):
now = util.utcnow()
invalid_key = dict(lac=None, cid=None)
observations = []
def obs_factory(**kw):
obs = CellObservationFactory.create(**kw)
observations.append(obs)
cell1 = CellFactory(new_measures=3, total_measures=3)
lat1, lon1 = (cell1.lat, cell1.lon)
key1 = dict(lac=cell1.lac, cid=cell1.cid)
obs_factory(lat=lat1, lon=lon1, created=now, **key1)
obs_factory(lat=lat1 + 0.004, lon=lon1 + 0.006, created=now, **key1)
obs_factory(lat=lat1 + 0.006, lon=lon1 + 0.009, created=now, **key1)
# The lac, cid are invalid and should be skipped
obs_factory(created=now, **invalid_key)
obs_factory(created=now, **invalid_key)
cell2 = CellFactory(lat=lat1 + 1.0, lon=lon1 + 1.0,
new_measures=2, total_measures=3)
lat2, lon2 = (cell2.lat, cell2.lon)
key2 = dict(lac=cell2.lac, cid=cell2.cid)
obs_factory(lat=lat2 + 0.001, lon=lon2 + 0.002, created=now, **key2)
obs_factory(lat=lat2 + 0.003, lon=lon2 + 0.006, created=now, **key2)
cell3 = CellFactory(new_measures=10, total_measures=100000)
lat3, lon3 = (cell3.lat, cell3.lon)
for i in range(10):
obs_factory(lat=lat3 + 1.0, lon=lon3 + 1.0,
**dict(lac=cell3.lac, cid=cell3.cid))
self.session.add_all(observations)
self.session.commit()
result = location_update_cell.delay()
self.assertEqual(result.get(), (3, 0))
self.check_stats(
timer=['task.data.location_update_cell'],
)
cells = self.session.query(Cell).all()
self.assertEqual(len(cells), 3)
self.assertEqual(set([c.new_measures for c in cells]), set([0]))
for cell in cells:
if cell.hashkey() == cell1.hashkey():
self.assertAlmostEqual(cell.lat, lat1 + 0.001667, 6)
self.assertAlmostEqual(cell.lon, lon1 + 0.0025, 6)
if cell.hashkey() == cell2.hashkey():
self.assertAlmostEqual(cell.lat, lat2 + 0.0008, 6)
self.assertAlmostEqual(cell.lon, lon2 + 0.0016, 6)
if cell.hashkey() == cell3.hashkey():
expected_lat = ((lat3 * 1000) + (lat3 + 1.0) * 10) / 1010
expected_lon = ((lon3 * 1000) + (lon3 + 1.0) * 10) / 1010
self.assertAlmostEqual(cell.lat, expected_lat, 7)
self.assertAlmostEqual(cell.lon, expected_lon, 7)
def test_location_update_cell(self):
now = util.utcnow()
before = now - timedelta(days=1)
schema = ValidCellKeySchema()
session = self.db_master_session
k1 = dict(radio=1, mcc=1, mnc=2, lac=3, cid=4)
k2 = dict(radio=1, mcc=1, mnc=2, lac=6, cid=8)
k3 = dict(radio=1,
mcc=1,
mnc=2,
lac=schema.fields['lac'].missing,
cid=schema.fields['cid'].missing)
data = [
Cell(new_measures=3, total_measures=5, **k1),
CellObservation(lat=1.0, lon=1.0, created=now, **k1),
CellObservation(lat=1.002, lon=1.003, created=now, **k1),
CellObservation(lat=1.004, lon=1.006, created=now, **k1),
# The lac, cid are invalid and should be skipped
CellObservation(lat=1.5, lon=1.5, created=now, **k3),
CellObservation(lat=1.502, lon=1.503, created=now, **k3),
Cell(lat=2.0, lon=2.0, new_measures=2, total_measures=4, **k2),
# the lat/lon is bogus and mismatches the line above on purpose
# to make sure old observations are skipped
CellObservation(lat=-1.0, lon=-1.0, created=before, **k2),
CellObservation(lat=-1.0, lon=-1.0, created=before, **k2),
CellObservation(lat=2.002, lon=2.004, created=now, **k2),
CellObservation(lat=2.002, lon=2.004, created=now, **k2),
]
session.add_all(data)
session.commit()
result = location_update_cell.delay(min_new=1)
self.assertEqual(result.get(), (2, 0))
self.check_stats(
total=2,
timer=['task.data.location_update_cell'],
gauge=['task.data.location_update_cell.new_measures_1_100'],
)
cells = session.query(Cell).all()
self.assertEqual(len(cells), 2)
self.assertEqual([c.new_measures for c in cells], [0, 0])
for cell in cells:
if cell.cid == 4:
self.assertEqual(cell.lat, 1.002)
self.assertEqual(cell.lon, 1.003)
elif cell.cid == 8:
self.assertEqual(cell.lat, 2.001)
self.assertEqual(cell.lon, 2.002)
def test_location_update_cell(self):
now = util.utcnow()
before = now - timedelta(days=1)
session = self.db_master_session
k1 = dict(radio=1, mcc=1, mnc=2, lac=3, cid=4)
k2 = dict(radio=1, mcc=1, mnc=2, lac=6, cid=8)
k3 = dict(radio=1, mcc=1, mnc=2, lac=-1, cid=-1)
data = [
Cell(new_measures=3, total_measures=5, **k1),
CellMeasure(lat=1.0, lon=1.0, **k1),
CellMeasure(lat=1.002, lon=1.003, **k1),
CellMeasure(lat=1.004, lon=1.006, **k1),
# The lac, cid are invalid and should be skipped
CellMeasure(lat=1.5, lon=1.5, **k3),
CellMeasure(lat=1.502, lon=1.503, **k3),
Cell(lat=2.0, lon=2.0,
new_measures=2, total_measures=4, **k2),
# the lat/lon is bogus and mismatches the line above on purpose
# to make sure old measures are skipped
CellMeasure(lat=-1.0, lon=-1.0, created=before, **k2),
CellMeasure(lat=-1.0, lon=-1.0, created=before, **k2),
CellMeasure(lat=2.002, lon=2.004, **k2),
CellMeasure(lat=2.002, lon=2.004, **k2),
]
session.add_all(data)
session.commit()
result = location_update_cell.delay(min_new=1)
self.assertEqual(result.get(), (2, 0))
self.check_stats(
total=2,
timer=['task.data.location_update_cell'],
gauge=['task.data.location_update_cell.new_measures_1_100'],
)
cells = session.query(Cell).filter(Cell.cid != CELLID_LAC).all()
self.assertEqual(len(cells), 2)
self.assertEqual([c.new_measures for c in cells], [0, 0])
for cell in cells:
if cell.cid == 4:
self.assertEqual(cell.lat, 1.002)
self.assertEqual(cell.lon, 1.003)
elif cell.cid == 8:
self.assertEqual(cell.lat, 2.001)
self.assertEqual(cell.lon, 2.002)
def test_scan_lacs_asymmetric(self):
session = self.db_master_session
big = 0.1
small = big / 10
keys = dict(radio=1, mcc=1, mnc=1, lac=1)
measures = [
CellMeasure(lat=ctr + xd, lon=ctr + yd, cid=cell, **keys)
for cell in range(6)
for ctr in [(2 ** cell) * big]
for (xd, yd) in [(small, small),
(small, -small),
(-small, small),
(-small, -small)]
]
session.add_all(measures)
cells = [
Cell(lat=ctr, lon=ctr, cid=cell,
new_measures=4, total_measures=1, **keys)
for cell in range(6)
for ctr in [(2 ** cell) * big]
]
session.add_all(cells)
session.commit()
result = location_update_cell.delay(min_new=0,
max_new=9999,
batch=len(measures))
self.assertEqual(result.get(), (len(cells), 0))
scan_lacs.delay()
lac = session.query(Cell).filter(
Cell.lac == 1,
Cell.cid == CELLID_LAC).first()
# We produced a sequence of 0.02-degree-on-a-side
# cell bounding boxes centered at
# [0, 0.2, 0.4, 0.8, 1.6, 3.2] degrees.
# So the lower-left corner is at (-0.01, -0.01)
# and the upper-right corner is at (3.21, 3.21)
# we should therefore see a LAC centroid at (1.05, 1.05)
# with a range of 339.540m
self.assertEqual(lac.lat, 1.05)
self.assertEqual(lac.lon, 1.05)
self.assertEqual(lac.range, 339540)
def test_scan_lacs_asymmetric(self):
session = self.db_master_session
big = 0.1
small = big / 10
keys = dict(radio=1, mcc=1, mnc=1, lac=1)
measures = [
CellMeasure(lat=ctr + xd, lon=ctr + yd, cid=cell, **keys)
for cell in range(6) for ctr in [(2**cell) * big]
for (xd, yd) in [(small, small), (small,
-small), (-small,
small), (-small,
-small)]
]
session.add_all(measures)
cells = [
Cell(lat=ctr,
lon=ctr,
cid=cell,
new_measures=4,
total_measures=1,
**keys) for cell in range(6) for ctr in [(2**cell) * big]
]
session.add_all(cells)
session.commit()
result = location_update_cell.delay(min_new=0,
max_new=9999,
batch=len(measures))
self.assertEqual(result.get(), (len(cells), 0))
scan_lacs.delay()
lac = session.query(Cell).filter(Cell.lac == 1,
Cell.cid == CELLID_LAC).first()
# We produced a sequence of 0.02-degree-on-a-side
# cell bounding boxes centered at
# [0, 0.2, 0.4, 0.8, 1.6, 3.2] degrees.
# So the lower-left corner is at (-0.01, -0.01)
# and the upper-right corner is at (3.21, 3.21)
# we should therefore see a LAC centroid at (1.05, 1.05)
# with a range of 339.540m
self.assertEqual(lac.lat, 1.05)
self.assertEqual(lac.lon, 1.05)
self.assertEqual(lac.range, 339540)
def test_scan_areas_race_with_location_update(self):
session = self.session
# First batch of cell observations for CID 1
keys = dict(radio=Radio.cdma, mcc=1, mnc=1, lac=1, cid=1)
cell = Cell(new_measures=4, total_measures=1, **keys)
observations = [
CellObservation(lat=1.0, lon=1.0, **keys),
CellObservation(lat=1.0, lon=1.0, **keys),
CellObservation(lat=1.0, lon=1.0, **keys),
CellObservation(lat=1.0, lon=1.0, **keys),
]
session.add(cell)
session.add_all(observations)
session.commit()
# Periodic location_update_cell runs and updates CID 1
# to have a location, inserts LAC 1 with new_measures=1
# which will be picked up by the next scan_lac.
result = location_update_cell.delay(min_new=1)
self.assertEqual(result.get(), (1, 0))
# Second batch of cell observations for CID 2
keys['cid'] = 2
cell = Cell(new_measures=4, total_measures=1, **keys)
observations = [
CellObservation(lat=1.0, lon=1.0, **keys),
CellObservation(lat=1.0, lon=1.0, **keys),
CellObservation(lat=1.0, lon=1.0, **keys),
CellObservation(lat=1.0, lon=1.0, **keys),
]
session.add(cell)
session.add_all(observations)
session.commit()
# Periodic LAC scan runs, picking up LAC 1; this could
# accidentally pick up CID 2, but it should not since it
# has not had its location updated yet. If there's no
# exception here, CID 2 is properly ignored.
scan_areas.delay()
def test_scan_areas_race_with_location_update(self):
session = self.session
# First batch of cell observations for CID 1
keys = dict(radio=Radio.cdma, mcc=1, mnc=1, lac=1, cid=1)
cell = Cell(new_measures=4, total_measures=1, **keys)
observations = [
CellObservation(lat=1.0, lon=1.0, **keys),
CellObservation(lat=1.0, lon=1.0, **keys),
CellObservation(lat=1.0, lon=1.0, **keys),
CellObservation(lat=1.0, lon=1.0, **keys),
]
session.add(cell)
session.add_all(observations)
session.commit()
# Periodic location_update_cell runs and updates CID 1
# to have a location, inserts LAC 1 with new_measures=1
# which will be picked up by the next scan_lac.
result = location_update_cell.delay(min_new=1)
self.assertEqual(result.get(), (1, 0))
# Second batch of cell observations for CID 2
keys['cid'] = 2
cell = Cell(new_measures=4, total_measures=1, **keys)
observations = [
CellObservation(lat=1.0, lon=1.0, **keys),
CellObservation(lat=1.0, lon=1.0, **keys),
CellObservation(lat=1.0, lon=1.0, **keys),
CellObservation(lat=1.0, lon=1.0, **keys),
]
session.add(cell)
session.add_all(observations)
session.commit()
# Periodic LAC scan runs, picking up LAC 1; this could
# accidentally pick up CID 2, but it should not since it
# has not had its location updated yet. If there's no
# exception here, CID 2 is properly ignored.
scan_areas.delay()
def test_max_min_range_update(self):
session = self.db_master_session
k1 = dict(radio=1, mcc=1, mnc=2, lac=3, cid=4)
data = [
Cell(lat=1.001,
lon=-1.001,
max_lat=1.002,
min_lat=1.0,
max_lon=-1.0,
min_lon=-1.002,
new_measures=2,
total_measures=4,
**k1),
CellMeasure(lat=1.001, lon=-1.003, **k1),
CellMeasure(lat=1.005, lon=-1.007, **k1),
]
session.add_all(data)
session.commit()
result = location_update_cell.delay(min_new=1)
self.assertEqual(result.get(), (1, 0))
cells = session.query(Cell).filter(Cell.cid != CELLID_LAC).all()
self.assertEqual(len(cells), 1)
cell = cells[0]
self.assertEqual(cell.lat, 1.002)
self.assertEqual(cell.max_lat, 1.005)
self.assertEqual(cell.min_lat, 1.0)
self.assertEqual(cell.lon, -1.003)
self.assertEqual(cell.max_lon, -1.0)
self.assertEqual(cell.min_lon, -1.007)
# independent calculation: the cell bounding box is
# (1.000, -1.007) to (1.005, -1.000), with centroid
# at (1.002, -1.003); worst distance from centroid
# to any corner is 556m
self.assertEqual(cell.range, 556)
def test_blacklist_temporary_and_permanent(self):
session = self.db_master_session
# This test simulates a cell that moves once a month, for 2 years.
# The first 2 * PERMANENT_BLACKLIST_THRESHOLD (12) moves should be
# temporary, forgotten after a week; after that it should be
# permanently blacklisted.
now = util.utcnow()
# Station moves between these 4 points, all in the USA:
points = [
# NYC
(40.0, -74.0),
# SF
(37.0, -122.0),
# Seattle
(47.0, -122.0),
# Miami
(25.0, -80.0),
]
N = 4 * PERMANENT_BLACKLIST_THRESHOLD
for month in range(0, N):
days_ago = (N - (month + 1)) * 30
time = now - timedelta(days=days_ago)
time_enc = encode_datetime(time)
measure = dict(id=month, radio=RADIO_TYPE['gsm'],
mcc=USA_MCC, mnc=ATT_MNC, lac=456, cid=123,
time=time_enc,
lat=points[month % 4][0],
lon=points[month % 4][1])
# insert_result is num-accepted-measures, override
# utcnow to set creation date
insert_result = insert_measures_cell.delay(
[measure], utcnow=time_enc)
# update_result is (num-stations, num-moving-stations)
update_result = location_update_cell.delay(min_new=1)
# Assuming PERMANENT_BLACKLIST_THRESHOLD == 6:
#
# 0th insert will create the station
# 1st insert will create first blacklist entry, delete station
# 2nd insert will recreate the station at new position
# 3rd insert will update blacklist, re-delete station
# 4th insert will recreate the station at new position
# 5th insert will update blacklist, re-delete station
# 6th insert will recreate the station at new position
# ...
# 11th insert will make blacklisting permanent, re-delete station
# 12th insert will not recreate station
# 13th insert will not recreate station
# ...
# 23rd insert will not recreate station
bl = session.query(CellBlacklist).all()
if month == 0:
self.assertEqual(len(bl), 0)
else:
self.assertEqual(len(bl), 1)
# force the blacklist back in time to whenever the
# measure was supposedly inserted.
bl = bl[0]
bl.time = time
session.add(bl)
session.commit()
if month < N / 2:
# We still haven't exceeded the threshold, so the
# measurement was admitted.
self.assertEqual(insert_result.get(), 1)
self.assertEqual(session.query(CellMeasure).count(), month + 1)
if month % 2 == 0:
# The station was (re)created.
self.assertEqual(update_result.get(), (1, 0))
# One cell + one cell-LAC record should exist.
self.assertEqual(session.query(Cell).count(), 2)
else:
# The station existed and was seen moving,
# thereby activating the blacklist and deleting the cell.
self.assertEqual(update_result.get(), (1, 1))
self.assertEqual(bl.count, ((month + 1) / 2))
self.assertEqual(session.query(CellBlacklist).count(), 1)
self.assertEqual(session.query(Cell).count(), 0)
# Try adding one more measurement 1 day later
# to be sure it is dropped by the now-active blacklist.
next_day = encode_datetime(time + timedelta(days=1))
measure['time'] = next_day
self.assertEqual(
0, insert_measures_cell.delay([measure],
utcnow=next_day).get())
else:
# Blacklist has exceeded threshold, gone to "permanent" mode,
# so no measures accepted, no stations seen.
self.assertEqual(insert_result.get(), 0)
self.assertEqual(update_result.get(), 0)
def test_blacklist_moving_cells(self):
now = util.utcnow()
long_ago = now - timedelta(days=40)
session = self.db_master_session
k1 = dict(radio=1, mcc=1, mnc=2, lac=3, cid=4)
k2 = dict(radio=1, mcc=1, mnc=2, lac=6, cid=8)
k3 = dict(radio=1, mcc=1, mnc=2, lac=9, cid=12)
k4 = dict(radio=1, mcc=1, mnc=2, lac=12, cid=16)
k5 = dict(radio=1, mcc=1, mnc=2, lac=15, cid=20)
k6 = dict(radio=1, mcc=1, mnc=2, lac=18, cid=24)
keys = set([CellKey(**k) for k in [k1, k2, k3, k4, k5, k6]])
# keys k2, k3 and k4 are expected to be detected as moving
data = [
# a cell with an entry but no prior position
Cell(new_measures=3, total_measures=0, **k1),
CellMeasure(lat=1.001, lon=1.001, **k1),
CellMeasure(lat=1.002, lon=1.005, **k1),
CellMeasure(lat=1.003, lon=1.009, **k1),
# a cell with a prior known position
Cell(lat=2.0, lon=2.0,
new_measures=2, total_measures=1, **k2),
CellMeasure(lat=2.0, lon=2.0, **k2),
CellMeasure(lat=4.0, lon=2.0, **k2),
# a cell with a very different prior position
Cell(lat=1.0, lon=1.0,
new_measures=2, total_measures=1, **k3),
CellMeasure(lat=3.0, lon=3.0, **k3),
CellMeasure(lat=-3.0, lon=3.0, **k3),
# another cell with a prior known position (and negative lat)
Cell(lat=-4.0, lon=4.0,
new_measures=2, total_measures=1, **k4),
CellMeasure(lat=-4.0, lon=4.0, **k4),
CellMeasure(lat=-6.0, lon=4.0, **k4),
# an already blacklisted cell
CellBlacklist(**k5),
CellMeasure(lat=5.0, lon=5.0, **k5),
CellMeasure(lat=8.0, lon=5.0, **k5),
# a cell with an old different record we ignore, position
# estimate has been updated since
Cell(lat=6.0, lon=6.0,
new_measures=2, total_measures=1, **k6),
CellMeasure(lat=6.9, lon=6.9, time=long_ago, **k6),
CellMeasure(lat=6.0, lon=6.0, **k6),
CellMeasure(lat=6.001, lon=6, **k6),
]
session.add_all(data)
session.commit()
result = location_update_cell.delay(min_new=1)
self.assertEqual(result.get(), (5, 3))
black = session.query(CellBlacklist).all()
self.assertEqual(set([to_cellkey(b) for b in black]),
set([CellKey(**k) for k in [k2, k3, k4, k5]]))
measures = session.query(CellMeasure).all()
self.assertEqual(len(measures), 14)
self.assertEqual(set([to_cellkey(m) for m in measures]), keys)
# test duplicate call
result = location_update_cell.delay(min_new=1)
self.assertEqual(result.get(), 0)
self.check_stats(
total=6,
timer=[
# We made duplicate calls
('task.data.location_update_cell', 2),
# One of those would've scheduled a remove_cell task
('task.data.remove_cell', 1)
],
gauge=[
('task.data.location_update_cell.new_measures_1_100', 2),
])
def test_location_update_cell(self):
now = util.utcnow()
before = now - timedelta(hours=1)
schema = ValidCellKeySchema()
obs_factory = CellObservationFactory
invalid_key = dict(lac=schema.fields['lac'].missing,
cid=schema.fields['cid'].missing)
cell1 = CellFactory(new_measures=3, total_measures=5)
lat1, lon1 = (cell1.lat, cell1.lon)
key1 = dict(lac=cell1.lac, cid=cell1.cid)
obs_factory(lat=lat1, lon=lon1, created=now, **key1)
obs_factory(lat=lat1 + 0.004, lon=lon1 + 0.006, created=now, **key1)
obs_factory(lat=lat1 + 0.006, lon=lon1 + 0.009, created=now, **key1)
# The lac, cid are invalid and should be skipped
obs_factory.create_batch(2, created=now, **invalid_key)
cell2 = CellFactory(lat=lat1 + 1.0,
lon=lon1 + 1.0,
new_measures=2,
total_measures=4)
lat2, lon2 = (cell2.lat, cell2.lon)
key2 = dict(lac=cell2.lac, cid=cell2.cid)
# the lat/lon is bogus and mismatches the line above on purpose
# to make sure old observations are skipped
obs_factory(lat=lat2 - 2.0, lon=lon2 - 2.0, created=before, **key2)
obs_factory(lat=lat2 - 2.0, lon=lon2 - 2.0, created=before, **key2)
obs_factory(lat=lat2 + 0.001, lon=lon2 + 0.002, created=now, **key2)
obs_factory(lat=lat2 + 0.003, lon=lon2 + 0.006, created=now, **key2)
cell3 = CellFactory(new_measures=10, total_measures=100000)
lat3, lon3 = (cell3.lat, cell3.lon)
obs_factory.create_batch(10,
lat=lat3 + 1.0,
lon=lon3 + 1.0,
**dict(lac=cell3.lac, cid=cell3.cid))
self.session.commit()
result = location_update_cell.delay(min_new=1)
self.assertEqual(result.get(), (3, 0))
self.check_stats(
total=2,
timer=['task.data.location_update_cell'],
gauge=['task.data.location_update_cell.new_measures_1_100'],
)
cells = self.session.query(Cell).all()
self.assertEqual(len(cells), 3)
self.assertEqual(set([c.new_measures for c in cells]), set([0]))
for cell in cells:
if cell.hashkey() == cell1.hashkey():
self.assertEqual(cell.lat, lat1 + 0.002)
self.assertEqual(cell.lon, lon1 + 0.003)
if cell.hashkey() == cell2.hashkey():
self.assertEqual(cell.lat, lat2 + 0.001)
self.assertEqual(cell.lon, lon2 + 0.002)
if cell.hashkey() == cell3.hashkey():
expected_lat = ((lat3 * 1000) + (lat3 + 1.0) * 10) / 1010
expected_lon = ((lon3 * 1000) + (lon3 + 1.0) * 10) / 1010
self.assertAlmostEqual(cell.lat, expected_lat, 7)
self.assertAlmostEqual(cell.lon, expected_lon, 7)
def test_blacklist_temporary_and_permanent(self):
session = self.db_master_session
# This test simulates a cell that moves once a month, for 2 years.
# The first 2 * PERMANENT_BLACKLIST_THRESHOLD (12) moves should be
# temporary, forgotten after a week; after that it should be
# permanently blacklisted.
now = util.utcnow()
# Station moves between these 4 points, all in the USA:
points = [
# NYC
(40.0, -74.0),
# SF
(37.0, -122.0),
# Seattle
(47.0, -122.0),
# Miami
(25.0, -80.0),
]
N = 4 * PERMANENT_BLACKLIST_THRESHOLD
for month in range(0, N):
days_ago = (N - (month + 1)) * 30
time = now - timedelta(days=days_ago)
time_enc = encode_datetime(time)
measure = dict(radio=RADIO_TYPE['gsm'],
mcc=USA_MCC,
mnc=ATT_MNC,
lac=456,
cid=123,
time=time_enc,
lat=points[month % 4][0],
lon=points[month % 4][1])
# insert_result is num-accepted-measures, override
# utcnow to set creation date
insert_result = insert_measures_cell.delay([measure],
utcnow=time_enc)
# update_result is (num-stations, num-moving-stations)
update_result = location_update_cell.delay(min_new=1)
# Assuming PERMANENT_BLACKLIST_THRESHOLD == 6:
#
# 0th insert will create the station
# 1st insert will create first blacklist entry, delete station
# 2nd insert will recreate the station at new position
# 3rd insert will update blacklist, re-delete station
# 4th insert will recreate the station at new position
# 5th insert will update blacklist, re-delete station
# 6th insert will recreate the station at new position
# ...
# 11th insert will make blacklisting permanent, re-delete station
# 12th insert will not recreate station
# 13th insert will not recreate station
# ...
# 23rd insert will not recreate station
bl = session.query(CellBlacklist).all()
if month == 0:
self.assertEqual(len(bl), 0)
else:
self.assertEqual(len(bl), 1)
# force the blacklist back in time to whenever the
# measure was supposedly inserted.
bl = bl[0]
bl.time = time
session.add(bl)
session.commit()
if month < N / 2:
# We still haven't exceeded the threshold, so the
# measurement was admitted.
self.assertEqual(insert_result.get(), 1)
self.assertEqual(session.query(CellMeasure).count(), month + 1)
if month % 2 == 0:
# The station was (re)created.
self.assertEqual(update_result.get(), (1, 0))
# Rescan lacs to update entries
self.assertEqual(scan_lacs.delay().get(), 1)
# One cell + one cell-LAC record should exist.
self.assertEqual(session.query(Cell).count(), 2)
else:
# The station existed and was seen moving,
# thereby activating the blacklist and deleting the cell.
self.assertEqual(update_result.get(), (1, 1))
# Rescan lacs to delete orphaned lac entry
self.assertEqual(scan_lacs.delay().get(), 1)
self.assertEqual(bl.count, ((month + 1) / 2))
self.assertEqual(session.query(CellBlacklist).count(), 1)
self.assertEqual(session.query(Cell).count(), 0)
# Try adding one more measurement 1 day later
# to be sure it is dropped by the now-active blacklist.
next_day = encode_datetime(time + timedelta(days=1))
measure['time'] = next_day
self.assertEqual(
0,
insert_measures_cell.delay([measure],
utcnow=next_day).get())
else:
# Blacklist has exceeded threshold, gone to "permanent" mode,
# so no measures accepted, no stations seen.
self.assertEqual(insert_result.get(), 0)
self.assertEqual(update_result.get(), 0)
def test_blacklist_moving_cells(self):
now = util.utcnow()
long_ago = now - timedelta(days=40)
session = self.db_master_session
k1 = dict(radio=1, mcc=1, mnc=2, lac=3, cid=4)
k2 = dict(radio=1, mcc=1, mnc=2, lac=6, cid=8)
k3 = dict(radio=1, mcc=1, mnc=2, lac=9, cid=12)
k4 = dict(radio=1, mcc=1, mnc=2, lac=12, cid=16)
k5 = dict(radio=1, mcc=1, mnc=2, lac=15, cid=20)
k6 = dict(radio=1, mcc=1, mnc=2, lac=18, cid=24)
keys = set([CellKey(**k) for k in [k1, k2, k3, k4, k5, k6]])
# keys k2, k3 and k4 are expected to be detected as moving
data = [
# a cell with an entry but no prior position
Cell(new_measures=3, total_measures=0, **k1),
CellMeasure(lat=1.001, lon=1.001, **k1),
CellMeasure(lat=1.002, lon=1.005, **k1),
CellMeasure(lat=1.003, lon=1.009, **k1),
# a cell with a prior known position
Cell(lat=2.0, lon=2.0, new_measures=2, total_measures=1, **k2),
CellMeasure(lat=2.0, lon=2.0, **k2),
CellMeasure(lat=4.0, lon=2.0, **k2),
# a cell with a very different prior position
Cell(lat=1.0, lon=1.0, new_measures=2, total_measures=1, **k3),
CellMeasure(lat=3.0, lon=3.0, **k3),
CellMeasure(lat=-3.0, lon=3.0, **k3),
# another cell with a prior known position (and negative lat)
Cell(lat=-4.0, lon=4.0, new_measures=2, total_measures=1, **k4),
CellMeasure(lat=-4.0, lon=4.0, **k4),
CellMeasure(lat=-6.0, lon=4.0, **k4),
# an already blacklisted cell
CellBlacklist(**k5),
CellMeasure(lat=5.0, lon=5.0, **k5),
CellMeasure(lat=8.0, lon=5.0, **k5),
# a cell with an old different record we ignore, position
# estimate has been updated since
Cell(lat=6.0, lon=6.0, new_measures=2, total_measures=1, **k6),
CellMeasure(lat=6.9, lon=6.9, time=long_ago, **k6),
CellMeasure(lat=6.0, lon=6.0, **k6),
CellMeasure(lat=6.001, lon=6, **k6),
]
session.add_all(data)
session.commit()
result = location_update_cell.delay(min_new=1)
self.assertEqual(result.get(), (5, 3))
black = session.query(CellBlacklist).all()
self.assertEqual(set([to_cellkey(b) for b in black]),
set([CellKey(**k) for k in [k2, k3, k4, k5]]))
measures = session.query(CellMeasure).all()
self.assertEqual(len(measures), 14)
self.assertEqual(set([to_cellkey(m) for m in measures]), keys)
# test duplicate call
result = location_update_cell.delay(min_new=1)
self.assertEqual(result.get(), 0)
self.check_stats(
total=6,
timer=[
# We made duplicate calls
('task.data.location_update_cell', 2),
# One of those would've scheduled a remove_cell task
('task.data.remove_cell', 1)
],
gauge=[
('task.data.location_update_cell.new_measures_1_100', 2),
])
