def add_line_of_cells_and_scan_lac(self):
session = self.db_master_session
big = 1.0
small = big / 10
keys = dict(radio=1, mcc=1, mnc=1, lac=1)
measures = [
CellMeasure(lat=ctr + xd, lon=ctr + yd, cid=cell, **keys)
for cell in range(10)
for ctr in [cell * big]
for (xd, yd) in [(small, small),
(small, -small),
(-small, small),
(-small, -small)]
]
session.add_all(measures)
cells = [
Cell(lat=ctr, lon=ctr, cid=cell,
new_measures=4, total_measures=1, **keys)
for cell in range(10)
for ctr in [cell * big]
]
session.add_all(cells)
session.commit()
result = location_update_cell.delay(min_new=0,
max_new=9999,
batch=len(measures))
self.assertEqual(result.get(), (len(cells), 0))
scan_lacs.delay()
def add_line_of_cells_and_scan_lac(self):
session = self.db_master_session
big = 1.0
small = big / 10
keys = dict(radio=1, mcc=1, mnc=1, lac=1)
measures = [
CellMeasure(lat=ctr + xd, lon=ctr + yd, cid=cell, **keys)
for cell in range(10) for ctr in [cell * big]
for (xd, yd) in [(small, small), (small,
-small), (-small,
small), (-small,
-small)]
]
session.add_all(measures)
cells = [
Cell(lat=ctr,
lon=ctr,
cid=cell,
new_measures=4,
total_measures=1,
**keys) for cell in range(10) for ctr in [cell * big]
]
session.add_all(cells)
session.commit()
result = location_update_cell.delay(min_new=0,
max_new=9999,
batch=len(measures))
self.assertEqual(result.get(), (len(cells), 0))
scan_lacs.delay()
def test_removal_updates_lac(self):
session = self.db_master_session
keys = dict(radio=1, mcc=1, mnc=1, lac=1)
# setup: build LAC as above
self.add_line_of_cells_and_scan_lac()
# confirm we got one
lac = session.query(Cell).filter(
Cell.lac == 1,
Cell.cid == CELLID_LAC).first()
self.assertEqual(lac.lat, 4.5)
self.assertEqual(lac.lon, 4.5)
self.assertEqual(lac.range, 723001)
# Remove cells one by one checking that the LAC
# changes shape along the way.
steps = [
((5.0, 5.0), 644242),
((5.5, 5.5), 565475),
((6.0, 6.0), 486721),
((6.5, 6.5), 408000),
((7.0, 7.0), 329334),
((7.5, 7.5), 250743),
((8.0, 8.0), 172249),
((8.5, 8.5), 93871),
((9.0, 9.0), 15630),
]
for i in range(9):
session.expire(lac)
k = CellKey(cid=i, **keys)
result = remove_cell.delay([k])
self.assertEqual(1, result.get())
result = scan_lacs.delay()
self.assertEqual(1, result.get())
lac = session.query(Cell).filter(
Cell.lac == 1,
Cell.cid == CELLID_LAC).first()
self.assertEqual(lac.lat, steps[i][0][0])
self.assertEqual(lac.lon, steps[i][0][1])
self.assertEqual(lac.range, steps[i][1])
# Remove final cell, check LAC is gone
k = CellKey(cid=9, **keys)
result = remove_cell.delay([k])
self.assertEqual(1, result.get())
result = scan_lacs.delay()
self.assertEqual(0, result.get())
lac = session.query(Cell).filter(
Cell.lac == 1,
Cell.cid == CELLID_LAC).first()
self.assertEqual(lac, None)
def test_removal_updates_lac(self):
session = self.db_master_session
keys = dict(radio=1, mcc=1, mnc=1, lac=1)
# setup: build LAC as above
self.add_line_of_cells_and_scan_lac()
# confirm we got one
lac = session.query(Cell).filter(Cell.lac == 1,
Cell.cid == CELLID_LAC).first()
self.assertEqual(lac.lat, 4.5)
self.assertEqual(lac.lon, 4.5)
self.assertEqual(lac.range, 723001)
# Remove cells one by one checking that the LAC
# changes shape along the way.
steps = [
((5.0, 5.0), 644242),
((5.5, 5.5), 565475),
((6.0, 6.0), 486721),
((6.5, 6.5), 408000),
((7.0, 7.0), 329334),
((7.5, 7.5), 250743),
((8.0, 8.0), 172249),
((8.5, 8.5), 93871),
((9.0, 9.0), 15630),
]
for i in range(9):
session.expire(lac)
k = CellKey(cid=i, **keys)
result = remove_cell.delay([k])
self.assertEqual(1, result.get())
result = scan_lacs.delay()
self.assertEqual(1, result.get())
lac = session.query(Cell).filter(Cell.lac == 1,
Cell.cid == CELLID_LAC).first()
self.assertEqual(lac.lat, steps[i][0][0])
self.assertEqual(lac.lon, steps[i][0][1])
self.assertEqual(lac.range, steps[i][1])
# Remove final cell, check LAC is gone
k = CellKey(cid=9, **keys)
result = remove_cell.delay([k])
self.assertEqual(1, result.get())
result = scan_lacs.delay()
self.assertEqual(1, result.get())
lac = session.query(Cell).filter(Cell.lac == 1,
Cell.cid == CELLID_LAC).first()
self.assertEqual(lac, None)
def test_scan_lacs_asymmetric(self):
session = self.db_master_session
big = 0.1
small = big / 10
keys = dict(radio=1, mcc=1, mnc=1, lac=1)
measures = [
CellMeasure(lat=ctr + xd, lon=ctr + yd, cid=cell, **keys)
for cell in range(6)
for ctr in [(2 ** cell) * big]
for (xd, yd) in [(small, small),
(small, -small),
(-small, small),
(-small, -small)]
]
session.add_all(measures)
cells = [
Cell(lat=ctr, lon=ctr, cid=cell,
new_measures=4, total_measures=1, **keys)
for cell in range(6)
for ctr in [(2 ** cell) * big]
]
session.add_all(cells)
session.commit()
result = location_update_cell.delay(min_new=0,
max_new=9999,
batch=len(measures))
self.assertEqual(result.get(), (len(cells), 0))
scan_lacs.delay()
lac = session.query(Cell).filter(
Cell.lac == 1,
Cell.cid == CELLID_LAC).first()
# We produced a sequence of 0.02-degree-on-a-side
# cell bounding boxes centered at
# [0, 0.2, 0.4, 0.8, 1.6, 3.2] degrees.
# So the lower-left corner is at (-0.01, -0.01)
# and the upper-right corner is at (3.21, 3.21)
# we should therefore see a LAC centroid at (1.05, 1.05)
# with a range of 339.540m
self.assertEqual(lac.lat, 1.05)
self.assertEqual(lac.lon, 1.05)
self.assertEqual(lac.range, 339540)
def test_scan_lacs_asymmetric(self):
session = self.db_master_session
big = 0.1
small = big / 10
keys = dict(radio=1, mcc=1, mnc=1, lac=1)
measures = [
CellMeasure(lat=ctr + xd, lon=ctr + yd, cid=cell, **keys)
for cell in range(6) for ctr in [(2**cell) * big]
for (xd, yd) in [(small, small), (small,
-small), (-small,
small), (-small,
-small)]
]
session.add_all(measures)
cells = [
Cell(lat=ctr,
lon=ctr,
cid=cell,
new_measures=4,
total_measures=1,
**keys) for cell in range(6) for ctr in [(2**cell) * big]
]
session.add_all(cells)
session.commit()
result = location_update_cell.delay(min_new=0,
max_new=9999,
batch=len(measures))
self.assertEqual(result.get(), (len(cells), 0))
scan_lacs.delay()
lac = session.query(Cell).filter(Cell.lac == 1,
Cell.cid == CELLID_LAC).first()
# We produced a sequence of 0.02-degree-on-a-side
# cell bounding boxes centered at
# [0, 0.2, 0.4, 0.8, 1.6, 3.2] degrees.
# So the lower-left corner is at (-0.01, -0.01)
# and the upper-right corner is at (3.21, 3.21)
# we should therefore see a LAC centroid at (1.05, 1.05)
# with a range of 339.540m
self.assertEqual(lac.lat, 1.05)
self.assertEqual(lac.lon, 1.05)
self.assertEqual(lac.range, 339540)
def test_scan_lacs_race_with_location_update(self):
session = self.db_master_session
# First batch of cell measurements for CID 1
keys = dict(radio=1, mcc=1, mnc=1, lac=1, cid=1)
cell = Cell(new_measures=4, total_measures=1, **keys)
measures = [
CellMeasure(lat=1.0, lon=1.0, **keys),
CellMeasure(lat=1.0, lon=1.0, **keys),
CellMeasure(lat=1.0, lon=1.0, **keys),
CellMeasure(lat=1.0, lon=1.0, **keys),
]
session.add(cell)
session.add_all(measures)
session.commit()
# Periodic location_update_cell runs and updates CID 1
# to have a location, inserts LAC 1 with new_measures=1
# which will be picked up by the next scan_lac.
result = location_update_cell.delay(min_new=1)
self.assertEqual(result.get(), (1, 0))
# Second batch of cell measurements for CID 2
keys['cid'] = 2
cell = Cell(new_measures=4, total_measures=1, **keys)
measures = [
CellMeasure(lat=1.0, lon=1.0, **keys),
CellMeasure(lat=1.0, lon=1.0, **keys),
CellMeasure(lat=1.0, lon=1.0, **keys),
CellMeasure(lat=1.0, lon=1.0, **keys),
]
session.add(cell)
session.add_all(measures)
session.commit()
# Periodic LAC scan runs, picking up LAC 1; this could
# accidentally pick up CID 2, but it should not since it
# has not had its location updated yet. If there's no
# exception here, CID 2 is properly ignored.
scan_lacs.delay()
def test_scan_lacs_race_with_location_update(self):
session = self.db_master_session
# First batch of cell measurements for CID 1
keys = dict(radio=1, mcc=1, mnc=1, lac=1, cid=1)
cell = Cell(new_measures=4, total_measures=1, **keys)
measures = [
CellMeasure(lat=1.0, lon=1.0, **keys),
CellMeasure(lat=1.0, lon=1.0, **keys),
CellMeasure(lat=1.0, lon=1.0, **keys),
CellMeasure(lat=1.0, lon=1.0, **keys),
]
session.add(cell)
session.add_all(measures)
session.commit()
# Periodic location_update_cell runs and updates CID 1
# to have a location, inserts LAC 1 with new_measures=1
# which will be picked up by the next scan_lac.
result = location_update_cell.delay(min_new=1)
self.assertEqual(result.get(), (1, 0))
# Second batch of cell measurements for CID 2
keys['cid'] = 2
cell = Cell(new_measures=4, total_measures=1, **keys)
measures = [
CellMeasure(lat=1.0, lon=1.0, **keys),
CellMeasure(lat=1.0, lon=1.0, **keys),
CellMeasure(lat=1.0, lon=1.0, **keys),
CellMeasure(lat=1.0, lon=1.0, **keys),
]
session.add(cell)
session.add_all(measures)
session.commit()
# Periodic LAC scan runs, picking up LAC 1; this could
# accidentally pick up CID 2, but it should not since it
# has not had its location updated yet. If there's no
# exception here, CID 2 is properly ignored.
scan_lacs.delay()
def test_scan_lacs_remove(self):
session = self.db_master_session
redis_client = self.redis_client
# create an orphaned lac entry
key = dict(radio=1, mcc=1, mnc=1, lac=1, cid=CELLID_LAC)
session.add(Cell(**key))
session.flush()
enqueue_lacs(session, redis_client, [CellKey(**key)])
# after scanning the orphaned record gets removed
self.assertEqual(scan_lacs.delay().get(), 1)
lacs = session.query(Cell).filter(Cell.cid == CELLID_LAC).all()
self.assertEqual(lacs, [])
def test_scan_lacs_remove(self):
session = self.db_master_session
redis_client = self.redis_client
# create an orphaned lac entry
key = dict(radio=1, mcc=1, mnc=1, lac=1, cid=CELLID_LAC)
session.add(Cell(**key))
session.flush()
enqueue_lacs(session, redis_client, [CellKey(**key)])
# after scanning the orphaned record gets removed
self.assertEqual(scan_lacs.delay().get(), 1)
lacs = session.query(Cell).filter(Cell.cid == CELLID_LAC).all()
self.assertEqual(lacs, [])
def test_scan_lacs_remove(self):
session = self.db_master_session
redis_client = self.redis_client
# create an orphaned lac entry
key = dict(radio=1, mcc=1, mnc=1, lac=1)
session.add(CellArea(**key))
session.flush()
enqueue_lacs(session, redis_client, [CellArea.to_hashkey(key)],
UPDATE_KEY['cell_lac'])
# after scanning the orphaned record gets removed
self.assertEqual(scan_lacs.delay().get(), 1)
lacs = session.query(CellArea).all()
self.assertEqual(lacs, [])
def test_scan_lacs_empty(self):
# test tasks with an empty queue
self.assertEqual(scan_lacs.delay().get(), 0)
self.check_expected_heka_messages(sentry=[('msg', RAVEN_ERROR, 0)])
def test_blacklist_temporary_and_permanent(self):
session = self.db_master_session
# This test simulates a cell that moves once a month, for 2 years.
# The first 2 * PERMANENT_BLACKLIST_THRESHOLD (12) moves should be
# temporary, forgotten after a week; after that it should be
# permanently blacklisted.
now = util.utcnow()
# Station moves between these 4 points, all in the USA:
points = [
# NYC
(40.0, -74.0),
# SF
(37.0, -122.0),
# Seattle
(47.0, -122.0),
# Miami
(25.0, -80.0),
]
N = 4 * PERMANENT_BLACKLIST_THRESHOLD
for month in range(0, N):
days_ago = (N - (month + 1)) * 30
time = now - timedelta(days=days_ago)
time_enc = encode_datetime(time)
measure = dict(radio=RADIO_TYPE['gsm'],
mcc=USA_MCC,
mnc=ATT_MNC,
lac=456,
cid=123,
time=time_enc,
lat=points[month % 4][0],
lon=points[month % 4][1])
# insert_result is num-accepted-measures, override
# utcnow to set creation date
insert_result = insert_measures_cell.delay([measure],
utcnow=time_enc)
# update_result is (num-stations, num-moving-stations)
update_result = location_update_cell.delay(min_new=1)
# Assuming PERMANENT_BLACKLIST_THRESHOLD == 6:
#
# 0th insert will create the station
# 1st insert will create first blacklist entry, delete station
# 2nd insert will recreate the station at new position
# 3rd insert will update blacklist, re-delete station
# 4th insert will recreate the station at new position
# 5th insert will update blacklist, re-delete station
# 6th insert will recreate the station at new position
# ...
# 11th insert will make blacklisting permanent, re-delete station
# 12th insert will not recreate station
# 13th insert will not recreate station
# ...
# 23rd insert will not recreate station
bl = session.query(CellBlacklist).all()
if month == 0:
self.assertEqual(len(bl), 0)
else:
self.assertEqual(len(bl), 1)
# force the blacklist back in time to whenever the
# measure was supposedly inserted.
bl = bl[0]
bl.time = time
session.add(bl)
session.commit()
if month < N / 2:
# We still haven't exceeded the threshold, so the
# measurement was admitted.
self.assertEqual(insert_result.get(), 1)
self.assertEqual(session.query(CellMeasure).count(), month + 1)
if month % 2 == 0:
# The station was (re)created.
self.assertEqual(update_result.get(), (1, 0))
# Rescan lacs to update entries
self.assertEqual(scan_lacs.delay().get(), 1)
# One cell + one cell-LAC record should exist.
self.assertEqual(session.query(Cell).count(), 2)
else:
# The station existed and was seen moving,
# thereby activating the blacklist and deleting the cell.
self.assertEqual(update_result.get(), (1, 1))
# Rescan lacs to delete orphaned lac entry
self.assertEqual(scan_lacs.delay().get(), 1)
self.assertEqual(bl.count, ((month + 1) / 2))
self.assertEqual(session.query(CellBlacklist).count(), 1)
self.assertEqual(session.query(Cell).count(), 0)
# Try adding one more measurement 1 day later
# to be sure it is dropped by the now-active blacklist.
next_day = encode_datetime(time + timedelta(days=1))
measure['time'] = next_day
self.assertEqual(
0,
insert_measures_cell.delay([measure],
utcnow=next_day).get())
else:
# Blacklist has exceeded threshold, gone to "permanent" mode,
# so no measures accepted, no stations seen.
self.assertEqual(insert_result.get(), 0)
self.assertEqual(update_result.get(), 0)
def test_scan_lacs_empty(self):
# test tasks with an empty queue
self.assertEqual(scan_lacs.delay().get(), 0)
self.check_expected_heka_messages(
sentry=[('msg', RAVEN_ERROR, 0)]
)
def test_blacklist_temporary_and_permanent(self):
session = self.db_master_session
# This test simulates a cell that moves once a month, for 2 years.
# The first 2 * PERMANENT_BLACKLIST_THRESHOLD (12) moves should be
# temporary, forgotten after a week; after that it should be
# permanently blacklisted.
now = util.utcnow()
# Station moves between these 4 points, all in the USA:
points = [
# NYC
(40.0, -74.0),
# SF
(37.0, -122.0),
# Seattle
(47.0, -122.0),
# Miami
(25.0, -80.0),
]
N = 4 * PERMANENT_BLACKLIST_THRESHOLD
for month in range(0, N):
days_ago = (N - (month + 1)) * 30
time = now - timedelta(days=days_ago)
time_enc = encode_datetime(time)
measure = dict(radio=RADIO_TYPE['gsm'],
mcc=USA_MCC, mnc=ATT_MNC, lac=456, cid=123,
time=time_enc,
lat=points[month % 4][0],
lon=points[month % 4][1])
# insert_result is num-accepted-measures, override
# utcnow to set creation date
insert_result = insert_measures_cell.delay(
[measure], utcnow=time_enc)
# update_result is (num-stations, num-moving-stations)
update_result = location_update_cell.delay(min_new=1)
# Assuming PERMANENT_BLACKLIST_THRESHOLD == 6:
#
# 0th insert will create the station
# 1st insert will create first blacklist entry, delete station
# 2nd insert will recreate the station at new position
# 3rd insert will update blacklist, re-delete station
# 4th insert will recreate the station at new position
# 5th insert will update blacklist, re-delete station
# 6th insert will recreate the station at new position
# ...
# 11th insert will make blacklisting permanent, re-delete station
# 12th insert will not recreate station
# 13th insert will not recreate station
# ...
# 23rd insert will not recreate station
bl = session.query(CellBlacklist).all()
if month == 0:
self.assertEqual(len(bl), 0)
else:
self.assertEqual(len(bl), 1)
# force the blacklist back in time to whenever the
# measure was supposedly inserted.
bl = bl[0]
bl.time = time
session.add(bl)
session.commit()
if month < N / 2:
# We still haven't exceeded the threshold, so the
# measurement was admitted.
self.assertEqual(insert_result.get(), 1)
self.assertEqual(session.query(CellMeasure).count(), month + 1)
if month % 2 == 0:
# The station was (re)created.
self.assertEqual(update_result.get(), (1, 0))
# Rescan lacs to update entries
self.assertEqual(scan_lacs.delay().get(), 1)
# One cell + one cell-LAC record should exist.
self.assertEqual(session.query(Cell).count(), 2)
else:
# The station existed and was seen moving,
# thereby activating the blacklist and deleting the cell.
self.assertEqual(update_result.get(), (1, 1))
# Rescan lacs to delete orphaned lac entry
self.assertEqual(scan_lacs.delay().get(), 1)
self.assertEqual(bl.count, ((month + 1) / 2))
self.assertEqual(session.query(CellBlacklist).count(), 1)
self.assertEqual(session.query(Cell).count(), 0)
# Try adding one more measurement 1 day later
# to be sure it is dropped by the now-active blacklist.
next_day = encode_datetime(time + timedelta(days=1))
measure['time'] = next_day
self.assertEqual(
0, insert_measures_cell.delay([measure],
utcnow=next_day).get())
else:
# Blacklist has exceeded threshold, gone to "permanent" mode,
# so no measures accepted, no stations seen.
self.assertEqual(insert_result.get(), 0)
self.assertEqual(update_result.get(), 0)