def add_line_of_cells_and_scan_lac(self):
session = self.session
big = 1.0
small = big / 10
keys = dict(radio=Radio.cdma, mcc=1, mnc=1, lac=1)
observations = [
CellObservation(lat=ctr + xd, lon=ctr + yd, cid=cell, **keys)
for cell in range(10)
for ctr in [cell * big]
for (xd, yd) in [(small, small),
(small, -small),
(-small, small),
(-small, -small)]
]
session.add_all(observations)
cells = [
Cell(lat=ctr, lon=ctr, cid=cell,
new_measures=4, total_measures=1, **keys)
for cell in range(10)
for ctr in [cell * big]
]
session.add_all(cells)
session.commit()
result = location_update_cell.delay(min_new=0,
max_new=9999,
batch=len(observations))
self.assertEqual(result.get(), (len(cells), 0))
scan_areas.delay()
def add_line_of_cells_and_scan_lac(self):
session = self.session
big = 1.0
small = big / 10
keys = dict(radio=Radio.cdma, mcc=1, mnc=1, lac=1)
observations = [
CellObservation(lat=ctr + xd, lon=ctr + yd, cid=cell, **keys)
for cell in range(10)
for ctr in [cell * big]
for (xd, yd) in [(small, small),
(small, -small),
(-small, small),
(-small, -small)]
]
session.add_all(observations)
cells = [
Cell(lat=ctr, lon=ctr, cid=cell,
new_measures=4, total_measures=1, **keys)
for cell in range(10)
for ctr in [cell * big]
]
session.add_all(cells)
session.commit()
result = location_update_cell.delay(min_new=0,
max_new=9999,
batch=len(observations))
self.assertEqual(result.get(), (len(cells), 0))
scan_areas.delay()
def test_removal_updates_lac(self):
session = self.session
keys = dict(radio=Radio.cdma, mcc=1, mnc=1, lac=1)
# setup: build LAC as above
self.add_line_of_cells_and_scan_lac()
# confirm we got one
lac = session.query(CellArea).filter(CellArea.lac == 1).first()
self.assertEqual(lac.lat, 4.5)
self.assertEqual(lac.lon, 4.5)
self.assertEqual(lac.range, 723001)
# Remove cells one by one checking that the LAC
# changes shape along the way.
steps = [
((5.0, 5.0), 644242),
((5.5, 5.5), 565475),
((6.0, 6.0), 486721),
((6.5, 6.5), 408000),
((7.0, 7.0), 329334),
((7.5, 7.5), 250743),
((8.0, 8.0), 172249),
((8.5, 8.5), 93871),
((9.0, 9.0), 15630),
]
for i in range(9):
session.expire(lac)
k = Cell.to_hashkey(cid=i, **keys)
result = remove_cell.delay([k])
self.assertEqual(1, result.get())
result = scan_areas.delay()
self.assertEqual(1, result.get())
lac = session.query(CellArea).filter(CellArea.lac == 1).first()
self.assertEqual(lac.lat, steps[i][0][0])
self.assertEqual(lac.lon, steps[i][0][1])
self.assertEqual(lac.range, steps[i][1])
# Remove final cell, check LAC is gone
k = Cell.to_hashkey(cid=9, **keys)
result = remove_cell.delay([k])
self.assertEqual(1, result.get())
result = scan_areas.delay()
self.assertEqual(1, result.get())
lac = session.query(CellArea).filter(CellArea.lac == 1).first()
self.assertEqual(lac, None)
def test_removal_updates_lac(self):
session = self.session
keys = dict(radio=Radio.cdma, mcc=1, mnc=1, lac=1)
# setup: build LAC as above
self.add_line_of_cells_and_scan_lac()
# confirm we got one
lac = session.query(CellArea).filter(CellArea.lac == 1).first()
self.assertEqual(lac.lat, 4.5)
self.assertEqual(lac.lon, 4.5)
self.assertEqual(lac.range, 723001)
# Remove cells one by one checking that the LAC
# changes shape along the way.
steps = [
((5.0, 5.0), 644242),
((5.5, 5.5), 565475),
((6.0, 6.0), 486721),
((6.5, 6.5), 408000),
((7.0, 7.0), 329334),
((7.5, 7.5), 250743),
((8.0, 8.0), 172249),
((8.5, 8.5), 93871),
((9.0, 9.0), 15630),
]
for i in range(9):
session.expire(lac)
k = Cell.to_hashkey(cid=i, **keys)
result = remove_cell.delay([k])
self.assertEqual(1, result.get())
result = scan_areas.delay()
self.assertEqual(1, result.get())
lac = session.query(CellArea).filter(CellArea.lac == 1).first()
self.assertEqual(lac.lat, steps[i][0][0])
self.assertEqual(lac.lon, steps[i][0][1])
self.assertEqual(lac.range, steps[i][1])
# Remove final cell, check LAC is gone
k = Cell.to_hashkey(cid=9, **keys)
result = remove_cell.delay([k])
self.assertEqual(1, result.get())
result = scan_areas.delay()
self.assertEqual(1, result.get())
lac = session.query(CellArea).filter(CellArea.lac == 1).first()
self.assertEqual(lac, None)
def test_remove(self):
area = CellAreaFactory()
self.session.flush()
self.area_queue.enqueue([area.hashkey()])
self.assertEqual(scan_areas.delay().get(), 1)
self.assertEqual(self.session.query(CellArea).count(), 0)
def test_scan_areas_remove(self):
# create an orphaned lac entry
area = CellAreaFactory()
self.session.flush()
self.area_queue.enqueue([area.hashkey()])
# after scanning the orphaned record gets removed
self.assertEqual(scan_areas.delay().get(), 1)
areas = self.session.query(CellArea).all()
self.assertEqual(areas, [])
def test_scan_areas_asymmetric(self):
session = self.session
big = 0.1
small = big / 10
keys = dict(radio=Radio.cdma, mcc=1, mnc=1, lac=1)
observations = [
CellObservation(lat=ctr + xd, lon=ctr + yd, cid=cell, **keys)
for cell in range(6)
for ctr in [(2 ** cell) * big]
for (xd, yd) in [(small, small),
(small, -small),
(-small, small),
(-small, -small)]
]
session.add_all(observations)
cells = [
Cell(lat=ctr, lon=ctr, cid=cell,
new_measures=4, total_measures=1, **keys)
for cell in range(6)
for ctr in [(2 ** cell) * big]
]
session.add_all(cells)
session.commit()
result = location_update_cell.delay(min_new=0,
max_new=9999,
batch=len(observations))
self.assertEqual(result.get(), (len(cells), 0))
scan_areas.delay()
lac = session.query(CellArea).filter(CellArea.lac == 1).first()
# We produced a sequence of 0.02-degree-on-a-side
# cell bounding boxes centered at
# [0, 0.2, 0.4, 0.8, 1.6, 3.2] degrees.
# So the lower-left corner is at (-0.01, -0.01)
# and the upper-right corner is at (3.21, 3.21)
# we should therefore see a LAC centroid at (1.05, 1.05)
# with a range of 339.540m
self.assertEqual(lac.lat, 1.05)
self.assertEqual(lac.lon, 1.05)
self.assertEqual(lac.range, 339540)
def test_scan_areas_asymmetric(self):
session = self.session
big = 0.1
small = big / 10
keys = dict(radio=Radio.cdma, mcc=1, mnc=1, lac=1)
observations = [
CellObservation(lat=ctr + xd, lon=ctr + yd, cid=cell, **keys)
for cell in range(6)
for ctr in [(2 ** cell) * big]
for (xd, yd) in [(small, small),
(small, -small),
(-small, small),
(-small, -small)]
]
session.add_all(observations)
cells = [
Cell(lat=ctr, lon=ctr, cid=cell,
new_measures=4, total_measures=1, **keys)
for cell in range(6)
for ctr in [(2 ** cell) * big]
]
session.add_all(cells)
session.commit()
result = location_update_cell.delay(min_new=0,
max_new=9999,
batch=len(observations))
self.assertEqual(result.get(), (len(cells), 0))
scan_areas.delay()
lac = session.query(CellArea).filter(CellArea.lac == 1).first()
# We produced a sequence of 0.02-degree-on-a-side
# cell bounding boxes centered at
# [0, 0.2, 0.4, 0.8, 1.6, 3.2] degrees.
# So the lower-left corner is at (-0.01, -0.01)
# and the upper-right corner is at (3.21, 3.21)
# we should therefore see a LAC centroid at (1.05, 1.05)
# with a range of 339.540m
self.assertEqual(lac.lat, 1.05)
self.assertEqual(lac.lon, 1.05)
self.assertEqual(lac.range, 339540)
def test_scan_areas_race_with_location_update(self):
session = self.session
# First batch of cell observations for CID 1
keys = dict(radio=Radio.cdma, mcc=1, mnc=1, lac=1, cid=1)
cell = Cell(new_measures=4, total_measures=1, **keys)
observations = [
CellObservation(lat=1.0, lon=1.0, **keys),
CellObservation(lat=1.0, lon=1.0, **keys),
CellObservation(lat=1.0, lon=1.0, **keys),
CellObservation(lat=1.0, lon=1.0, **keys),
]
session.add(cell)
session.add_all(observations)
session.commit()
# Periodic location_update_cell runs and updates CID 1
# to have a location, inserts LAC 1 with new_measures=1
# which will be picked up by the next scan_lac.
result = location_update_cell.delay(min_new=1)
self.assertEqual(result.get(), (1, 0))
# Second batch of cell observations for CID 2
keys['cid'] = 2
cell = Cell(new_measures=4, total_measures=1, **keys)
observations = [
CellObservation(lat=1.0, lon=1.0, **keys),
CellObservation(lat=1.0, lon=1.0, **keys),
CellObservation(lat=1.0, lon=1.0, **keys),
CellObservation(lat=1.0, lon=1.0, **keys),
]
session.add(cell)
session.add_all(observations)
session.commit()
# Periodic LAC scan runs, picking up LAC 1; this could
# accidentally pick up CID 2, but it should not since it
# has not had its location updated yet. If there's no
# exception here, CID 2 is properly ignored.
scan_areas.delay()
def test_scan_areas_race_with_location_update(self):
session = self.session
# First batch of cell observations for CID 1
keys = dict(radio=Radio.cdma, mcc=1, mnc=1, lac=1, cid=1)
cell = Cell(new_measures=4, total_measures=1, **keys)
observations = [
CellObservation(lat=1.0, lon=1.0, **keys),
CellObservation(lat=1.0, lon=1.0, **keys),
CellObservation(lat=1.0, lon=1.0, **keys),
CellObservation(lat=1.0, lon=1.0, **keys),
]
session.add(cell)
session.add_all(observations)
session.commit()
# Periodic location_update_cell runs and updates CID 1
# to have a location, inserts LAC 1 with new_measures=1
# which will be picked up by the next scan_lac.
result = location_update_cell.delay(min_new=1)
self.assertEqual(result.get(), (1, 0))
# Second batch of cell observations for CID 2
keys['cid'] = 2
cell = Cell(new_measures=4, total_measures=1, **keys)
observations = [
CellObservation(lat=1.0, lon=1.0, **keys),
CellObservation(lat=1.0, lon=1.0, **keys),
CellObservation(lat=1.0, lon=1.0, **keys),
CellObservation(lat=1.0, lon=1.0, **keys),
]
session.add(cell)
session.add_all(observations)
session.commit()
# Periodic LAC scan runs, picking up LAC 1; this could
# accidentally pick up CID 2, but it should not since it
# has not had its location updated yet. If there's no
# exception here, CID 2 is properly ignored.
scan_areas.delay()
def test_new(self):
cell = CellFactory()
self.session.flush()
area_key = CellArea.to_hashkey(cell)
self.area_queue.enqueue([area_key])
self.assertEqual(scan_areas.delay().get(), 1)
area = self.session.query(CellArea).one()
self.assertAlmostEqual(area.lat, cell.lat)
self.assertAlmostEqual(area.lon, cell.lon)
self.assertEqual(area.range, 0)
self.assertEqual(area.num_cells, 1)
self.assertEqual(area.avg_cell_range, cell.range)
def test_scan_areas_remove(self):
session = self.session
redis_client = self.redis_client
# create an orphaned lac entry
area = CellAreaFactory()
session.flush()
enqueue_areas(session, redis_client,
[area.hashkey()], UPDATE_KEY['cell_lac'])
# after scanning the orphaned record gets removed
self.assertEqual(scan_areas.delay().get(), 1)
areas = session.query(CellArea).all()
self.assertEqual(areas, [])
def test_scan_areas_remove(self):
session = self.session
redis_client = self.redis_client
# create an orphaned lac entry
area = CellAreaFactory()
session.flush()
redis_key = self.celery_app.data_queues['cell_area_update']
enqueue_areas(session, redis_client,
[area.hashkey()], redis_key)
# after scanning the orphaned record gets removed
self.assertEqual(scan_areas.delay().get(), 1)
areas = session.query(CellArea).all()
self.assertEqual(areas, [])
def test_update(self):
area = CellAreaFactory(num_cells=2, range=500, avg_cell_range=100)
area_key = area.hashkey()
cell = CellFactory(
lat=area.lat, lon=area.lon, range=200, **area_key.__dict__)
self.session.commit()
self.area_queue.enqueue([area_key])
self.assertEqual(scan_areas.delay().get(), 1)
area = self.session.query(CellArea).one()
self.assertAlmostEqual(area.lat, cell.lat)
self.assertAlmostEqual(area.lon, cell.lon)
self.assertEqual(area.range, 0)
self.assertEqual(area.num_cells, 1)
self.assertEqual(area.avg_cell_range, 200)
def test_update_incomplete_cell(self):
area = CellAreaFactory(range=500)
area_key = area.hashkey()
cell = CellFactory(
lat=area.lat + 0.0002, lon=area.lon, **area_key.__dict__)
CellFactory(lat=None, lon=None, **area_key.__dict__)
CellFactory(lat=area.lat, lon=area.lon,
max_lat=None, min_lon=None, **area_key.__dict__)
self.session.commit()
self.area_queue.enqueue([area_key])
self.assertEqual(scan_areas.delay().get(), 1)
area = self.session.query(CellArea).one()
self.assertAlmostEqual(area.lat, cell.lat - 0.0001)
self.assertAlmostEqual(area.lon, cell.lon)
self.assertEqual(area.num_cells, 2)
def test_scan_areas_empty(self):
# test tasks with an empty queue
self.assertEqual(scan_areas.delay().get(), 0)
def test_scan_areas_empty(self):
# test tasks with an empty queue
self.assertEqual(scan_areas.delay().get(), 0)
self.check_raven(total=0)
def test_blacklist_temporary_and_permanent(self):
session = self.session
# This test simulates a cell that moves once a month, for 2 years.
# The first 2 * PERMANENT_BLACKLIST_THRESHOLD (12) moves should be
# temporary, forgotten after a week; after that it should be
# permanently blacklisted.
now = util.utcnow()
# Station moves between these 4 points, all in the USA:
points = [
# NYC
(40.0, -74.0),
# SF
(37.0, -122.0),
# Seattle
(47.0, -122.0),
# Miami
(25.0, -80.0),
]
N = 4 * PERMANENT_BLACKLIST_THRESHOLD
for month in range(0, N):
days_ago = (N - (month + 1)) * 30
time = now - timedelta(days=days_ago)
obs = dict(radio=int(Radio.gsm),
mcc=USA_MCC,
mnc=ATT_MNC,
lac=456,
cid=123,
time=time,
lat=points[month % 4][0],
lon=points[month % 4][1])
# insert_result is num-accepted-observations, override
# utcnow to set creation date
insert_result = insert_measures_cell.delay([obs], utcnow=time)
# update_result is (num-stations, num-moving-stations)
update_result = location_update_cell.delay(min_new=1)
# Assuming PERMANENT_BLACKLIST_THRESHOLD == 6:
#
# 0th insert will create the station
# 1st insert will create first blacklist entry, delete station
# 2nd insert will recreate the station at new position
# 3rd insert will update blacklist, re-delete station
# 4th insert will recreate the station at new position
# 5th insert will update blacklist, re-delete station
# 6th insert will recreate the station at new position
# ...
# 11th insert will make blacklisting permanent, re-delete station
# 12th insert will not recreate station
# 13th insert will not recreate station
# ...
# 23rd insert will not recreate station
bl = session.query(CellBlacklist).all()
if month == 0:
self.assertEqual(len(bl), 0)
else:
self.assertEqual(len(bl), 1)
# force the blacklist back in time to whenever the
# observation was supposedly inserted.
bl = bl[0]
bl.time = time
session.add(bl)
session.commit()
if month < N / 2:
# We still haven't exceeded the threshold, so the
# observation was admitted.
self.assertEqual(insert_result.get(), 1)
self.assertEqual(
session.query(CellObservation).count(), month + 1)
if month % 2 == 0:
# The station was (re)created.
self.assertEqual(update_result.get(), (1, 0))
# Rescan lacs to update entries
self.assertEqual(scan_areas.delay().get(), 1)
# One cell + one cell-LAC record should exist.
self.assertEqual(session.query(Cell).count(), 1)
self.assertEqual(session.query(CellArea).count(), 1)
else:
# The station existed and was seen moving,
# thereby activating the blacklist and deleting the cell.
self.assertEqual(update_result.get(), (1, 1))
# Rescan lacs to delete orphaned lac entry
self.assertEqual(scan_areas.delay().get(), 1)
self.assertEqual(bl.count, ((month + 1) / 2))
self.assertEqual(session.query(CellBlacklist).count(), 1)
self.assertEqual(session.query(Cell).count(), 0)
# Try adding one more observation 1 day later
# to be sure it is dropped by the now-active blacklist.
next_day = time + timedelta(days=1)
obs['time'] = next_day
self.assertEqual(
0,
insert_measures_cell.delay([obs],
utcnow=next_day).get())
else:
# Blacklist has exceeded threshold, gone to "permanent" mode,
# so no observation accepted, no stations seen.
self.assertEqual(insert_result.get(), 0)
self.assertEqual(update_result.get(), (0, 0))
def test_scan_areas_empty(self):
# test tasks with an empty queue
self.assertEqual(scan_areas.delay().get(), 0)
self.check_raven(total=0)
def test_blacklist_temporary_and_permanent(self):
# This test simulates a cell that moves once a month, for 2 years.
# The first 2 * PERMANENT_BLACKLIST_THRESHOLD (12) moves should be
# temporary, forgotten after a week; after that it should be
# permanently blacklisted.
now = util.utcnow()
# Station moves between these 4 points, all in the USA:
points = [
(40.0, -74.0), # NYC
(37.0, -122.0), # SF
(47.0, -122.0), # Seattle
(25.0, -80.0), # Miami
]
N = 4 * PERMANENT_BLACKLIST_THRESHOLD
for month in range(0, N):
days_ago = (N - (month + 1)) * 30
time = now - timedelta(days=days_ago)
obs = dict(radio=int(Radio.gsm),
mcc=USA_MCC, mnc=ATT_MNC, lac=456, cid=123,
time=time,
lat=points[month % 4][0],
lon=points[month % 4][1])
# insert_result is num-accepted-observations, override
# utcnow to set creation date
insert_result = insert_measures_cell.delay(
[obs], utcnow=time)
# update_result is (num-stations, num-moving-stations)
update_result = update_cell.delay()
# Assuming PERMANENT_BLACKLIST_THRESHOLD == 6:
#
# 0th insert will create the station
# 1st insert will create first blacklist entry, delete station
# 2nd insert will recreate the station at new position
# 3rd insert will update blacklist, re-delete station
# 4th insert will recreate the station at new position
# 5th insert will update blacklist, re-delete station
# 6th insert will recreate the station at new position
# ...
# 11th insert will make blacklisting permanent, re-delete station
# 12th insert will not recreate station
# 13th insert will not recreate station
# ...
# 23rd insert will not recreate station
bl = self.session.query(CellBlacklist).all()
if month == 0:
self.assertEqual(len(bl), 0)
else:
self.assertEqual(len(bl), 1)
# force the blacklist back in time to whenever the
# observation was supposedly inserted.
bl = bl[0]
bl.time = time
self.session.add(bl)
self.session.commit()
if month < N / 2:
# We still haven't exceeded the threshold, so the
# observation was admitted.
self.assertEqual(insert_result.get(), 1)
if month % 2 == 0:
# The station was (re)created.
self.assertEqual(update_result.get(), (1, 0))
# Rescan lacs to update entries
self.assertEqual(
scan_areas.delay().get(), 1)
# One cell + one cell-LAC record should exist.
self.assertEqual(self.session.query(Cell).count(), 1)
self.assertEqual(self.session.query(CellArea).count(), 1)
else:
# The station existed and was seen moving,
# thereby activating the blacklist and deleting the cell.
self.assertEqual(update_result.get(), (1, 1))
# Rescan lacs to delete orphaned lac entry
self.assertEqual(
scan_areas.delay().get(), 1)
self.assertEqual(bl.count, ((month + 1) / 2))
self.assertEqual(
self.session.query(CellBlacklist).count(), 1)
self.assertEqual(self.session.query(Cell).count(), 0)
# Try adding one more observation 1 day later
# to be sure it is dropped by the now-active blacklist.
next_day = time + timedelta(days=1)
obs['time'] = next_day
self.assertEqual(
0, insert_measures_cell.delay([obs],
utcnow=next_day).get())
else:
# Blacklist has exceeded threshold, gone to permanent mode,
# so no observation accepted, no stations seen.
self.assertEqual(insert_result.get(), 0)
self.assertEqual(update_result.get(), (0, 0))
def test_blocklist_temporary_and_permanent(self):
# This test simulates a cell that moves once a month, for 2 years.
# The first 2 * PERMANENT_BLOCKLIST_THRESHOLD (12) moves should be
# temporary, forgotten after a week; after that it should be
# permanently blocklisted.
now = util.utcnow()
# Station moves between these 4 points, all in the USA:
points = [
(40.0, -74.0), # NYC
(37.0, -122.0), # SF
(47.0, -122.0), # Seattle
(25.0, -80.0), # Miami
]
obs = CellObservationFactory(
mcc=310, lat=points[0][0], lon=points[0][1])
N = 4 * PERMANENT_BLOCKLIST_THRESHOLD
for month in range(0, N):
days_ago = (N - (month + 1)) * 30
time = now - timedelta(days=days_ago)
obs.lat = points[month % 4][0]
obs.lon = points[month % 4][1]
# Assuming PERMANENT_BLOCKLIST_THRESHOLD == 6:
#
# 0th insert will create the station
# 1st insert will create first blocklist entry, delete station
# 2nd insert will recreate the station at new position
# 3rd insert will update blocklist, re-delete station
# 4th insert will recreate the station at new position
# 5th insert will update blocklist, re-delete station
# 6th insert will recreate the station at new position
# ...
# 11th insert will make blocklisting permanent, re-delete station
# 12th insert will not recreate station
# 13th insert will not recreate station
# ...
# 23rd insert will not recreate station
blocks = self.session.query(CellBlocklist).all()
if month < 2:
self.assertEqual(len(blocks), 0)
else:
self.assertEqual(len(blocks), 1)
# force the blocklist back in time to whenever the
# observation was supposedly inserted.
block = blocks[0]
block.time = time
self.session.commit()
if month < N / 2:
# We still haven't exceeded the threshold, so the
# observation was admitted.
self.data_queue.enqueue([obs])
if month % 2 == 0:
# The station was (re)created.
self.assertEqual(update_cell.delay().get(), (1, 0))
# Rescan lacs to update entries
self.assertEqual(scan_areas.delay().get(), 1)
# One cell + one cell-LAC record should exist.
self.assertEqual(self.session.query(Cell).count(), 1)
self.assertEqual(self.session.query(CellArea).count(), 1)
else:
# The station existed and was seen moving,
# thereby activating the blocklist and deleting the cell.
self.assertEqual(update_cell.delay().get(), (1, 1))
# Rescan lacs to delete orphaned lac entry
self.assertEqual(scan_areas.delay().get(), 1)
if month > 1:
self.assertEqual(block.count, ((month + 1) / 2))
self.assertEqual(
self.session.query(CellBlocklist).count(), 1)
self.assertEqual(self.session.query(Cell).count(), 0)
# Try adding one more observation
# to be sure it is dropped by the now-active blocklist.
self.data_queue.enqueue([obs])
self.assertEqual(update_cell.delay().get(), (0, 0))
else:
# Blocklist has exceeded threshold, gone to permanent mode,
# so no observation accepted, no stations seen.
self.data_queue.enqueue([obs])
self.assertEqual(update_cell.delay().get(), (0, 0))