def test_cell_blacklist(self):
from ichnaea.service.submit.tasks import insert_cell_measures
session = self.db_master_session
measures = [
dict(mcc=1,
mnc=2,
lac=3,
cid=i,
psc=5,
radio=RADIO_TYPE['gsm'],
id=0,
lat=10000000 + i,
lon=20000000 + i) for i in range(3)
]
black = CellBlacklist(
mcc=1,
mnc=2,
lac=3,
cid=1,
radio=RADIO_TYPE['gsm'],
)
session.add(black)
session.flush()
result = insert_cell_measures.delay(measures)
self.assertEqual(result.get(), 3)
measures = session.query(CellMeasure).all()
self.assertEqual(len(measures), 3)
cells = session.query(Cell).all()
self.assertEqual(len(cells), 2)
def test_blacklist(self):
session = self.db_master_session
measures = [
dict(mcc=FRANCE_MCC,
mnc=2,
lac=3,
cid=i,
psc=5,
radio=RADIO_TYPE['gsm'],
lat=PARIS_LAT + i * 0.0000001,
lon=PARIS_LON + i * 0.0000001) for i in range(1, 4)
]
black = CellBlacklist(
mcc=FRANCE_MCC,
mnc=2,
lac=3,
cid=1,
radio=RADIO_TYPE['gsm'],
)
session.add(black)
session.flush()
result = insert_measures_cell.delay(measures)
self.assertEqual(result.get(), 2)
measures = session.query(CellMeasure).all()
self.assertEqual(len(measures), 2)
cells = session.query(Cell).all()
self.assertEqual(len(cells), 2)
def test_blacklist(self):
now = util.utcnow()
session = self.db_master_session
observations = [
dict(mcc=FRANCE_MCC,
mnc=2,
lac=3,
cid=i,
psc=5,
radio=RADIO_TYPE['gsm'],
lat=PARIS_LAT + i * 0.0000001,
lon=PARIS_LON + i * 0.0000001) for i in range(1, 4)
]
black = CellBlacklist(
mcc=FRANCE_MCC,
mnc=2,
lac=3,
cid=1,
radio=RADIO_TYPE['gsm'],
time=now,
count=1,
)
session.add(black)
session.flush()
result = insert_measures_cell.delay(observations)
self.assertEqual(result.get(), 2)
cell_observations = session.query(CellObservation).all()
self.assertEqual(len(cell_observations), 2)
cells = session.query(Cell).all()
self.assertEqual(len(cells), 2)
def test_blacklist_time_used_as_creation_time(self):
now = util.utcnow()
last_week = now - TEMPORARY_BLACKLIST_DURATION - timedelta(days=1)
session = self.db_master_session
cell_key = {
'radio': RADIO_TYPE['gsm'],
'mcc': FRANCE_MCC,
'mnc': 2,
'lac': 3,
'cid': 1
}
session.add(CellBlacklist(time=last_week, count=1, **cell_key))
session.flush()
# add a new entry for the previously blacklisted cell
obs = dict(lat=PARIS_LAT, lon=PARIS_LON, **cell_key)
insert_measures_cell.delay([obs]).get()
# the cell was inserted again
cells = session.query(Cell).all()
self.assertEqual(len(cells), 1)
# and the creation date was set to the date of the blacklist entry
self.assertEqual(cells[0].created, last_week)
def mark_moving_cells(session, moving_cells):
moving_keys = []
blacklist = set()
for cell in moving_cells:
query = session.query(CellBlacklist).filter(
*join_cellkey(CellBlacklist, cell))
b = query.first()
if b is None:
key = to_cellkey(cell)._asdict()
blacklist.add(CellBlacklist(**key))
moving_keys.append(key)
get_heka_client().incr("items.blacklisted.cell_moving", len(moving_keys))
session.add_all(blacklist)
remove_cell.delay(moving_keys)
def test_blacklist_moving_cells(self):
now = util.utcnow()
k1 = dict(radio=Radio.cdma, mcc=1, mnc=2, lac=3, cid=4)
k2 = dict(radio=Radio.cdma, mcc=1, mnc=2, lac=6, cid=8)
k3 = dict(radio=Radio.cdma, mcc=1, mnc=2, lac=9, cid=12)
k4 = dict(radio=Radio.cdma, mcc=1, mnc=2, lac=12, cid=16)
k5 = dict(radio=Radio.cdma, mcc=1, mnc=2, lac=15, cid=20)
# keys k2, k3 and k4 are expected to be detected as moving
data = [
# a cell with an entry but no prior position
Cell(total_measures=0, **k1),
CellObservation(lat=1.001, lon=1.001, **k1),
CellObservation(lat=1.002, lon=1.005, **k1),
CellObservation(lat=1.003, lon=1.009, **k1),
# a cell with a prior known position
Cell(lat=2.0, lon=2.0, total_measures=1, **k2),
CellObservation(lat=2.0, lon=2.0, **k2),
CellObservation(lat=4.0, lon=2.0, **k2),
# a cell with a very different prior position
Cell(lat=1.0, lon=1.0, total_measures=1, **k3),
CellObservation(lat=3.0, lon=3.0, **k3),
CellObservation(lat=-3.0, lon=3.0, **k3),
# another cell with a prior known position (and negative lat)
Cell(lat=-4.0, lon=4.0, total_measures=1, **k4),
CellObservation(lat=-4.0, lon=4.0, **k4),
CellObservation(lat=-6.0, lon=4.0, **k4),
# an already blacklisted cell
CellBlacklist(time=now, count=1, **k5),
CellObservation(lat=5.0, lon=5.0, **k5),
CellObservation(lat=8.0, lon=5.0, **k5),
]
observations = []
for obj in data:
if isinstance(obj, CellObservation):
observations.append(obj)
else:
self.session.add(obj)
self.data_queue.enqueue(observations)
self.session.commit()
result = update_cell.delay()
self.assertEqual(result.get(), (4, 3))
moving = [k2, k3, k4, k5]
black = self.session.query(CellBlacklist).all()
self.assertEqual(set([b.hashkey() for b in black]),
set([CellBlacklist.to_hashkey(k) for k in moving]))
# test duplicate call
result = update_cell.delay()
self.assertEqual(result.get(), (0, 0))
self.check_stats(
timer=[
# We made duplicate calls
('task.data.update_cell', 2),
# One of those would've scheduled a remove_cell task
('task.data.remove_cell', 1)
])
def test_blacklist_moving_cells(self):
now = util.utcnow()
long_ago = now - timedelta(days=40)
session = self.db_master_session
k1 = dict(radio=1, mcc=1, mnc=2, lac=3, cid=4)
k2 = dict(radio=1, mcc=1, mnc=2, lac=6, cid=8)
k3 = dict(radio=1, mcc=1, mnc=2, lac=9, cid=12)
k4 = dict(radio=1, mcc=1, mnc=2, lac=12, cid=16)
k5 = dict(radio=1, mcc=1, mnc=2, lac=15, cid=20)
k6 = dict(radio=1, mcc=1, mnc=2, lac=18, cid=24)
keys = set([CellKey(**k) for k in [k1, k2, k3, k4, k5, k6]])
# keys k2, k3 and k4 are expected to be detected as moving
data = [
# a cell with an entry but no prior position
Cell(new_measures=3, total_measures=0, **k1),
CellMeasure(lat=1.001, lon=1.001, **k1),
CellMeasure(lat=1.002, lon=1.005, **k1),
CellMeasure(lat=1.003, lon=1.009, **k1),
# a cell with a prior known position
Cell(lat=2.0, lon=2.0, new_measures=2, total_measures=1, **k2),
CellMeasure(lat=2.0, lon=2.0, **k2),
CellMeasure(lat=4.0, lon=2.0, **k2),
# a cell with a very different prior position
Cell(lat=1.0, lon=1.0, new_measures=2, total_measures=1, **k3),
CellMeasure(lat=3.0, lon=3.0, **k3),
CellMeasure(lat=-3.0, lon=3.0, **k3),
# another cell with a prior known position (and negative lat)
Cell(lat=-4.0, lon=4.0, new_measures=2, total_measures=1, **k4),
CellMeasure(lat=-4.0, lon=4.0, **k4),
CellMeasure(lat=-6.0, lon=4.0, **k4),
# an already blacklisted cell
CellBlacklist(**k5),
CellMeasure(lat=5.0, lon=5.0, **k5),
CellMeasure(lat=8.0, lon=5.0, **k5),
# a cell with an old different record we ignore, position
# estimate has been updated since
Cell(lat=6.0, lon=6.0, new_measures=2, total_measures=1, **k6),
CellMeasure(lat=6.9, lon=6.9, time=long_ago, **k6),
CellMeasure(lat=6.0, lon=6.0, **k6),
CellMeasure(lat=6.001, lon=6, **k6),
]
session.add_all(data)
session.commit()
result = location_update_cell.delay(min_new=1)
self.assertEqual(result.get(), (5, 3))
black = session.query(CellBlacklist).all()
self.assertEqual(set([to_cellkey(b) for b in black]),
set([CellKey(**k) for k in [k2, k3, k4, k5]]))
measures = session.query(CellMeasure).all()
self.assertEqual(len(measures), 14)
self.assertEqual(set([to_cellkey(m) for m in measures]), keys)
# test duplicate call
result = location_update_cell.delay(min_new=1)
self.assertEqual(result.get(), 0)
self.check_stats(
total=6,
timer=[
# We made duplicate calls
('task.data.location_update_cell', 2),
# One of those would've scheduled a remove_cell task
('task.data.remove_cell', 1)
],
gauge=[
('task.data.location_update_cell.new_measures_1_100', 2),
])
def test_blacklist_moving_cells(self):
now = util.utcnow()
long_ago = now - timedelta(days=40)
session = self.session
k1 = dict(radio=Radio.cdma, mcc=1, mnc=2, lac=3, cid=4)
k2 = dict(radio=Radio.cdma, mcc=1, mnc=2, lac=6, cid=8)
k3 = dict(radio=Radio.cdma, mcc=1, mnc=2, lac=9, cid=12)
k4 = dict(radio=Radio.cdma, mcc=1, mnc=2, lac=12, cid=16)
k5 = dict(radio=Radio.cdma, mcc=1, mnc=2, lac=15, cid=20)
k6 = dict(radio=Radio.cdma, mcc=1, mnc=2, lac=18, cid=24)
# keys k2, k3 and k4 are expected to be detected as moving
data = [
# a cell with an entry but no prior position
Cell(new_measures=3, total_measures=0, **k1),
CellObservation(lat=1.001, lon=1.001, **k1),
CellObservation(lat=1.002, lon=1.005, **k1),
CellObservation(lat=1.003, lon=1.009, **k1),
# a cell with a prior known position
Cell(lat=2.0, lon=2.0,
new_measures=2, total_measures=1, **k2),
CellObservation(lat=2.0, lon=2.0, **k2),
CellObservation(lat=4.0, lon=2.0, **k2),
# a cell with a very different prior position
Cell(lat=1.0, lon=1.0,
new_measures=2, total_measures=1, **k3),
CellObservation(lat=3.0, lon=3.0, **k3),
CellObservation(lat=-3.0, lon=3.0, **k3),
# another cell with a prior known position (and negative lat)
Cell(lat=-4.0, lon=4.0,
new_measures=2, total_measures=1, **k4),
CellObservation(lat=-4.0, lon=4.0, **k4),
CellObservation(lat=-6.0, lon=4.0, **k4),
# an already blacklisted cell
CellBlacklist(time=now, count=1, **k5),
CellObservation(lat=5.0, lon=5.0, **k5),
CellObservation(lat=8.0, lon=5.0, **k5),
# a cell with an old different record we ignore, position
# estimate has been updated since
Cell(lat=6.0, lon=6.0,
new_measures=2, total_measures=1, **k6),
CellObservation(lat=6.9, lon=6.9, time=long_ago, **k6),
CellObservation(lat=6.0, lon=6.0, **k6),
CellObservation(lat=6.001, lon=6, **k6),
]
session.add_all(data)
session.commit()
result = location_update_cell.delay(min_new=1)
self.assertEqual(result.get(), (5, 3))
moving = [k2, k3, k4, k5]
black = session.query(CellBlacklist).all()
self.assertEqual(set([b.hashkey() for b in black]),
set([CellBlacklist.to_hashkey(k) for k in moving]))
keys = [k1, k2, k3, k4, k5, k6]
observations = session.query(CellObservation).all()
self.assertEqual(len(observations), 14)
self.assertEqual(set([obs.hashkey() for obs in observations]),
set([CellObservation.to_hashkey(k) for k in keys]))
# test duplicate call
result = location_update_cell.delay(min_new=1)
self.assertEqual(result.get(), (0, 0))
self.check_stats(
total=6,
timer=[
# We made duplicate calls
('task.data.location_update_cell', 2),
# One of those would've scheduled a remove_cell task
('task.data.remove_cell', 1)
],
gauge=[
('task.data.location_update_cell.new_measures_1_100', 2),
])