def test_blacklist_time_used_as_creation_time(self):
now = util.utcnow()
last_week = now - TEMPORARY_BLACKLIST_DURATION - timedelta(days=1)
cell = CellFactory.build()
self.session.add(
CellBlacklist(time=last_week, count=1,
radio=cell.radio, mcc=cell.mcc,
mnc=cell.mnc, lac=cell.lac, cid=cell.cid))
self.session.flush()
# add a new entry for the previously blacklisted cell
obs = dict(lat=cell.lat, lon=cell.lon,
radio=int(cell.radio), mcc=cell.mcc, mnc=cell.mnc,
lac=cell.lac, cid=cell.cid)
insert_measures_cell.delay([obs]).get()
self.assertEqual(self.data_queue.size(), 1)
update_cell.delay().get()
# the cell was inserted again
cells = self.session.query(Cell).all()
self.assertEqual(len(cells), 1)
# and the creation date was set to the date of the blacklist entry
self.assertEqual(cells[0].created, last_week)
self.check_statcounter(StatKey.cell, 1)
self.check_statcounter(StatKey.unique_cell, 0)
def test_blacklist(self):
now = util.utcnow()
cell = CellFactory.build()
observations = [dict(radio=int(cell.radio), mcc=cell.mcc,
mnc=cell.mnc, lac=cell.lac, cid=cell.cid + i,
psc=cell.psc,
lat=cell.lat + i * 0.0000001,
lon=cell.lon + i * 0.0000001)
for i in range(1, 4)]
black = CellBlacklist(
radio=cell.radio, mcc=cell.mcc, mnc=cell.mnc,
lac=cell.lac, cid=cell.cid + 1,
time=now, count=1,
)
self.session.add(black)
self.session.flush()
result = insert_measures_cell.delay(observations)
self.assertEqual(result.get(), 2)
self.assertEqual(self.data_queue.size(), 2)
update_cell.delay().get()
cells = self.session.query(Cell).all()
self.assertEqual(len(cells), 2)
self.check_statcounter(StatKey.cell, 2)
self.check_statcounter(StatKey.unique_cell, 2)
def test_stats(self):
self.session.add(ApiKey(valid_key='e5444-794', log=True))
self.session.flush()
self.add_reports(3, email='secretemail@localhost',
ip=self.geoip_data['London']['ip'])
self.add_reports(6, api_key='e5444-794')
self.add_reports(3, api_key=None)
schedule_export_reports.delay().get()
update_cell.delay().get()
update_wifi.delay().get()
self.check_stats(counter=[
('data.export.batch', 1, 1, ['key:internal']),
('data.report.upload', 2, 3),
('data.report.upload', 1, 3, ['key:test']),
('data.report.upload', 1, 6, ['key:e5444-794']),
('data.observation.insert', 1, 12, ['type:cell']),
('data.observation.insert', 1, 24, ['type:wifi']),
('data.observation.upload', 1, 3, ['type:cell', 'key:test']),
('data.observation.upload', 1, 6, ['type:wifi', 'key:test']),
('data.observation.upload', 0, ['type:cell', 'key:no_key']),
('data.observation.upload', 1, 6, ['type:cell', 'key:e5444-794']),
('data.observation.upload', 1, 12, ['type:wifi', 'key:e5444-794']),
])
def test_blacklist_time_used_as_creation_time(self):
now = util.utcnow()
last_week = now - TEMPORARY_BLACKLIST_DURATION - timedelta(days=1)
session = self.session
cell_key = {'mcc': FRANCE_MCC, 'mnc': 2, 'lac': 3, 'cid': 1}
session.add(CellBlacklist(time=last_week, count=1,
radio=Radio.gsm, **cell_key))
session.flush()
# add a new entry for the previously blacklisted cell
obs = dict(lat=PARIS_LAT, lon=PARIS_LON,
radio=int(Radio.gsm), **cell_key)
insert_measures_cell.delay([obs]).get()
self.assertEqual(self.data_queue.size(), 1)
update_cell.delay().get()
# the cell was inserted again
cells = session.query(Cell).all()
self.assertEqual(len(cells), 1)
# and the creation date was set to the date of the blacklist entry
self.assertEqual(cells[0].created, last_week)
self.check_statcounter(StatKey.cell, 1)
self.check_statcounter(StatKey.unique_cell, 0)
def test_created_from_blocklist_time(self):
now = util.utcnow()
last_week = now - TEMPORARY_BLOCKLIST_DURATION - timedelta(days=1)
obs = CellObservationFactory.build()
self.session.add(
CellBlocklist(time=last_week, count=1,
radio=obs.radio, mcc=obs.mcc,
mnc=obs.mnc, lac=obs.lac, cid=obs.cid))
self.session.flush()
# add a new entry for the previously blocklisted cell
self.data_queue.enqueue([obs])
self.assertEqual(self.data_queue.size(), 1)
update_cell.delay().get()
# the cell was inserted again
cells = self.session.query(Cell).all()
self.assertEqual(len(cells), 1)
# and the creation date was set to the date of the blocklist entry
self.assertEqual(cells[0].created, last_week)
self.check_statcounter(StatKey.cell, 1)
self.check_statcounter(StatKey.unique_cell, 0)
def test_blacklist(self):
now = util.utcnow()
session = self.session
observations = [dict(mcc=FRANCE_MCC, mnc=2, lac=3, cid=i, psc=5,
radio=int(Radio.gsm),
lat=PARIS_LAT + i * 0.0000001,
lon=PARIS_LON + i * 0.0000001)
for i in range(1, 4)]
black = CellBlacklist(
mcc=FRANCE_MCC, mnc=2, lac=3, cid=1,
radio=Radio.gsm, time=now, count=1,
)
session.add(black)
session.flush()
result = insert_measures_cell.delay(observations)
self.assertEqual(result.get(), 2)
self.assertEqual(self.data_queue.size(), 2)
update_cell.delay().get()
cells = session.query(Cell).all()
self.assertEqual(len(cells), 2)
self.check_statcounter(StatKey.cell, 2)
self.check_statcounter(StatKey.unique_cell, 2)
def test_insert_observations_invalid_lac(self):
time = util.utcnow() - timedelta(days=1)
today = util.utcnow().date()
cell = CellFactory(total_measures=5)
self.session.add(Score(key=ScoreKey.new_cell,
userid=1, time=today, value=7))
self.session.flush()
obs = dict(
radio=int(cell.radio), mcc=cell.mcc, mnc=cell.mnc, psc=cell.psc,
created=time, time=time, lat=cell.lat, lon=cell.lon,
accuracy=0, altitude=0, altitude_accuracy=0)
entries = [
{'lac': constants.MAX_LAC_ALL + 1,
'cid': constants.MAX_CID_ALL + 1, 'asu': 8},
{'lac': None, 'cid': None, 'asu': 8},
]
for entry in entries:
entry.update(obs)
result = insert_measures_cell.delay(entries, userid=1)
self.assertEqual(result.get(), 0)
# The incomplete observations never make it into the queue
self.assertEqual(self.data_queue.size(), 0)
update_cell.delay().get()
# Nothing should change in the initially created Cell record
self.session.refresh(cell)
cells = self.session.query(Cell).all()
self.assertEqual(len(cells), 1)
self._compare_sets([c.total_measures for c in cells], [5])
def test_stats(self):
self.session.add(ApiKey(valid_key='e5444-794', log_submit=True))
self.session.flush()
self.add_reports(3, email='secretemail@localhost',
ip=self.geoip_data['London']['ip'])
self.add_reports(6, api_key='e5444-794')
self.add_reports(3, api_key=None)
schedule_export_reports.delay().get()
update_cell.delay().get()
for i in range(16):
update_wifi.delay(shard_id='%x' % i).get()
self.check_stats(counter=[
('data.export.batch', 1, 1, ['key:internal']),
('data.report.upload', 2, 3),
('data.report.upload', 1, 3, ['key:test']),
('data.report.upload', 1, 6, ['key:e5444-794']),
('data.observation.insert', 1, 12, ['type:cell']),
('data.observation.upload', 1, 3, ['type:cell', 'key:test']),
('data.observation.upload', 1, 6, ['type:wifi', 'key:test']),
('data.observation.upload', 0, ['type:cell', 'key:no_key']),
('data.observation.upload', 1, 6, ['type:cell', 'key:e5444-794']),
('data.observation.upload', 1, 12, ['type:wifi', 'key:e5444-794']),
])
# we get a variable number of statsd messages and are only
# interested in the sum-total
insert_msgs = [msg for msg in self.stats_client.msgs
if (msg.startswith('data.observation.insert') and
'type:wifi' in msg)]
self.assertEqual(
sum([int(msg.split(':')[1].split('|')[0]) for msg in insert_msgs]),
24)
def _update_all(self):
schedule_export_reports.delay().get()
for shard_id in CellShard.shards().keys():
update_cell.delay(shard_id=shard_id).get()
for shard_id in WifiShard.shards().keys():
update_wifi.delay(shard_id=shard_id).get()
def _update_all(self):
schedule_export_reports.delay().get()
for shard_id in CellShard.shards().keys():
update_cell.delay(shard_id=shard_id).get()
for shard_id in WifiShard.shards().keys():
update_wifi.delay(shard_id=shard_id).get()
def _queue_and_update_cell(self, obs):
sharded_obs = defaultdict(list)
for ob in obs:
sharded_obs[CellShard.shard_id(ob.cellid)].append(ob)
for shard_id, values in sharded_obs.items():
queue = self.celery_app.data_queues['update_cell_' + shard_id]
queue.enqueue([value.to_json() for value in values])
update_cell.delay(shard_id=shard_id).get()
def _queue_and_update(self, obs):
sharded_obs = defaultdict(list)
for ob in obs:
sharded_obs[CellShard.shard_id(ob.cellid)].append(ob)
for shard_id, values in sharded_obs.items():
queue = self.celery_app.data_queues['update_cell_' + shard_id]
queue.enqueue(values)
update_cell.delay(shard_id=shard_id).get()
def test_stats(self):
self.add_reports(3, api_key='test')
schedule_export_reports.delay().get()
update_cell.delay().get()
update_wifi.delay().get()
self.check_stats(
counter=[('items.api_log.test.uploaded.reports', 1, 3),
('items.uploaded.reports', 1, 3)])
def test_insert_observations(self):
session = self.session
time = util.utcnow() - timedelta(days=1)
mcc = FRANCE_MCC
session.add(Cell(radio=Radio.gsm, mcc=mcc, mnc=2, lac=3,
cid=4, psc=5, total_measures=5))
user = User(nickname=u'test')
session.add(user)
session.flush()
obs = dict(
created=time,
lat=PARIS_LAT,
lon=PARIS_LON,
time=time, accuracy=0, altitude=0,
altitude_accuracy=0, radio=int(Radio.gsm),
)
entries = [
# Note that this first entry will be skipped as it does
# not include (lac, cid) or (psc)
{'mcc': mcc, 'mnc': 2, 'signal': -100},
{'mcc': mcc, 'mnc': 2, 'lac': 3, 'cid': 4, 'psc': 5, 'asu': 8},
{'mcc': mcc, 'mnc': 2, 'lac': 3, 'cid': 4, 'psc': 5, 'asu': 8},
{'mcc': mcc, 'mnc': 2, 'lac': 3, 'cid': 4, 'psc': 5, 'asu': 15},
{'mcc': mcc, 'mnc': 2, 'lac': 3, 'cid': 7, 'psc': 5},
]
for entry in entries:
entry.update(obs)
result = insert_measures_cell.delay(entries, userid=user.id)
self.assertEqual(result.get(), 4)
self.assertEqual(self.data_queue.size(), 4)
update_cell.delay().get()
cells = session.query(Cell).all()
self.assertEqual(len(cells), 2)
self._compare_sets([c.mcc for c in cells], [mcc])
self._compare_sets([c.mnc for c in cells], [2])
self._compare_sets([c.lac for c in cells], [3])
self._compare_sets([c.cid for c in cells], [4, 7])
self._compare_sets([c.psc for c in cells], [5])
self._compare_sets([c.total_measures for c in cells], [1, 8])
score_queue = self.celery_app.data_queues['update_score']
scores = score_queue.dequeue()
self.assertEqual(len(scores), 1)
score = scores[0]
self.assertEqual(score['hashkey'].userid, user.id)
self.assertEqual(score['hashkey'].key, ScoreKey.new_cell)
self.assertEqual(score['value'], 1)
self.check_statcounter(StatKey.cell, 4)
self.check_statcounter(StatKey.unique_cell, 1)
def test_cell_invalid(self):
self.add_reports(cell_factor=1, wifi_factor=0, cell_mcc=-2)
schedule_export_reports.delay().get()
update_cell.delay().get()
self.assertEqual(self.session.query(Cell).count(), 0)
self.check_stats(counter=[
('data.report.upload', 1, 1, ['key:test']),
('data.report.drop', 1, 1, ['reason:malformed', 'key:test']),
('data.observation.drop', 1, 1,
['type:cell', 'reason:malformed', 'key:test']),
])
def _update_all(self):
update_incoming.delay().get()
for shard_id in BlueShard.shards().keys():
update_blue.delay(shard_id=shard_id).get()
for shard_id in CellShard.shards().keys():
update_cell.delay(shard_id=shard_id).get()
for shard_id in WifiShard.shards().keys():
update_wifi.delay(shard_id=shard_id).get()
def test_insert_observations(self):
time = util.utcnow() - timedelta(days=1)
cell = CellFactory(radio=Radio.gsm, total_measures=5)
user = User(nickname=u'test')
self.session.add(user)
self.session.flush()
obs = dict(
radio=int(cell.radio), mcc=cell.mcc, mnc=cell.mnc,
created=time, time=time, lat=cell.lat, lon=cell.lon,
accuracy=0, altitude=0, altitude_accuracy=0,
)
entries = [
# Note that this first entry will be skipped as it does
# not include (lac, cid) or (psc)
{'signal': -100},
{'lac': cell.lac, 'cid': cell.cid, 'psc': cell.psc, 'asu': 8},
{'lac': cell.lac, 'cid': cell.cid, 'psc': cell.psc, 'asu': 8},
{'lac': cell.lac, 'cid': cell.cid, 'psc': cell.psc, 'asu': 15},
{'lac': cell.lac, 'cid': cell.cid + 1, 'psc': cell.psc},
]
for entry in entries:
entry.update(obs)
result = insert_measures_cell.delay(entries, userid=user.id)
self.assertEqual(result.get(), 4)
self.assertEqual(self.data_queue.size(), 4)
update_cell.delay().get()
self.session.refresh(cell)
cells = self.session.query(Cell).all()
self.assertEqual(len(cells), 2)
self._compare_sets([c.mcc for c in cells], [cell.mcc])
self._compare_sets([c.mnc for c in cells], [cell.mnc])
self._compare_sets([c.lac for c in cells], [cell.lac])
self._compare_sets([c.cid for c in cells], [cell.cid, cell.cid + 1])
self._compare_sets([c.psc for c in cells], [cell.psc])
self._compare_sets([c.total_measures for c in cells], [1, 8])
score_queue = self.celery_app.data_queues['update_score']
scores = score_queue.dequeue()
self.assertEqual(len(scores), 1)
score = scores[0]
self.assertEqual(score['hashkey'].userid, user.id)
self.assertEqual(score['hashkey'].key, ScoreKey.new_cell)
self.assertEqual(score['value'], 1)
self.check_statcounter(StatKey.cell, 4)
self.check_statcounter(StatKey.unique_cell, 1)
def test_shard_queues(self): # BBB
observations = CellObservationFactory.build_batch(3)
data_queues = self.celery_app.data_queues
single_queue = data_queues['update_cell']
single_queue.enqueue(observations)
update_cell.delay().get()
self.assertEqual(single_queue.size(), 0)
total = 0
for shard_id in CellShard.shards().keys():
total += data_queues['update_cell_' + shard_id].size()
self.assertEqual(total, 3)
def test_shard_queues(self): # BBB
observations = CellObservationFactory.build_batch(3)
data_queues = self.celery_app.data_queues
single_queue = data_queues['update_cell']
single_queue.enqueue(observations)
update_cell.delay().get()
self.assertEqual(single_queue.size(), 0)
total = 0
for shard_id in CellShard.shards().keys():
total += data_queues['update_cell_' + shard_id].size()
self.assertEqual(total, 3)
def _update_all(self, session, datamap_only=False):
ExportConfigFactory(name="internal", batch=0, schema="internal")
session.flush()
update_incoming.delay().get()
if datamap_only:
return
for shard_id in BlueShard.shards().keys():
update_blue.delay(shard_id=shard_id).get()
for shard_id in CellShard.shards().keys():
update_cell.delay(shard_id=shard_id).get()
for shard_id in WifiShard.shards().keys():
update_wifi.delay(shard_id=shard_id).get()
def test_max_min_range_update(self):
k1 = dict(radio=Radio.cdma, mcc=1, mnc=2, lac=3, cid=4)
self.session.add(
Cell(lat=1.001, lon=-1.001,
max_lat=1.002, min_lat=1.0,
max_lon=-1.0, min_lon=-1.002,
total_measures=3, **k1))
observations = [
CellObservation(lat=1.001, lon=-1.003, **k1),
CellObservation(lat=1.005, lon=-1.007, **k1),
]
self.data_queue.enqueue(observations)
self.session.commit()
result = update_cell.delay()
self.assertEqual(result.get(), (1, 0))
cells = self.session.query(Cell).all()
self.assertEqual(len(cells), 1)
cell = cells[0]
self.assertAlmostEqual(cell.lat, 1.0018, 6)
self.assertEqual(cell.max_lat, 1.005)
self.assertEqual(cell.min_lat, 1.0)
self.assertAlmostEqual(cell.lon, -1.0026, 6)
self.assertEqual(cell.max_lon, -1.0)
self.assertEqual(cell.min_lon, -1.007)
self.assertEqual(cell.range, 605)
def _update_all(self, session, datamap_only=False):
ExportConfigFactory(name='internal', batch=0, schema='internal')
session.flush()
update_incoming.delay().get()
if datamap_only:
return
for shard_id in BlueShard.shards().keys():
update_blue.delay(shard_id=shard_id).get()
for shard_id in CellShard.shards().keys():
update_cell.delay(shard_id=shard_id).get()
for shard_id in WifiShard.shards().keys():
update_wifi.delay(shard_id=shard_id).get()
def test_max_min_range_update(self):
cell = CellFactory(range=150, total_measures=3)
cell_lat = cell.lat
cell_lon = cell.lon
cell.max_lat = cell.lat + 0.001
cell.min_lat = cell.lat - 0.001
cell.max_lon = cell.lon + 0.001
cell.min_lon = cell.lon - 0.001
k1 = cell.hashkey().__dict__
observations = [
CellObservation(lat=cell.lat, lon=cell.lon - 0.002, **k1),
CellObservation(lat=cell.lat + 0.004, lon=cell.lon - 0.006, **k1),
]
self.data_queue.enqueue(observations)
self.session.commit()
self.assertEqual(update_cell.delay().get(), (1, 0))
cells = self.session.query(Cell).all()
self.assertEqual(len(cells), 1)
cell = cells[0]
self.assertAlmostEqual(cell.lat, cell_lat + 0.0008)
self.assertAlmostEqual(cell.max_lat, cell_lat + 0.004)
self.assertAlmostEqual(cell.min_lat, cell_lat - 0.001)
self.assertAlmostEqual(cell.lon, cell_lon - 0.0016)
self.assertAlmostEqual(cell.max_lon, cell_lon + 0.001)
self.assertAlmostEqual(cell.min_lon, cell_lon - 0.006)
self.assertEqual(cell.range, 468)
self.assertEqual(cell.total_measures, 5)
def add_line_of_cells_and_scan_lac(self):
big = 1.0
small = big / 10
keys = dict(radio=Radio.cdma, mcc=1, mnc=1, lac=1)
observations = [
CellObservation(lat=ctr + xd, lon=ctr + yd, cid=cell, **keys)
for cell in range(10)
for ctr in [cell * big]
for (xd, yd) in [(small, small),
(small, -small),
(-small, small),
(-small, -small)]
]
self.obs_queue.enqueue(observations)
cells = [
Cell(lat=ctr, lon=ctr, cid=cell, total_measures=1, **keys)
for cell in range(10)
for ctr in [cell * big]
]
self.session.add_all(cells)
self.session.commit()
result = update_cell.delay()
self.assertEqual(result.get(), (len(cells), 0))
scan_areas.delay().get()
def test_upload_duplicated_cell(self):
self.add_reports(cell_factor=1, wifi_factor=0)
# duplicate the cell entry inside the report
queue = self.celery_app.export_queues['internal']
items = queue.dequeue(queue.queue_key())
report = items[0]['report']
cell = report['cellTowers'][0].copy()
report['cellTowers'].append(cell)
report['cellTowers'][1]['signalStrength'] += 2
queue.enqueue(items, queue.queue_key())
schedule_export_reports.delay().get()
update_cell.delay().get()
cells = self.session.query(Cell).all()
self.assertEqual(len(cells), 1)
self.assertEqual(cells[0].total_measures, 1)
def test_update_cell(self):
now = util.utcnow()
invalid_key = dict(lac=None, cid=None)
observations = []
def obs_factory(**kw):
obs = CellObservationFactory.build(**kw)
observations.append(obs)
cell1 = CellFactory(total_measures=3)
lat1, lon1 = (cell1.lat, cell1.lon)
key1 = dict(lac=cell1.lac, cid=cell1.cid)
obs_factory(lat=lat1, lon=lon1, created=now, **key1)
obs_factory(lat=lat1 + 0.004, lon=lon1 + 0.006, created=now, **key1)
obs_factory(lat=lat1 + 0.006, lon=lon1 + 0.009, created=now, **key1)
# The lac, cid are invalid and should be skipped
obs_factory(created=now, **invalid_key)
obs_factory(created=now, **invalid_key)
cell2 = CellFactory(lat=lat1 + 1.0, lon=lon1 + 1.0, total_measures=3)
lat2, lon2 = (cell2.lat, cell2.lon)
key2 = dict(lac=cell2.lac, cid=cell2.cid)
obs_factory(lat=lat2 + 0.001, lon=lon2 + 0.002, created=now, **key2)
obs_factory(lat=lat2 + 0.003, lon=lon2 + 0.006, created=now, **key2)
cell3 = CellFactory(total_measures=100000)
lat3, lon3 = (cell3.lat, cell3.lon)
for i in range(10):
obs_factory(lat=lat3 + 1.0, lon=lon3 + 1.0,
**dict(lac=cell3.lac, cid=cell3.cid))
self.data_queue.enqueue(observations)
self.session.commit()
result = update_cell.delay()
self.assertEqual(result.get(), (3, 0))
self.check_stats(
timer=['task.data.update_cell'],
)
cells = self.session.query(Cell).all()
self.assertEqual(len(cells), 3)
for cell in cells:
if cell.hashkey() == cell1.hashkey():
self.assertAlmostEqual(cell.lat, lat1 + 0.001667, 6)
self.assertAlmostEqual(cell.lon, lon1 + 0.0025, 6)
if cell.hashkey() == cell2.hashkey():
self.assertAlmostEqual(cell.lat, lat2 + 0.0008, 6)
self.assertAlmostEqual(cell.lon, lon2 + 0.0016, 6)
if cell.hashkey() == cell3.hashkey():
expected_lat = ((lat3 * 1000) + (lat3 + 1.0) * 10) / 1010
expected_lon = ((lon3 * 1000) + (lon3 + 1.0) * 10) / 1010
self.assertAlmostEqual(cell.lat, expected_lat, 7)
self.assertAlmostEqual(cell.lon, expected_lon, 7)
def test_upload_cell(self):
reports = self.add_reports(cell_factor=1, wifi_factor=0)
schedule_export_reports.delay().get()
update_cell.delay().get()
position = reports[0]['position']
cell_data = reports[0]['cellTowers'][0]
cells = self.session.query(Cell).all()
self.assertEqual(len(cells), 1)
cell = cells[0]
self.assertEqual(cell.lat, position['latitude'])
self.assertEqual(cell.lon, position['longitude'])
self.assertEqual(cell.radio.name, cell_data['radioType'])
self.assertEqual(cell.mcc, cell_data['mobileCountryCode'])
self.assertEqual(cell.mnc, cell_data['mobileNetworkCode'])
self.assertEqual(cell.lac, cell_data['locationAreaCode'])
self.assertEqual(cell.cid, cell_data['cellId'])
self.assertEqual(cell.psc, cell_data['primaryScramblingCode'])
self.assertEqual(cell.total_measures, 1)
def test_blocklist(self):
now = util.utcnow()
observations = CellObservationFactory.build_batch(3)
obs = observations[0]
block = CellBlocklist(
radio=obs.radio, mcc=obs.mcc, mnc=obs.mnc,
lac=obs.lac, cid=obs.cid,
time=now, count=1,
)
self.session.add(block)
self.session.flush()
self.data_queue.enqueue(observations)
self.assertEqual(self.data_queue.size(), 3)
update_cell.delay().get()
cells = self.session.query(Cell).all()
self.assertEqual(len(cells), 2)
self.check_statcounter(StatKey.cell, 2)
self.check_statcounter(StatKey.unique_cell, 2)
def test_insert_observations_invalid_lac(self):
session = self.session
time = util.utcnow() - timedelta(days=1)
today = util.utcnow().date()
session.add(Cell(radio=Radio.gsm, mcc=FRANCE_MCC, mnc=2,
lac=3, cid=4, total_measures=5))
session.add(Score(key=ScoreKey.new_cell,
userid=1, time=today, value=7))
session.flush()
obs = dict(
created=time,
lat=PARIS_LAT,
lon=PARIS_LON,
time=time, accuracy=0, altitude=0,
altitude_accuracy=0, radio=int(Radio.gsm))
entries = [
{'mcc': FRANCE_MCC, 'mnc': 2, 'lac': constants.MAX_LAC_ALL + 1,
'cid': constants.MAX_CID_ALL + 1, 'psc': 5, 'asu': 8},
{'mcc': FRANCE_MCC, 'mnc': 2, 'lac': None,
'cid': None, 'psc': 5, 'asu': 8},
]
for entry in entries:
entry.update(obs)
result = insert_measures_cell.delay(entries, userid=1)
self.assertEqual(result.get(), 0)
# The incomplete observations never make it into the queue
self.assertEqual(self.data_queue.size(), 0)
update_cell.delay().get()
# Nothing should change in the initially created Cell record
cells = session.query(Cell).all()
self.assertEqual(len(cells), 1)
self._compare_sets([c.total_measures for c in cells], [5])
def test_upload(self):
self.session.add(ApiKey(valid_key='e5444-794', log=True))
self.session.flush()
self.add_reports(3, email='secretemail@localhost',
ip=self.geoip_data['London']['ip'])
self.add_reports(6, api_key='e5444-794')
self.add_reports(3, api_key=None)
schedule_export_reports.delay().get()
update_cell.delay().get()
update_wifi.delay().get()
self.assertEqual(self.session.query(Cell).count(), 12)
self.assertEqual(self.session.query(Wifi).count(), 24)
self.check_stats(counter=[
('items.export.internal.batches', 1, 1),
('items.api_log.test.uploaded.cell_observations', 1, 3),
('items.api_log.test.uploaded.wifi_observations', 1, 6),
('items.api_log.no_key.uploaded.cell_observations', 0),
('items.api_log.e5444-794.uploaded.cell_observations', 1, 6),
('items.api_log.e5444-794.uploaded.wifi_observations', 1, 12),
])
def test_scan_areas_race_with_update_cell(self):
session = self.session
# First batch of cell observations for CID 1
keys = dict(radio=Radio.cdma, mcc=1, mnc=1, lac=1, cid=1)
cell = Cell(total_measures=1, **keys)
observations = [
CellObservation(lat=1.0, lon=1.0, **keys),
CellObservation(lat=1.0, lon=1.0, **keys),
CellObservation(lat=1.0, lon=1.0, **keys),
CellObservation(lat=1.0, lon=1.0, **keys),
]
session.add(cell)
self.obs_queue.enqueue(observations)
session.commit()
# Periodic update_cell runs and updates CID 1
# to have a location, inserts LAC 1
# which will be picked up by the next scan_lac.
result = update_cell.delay()
self.assertEqual(result.get(), (1, 0))
# Second batch of cell observations for CID 2
keys['cid'] = 2
cell = Cell(total_measures=1, **keys)
observations = [
CellObservation(lat=1.0, lon=1.0, **keys),
CellObservation(lat=1.0, lon=1.0, **keys),
CellObservation(lat=1.0, lon=1.0, **keys),
CellObservation(lat=1.0, lon=1.0, **keys),
]
session.add(cell)
self.obs_queue.enqueue(observations)
session.commit()
# Periodic LAC scan runs, picking up LAC 1; this could
# accidentally pick up CID 2, but it should not since it
# has not had its location updated yet. If there's no
# exception here, CID 2 is properly ignored.
scan_areas.delay()
def test_scan_areas_asymmetric(self):
session = self.session
big = 0.1
small = big / 10
keys = dict(radio=Radio.cdma, mcc=1, mnc=1, lac=1)
observations = [
CellObservation(lat=ctr + xd, lon=ctr + yd, cid=cell, **keys)
for cell in range(6)
for ctr in [(2 ** cell) * big]
for (xd, yd) in [(small, small),
(small, -small),
(-small, small),
(-small, -small)]
]
self.obs_queue.enqueue(observations)
cells = [
Cell(lat=ctr, lon=ctr, cid=cell, total_measures=1, **keys)
for cell in range(6)
for ctr in [(2 ** cell) * big]
]
session.add_all(cells)
session.commit()
result = update_cell.delay()
self.assertEqual(result.get(), (len(cells), 0))
scan_areas.delay()
lac = session.query(CellArea).filter(CellArea.lac == 1).first()
# We produced a sequence of 0.02-degree-on-a-side
# cell bounding boxes centered at
# [0, 0.2, 0.4, 0.8, 1.6, 3.2] degrees.
# So the lower-left corner is at (-0.01, -0.01)
# and the upper-right corner is at (3.21, 3.21)
# we should therefore see a LAC centroid at (1.05, 1.05)
# with a range of 339.540m
self.assertEqual(lac.lat, 1.05)
self.assertEqual(lac.lon, 1.05)
self.assertEqual(lac.range, 339540)
def test_upload_invalid_position(self):
self.add_reports(1, cell_factor=1, wifi_factor=0, lat=-90.1)
self.add_reports(1, cell_factor=1, wifi_factor=0)
schedule_export_reports.delay().get()
update_cell.delay().get()
self.assertEqual(self.session.query(Cell).count(), 1)
def test_upload_invalid_cell(self):
self.add_reports(cell_factor=1, wifi_factor=0, cell_mcc=-2)
schedule_export_reports.delay().get()
update_cell.delay().get()
self.assertEqual(self.session.query(Cell).count(), 0)
def test_blacklist_moving_cells(self):
now = util.utcnow()
k1 = dict(radio=Radio.cdma, mcc=1, mnc=2, lac=3, cid=4)
k2 = dict(radio=Radio.cdma, mcc=1, mnc=2, lac=6, cid=8)
k3 = dict(radio=Radio.cdma, mcc=1, mnc=2, lac=9, cid=12)
k4 = dict(radio=Radio.cdma, mcc=1, mnc=2, lac=12, cid=16)
k5 = dict(radio=Radio.cdma, mcc=1, mnc=2, lac=15, cid=20)
# keys k2, k3 and k4 are expected to be detected as moving
data = [
# a cell with an entry but no prior position
Cell(total_measures=0, **k1),
CellObservation(lat=1.001, lon=1.001, **k1),
CellObservation(lat=1.002, lon=1.005, **k1),
CellObservation(lat=1.003, lon=1.009, **k1),
# a cell with a prior known position
Cell(lat=2.0, lon=2.0, total_measures=1, **k2),
CellObservation(lat=2.0, lon=2.0, **k2),
CellObservation(lat=4.0, lon=2.0, **k2),
# a cell with a very different prior position
Cell(lat=1.0, lon=1.0, total_measures=1, **k3),
CellObservation(lat=3.0, lon=3.0, **k3),
CellObservation(lat=-3.0, lon=3.0, **k3),
# another cell with a prior known position (and negative lat)
Cell(lat=-4.0, lon=4.0, total_measures=1, **k4),
CellObservation(lat=-4.0, lon=4.0, **k4),
CellObservation(lat=-6.0, lon=4.0, **k4),
# an already blacklisted cell
CellBlacklist(time=now, count=1, **k5),
CellObservation(lat=5.0, lon=5.0, **k5),
CellObservation(lat=8.0, lon=5.0, **k5),
]
observations = []
for obj in data:
if isinstance(obj, CellObservation):
observations.append(obj)
else:
self.session.add(obj)
self.data_queue.enqueue(observations)
self.session.commit()
result = update_cell.delay()
self.assertEqual(result.get(), (4, 3))
moving = [k2, k3, k4, k5]
black = self.session.query(CellBlacklist).all()
self.assertEqual(set([b.hashkey() for b in black]),
set([CellBlacklist.to_hashkey(k) for k in moving]))
# test duplicate call
result = update_cell.delay()
self.assertEqual(result.get(), (0, 0))
self.check_stats(
timer=[
# We made duplicate calls
('task.data.update_cell', 2),
# One of those would've scheduled a remove_cell task
('task.data.remove_cell', 1)
])
