def test_blacklist_time_used_as_creation_time(self): now = util.utcnow() last_week = now - TEMPORARY_BLACKLIST_DURATION - timedelta(days=1) cell = CellFactory.build() self.session.add( CellBlacklist(time=last_week, count=1, radio=cell.radio, mcc=cell.mcc, mnc=cell.mnc, lac=cell.lac, cid=cell.cid)) self.session.flush() # add a new entry for the previously blacklisted cell obs = dict(lat=cell.lat, lon=cell.lon, radio=int(cell.radio), mcc=cell.mcc, mnc=cell.mnc, lac=cell.lac, cid=cell.cid) insert_measures_cell.delay([obs]).get() self.assertEqual(self.data_queue.size(), 1) update_cell.delay().get() # the cell was inserted again cells = self.session.query(Cell).all() self.assertEqual(len(cells), 1) # and the creation date was set to the date of the blacklist entry self.assertEqual(cells[0].created, last_week) self.check_statcounter(StatKey.cell, 1) self.check_statcounter(StatKey.unique_cell, 0)
def test_blacklist(self): now = util.utcnow() cell = CellFactory.build() observations = [dict(radio=int(cell.radio), mcc=cell.mcc, mnc=cell.mnc, lac=cell.lac, cid=cell.cid + i, psc=cell.psc, lat=cell.lat + i * 0.0000001, lon=cell.lon + i * 0.0000001) for i in range(1, 4)] black = CellBlacklist( radio=cell.radio, mcc=cell.mcc, mnc=cell.mnc, lac=cell.lac, cid=cell.cid + 1, time=now, count=1, ) self.session.add(black) self.session.flush() result = insert_measures_cell.delay(observations) self.assertEqual(result.get(), 2) self.assertEqual(self.data_queue.size(), 2) update_cell.delay().get() cells = self.session.query(Cell).all() self.assertEqual(len(cells), 2) self.check_statcounter(StatKey.cell, 2) self.check_statcounter(StatKey.unique_cell, 2)
def test_stats(self): self.session.add(ApiKey(valid_key='e5444-794', log=True)) self.session.flush() self.add_reports(3, email='secretemail@localhost', ip=self.geoip_data['London']['ip']) self.add_reports(6, api_key='e5444-794') self.add_reports(3, api_key=None) schedule_export_reports.delay().get() update_cell.delay().get() update_wifi.delay().get() self.check_stats(counter=[ ('data.export.batch', 1, 1, ['key:internal']), ('data.report.upload', 2, 3), ('data.report.upload', 1, 3, ['key:test']), ('data.report.upload', 1, 6, ['key:e5444-794']), ('data.observation.insert', 1, 12, ['type:cell']), ('data.observation.insert', 1, 24, ['type:wifi']), ('data.observation.upload', 1, 3, ['type:cell', 'key:test']), ('data.observation.upload', 1, 6, ['type:wifi', 'key:test']), ('data.observation.upload', 0, ['type:cell', 'key:no_key']), ('data.observation.upload', 1, 6, ['type:cell', 'key:e5444-794']), ('data.observation.upload', 1, 12, ['type:wifi', 'key:e5444-794']), ])
def test_blacklist_time_used_as_creation_time(self): now = util.utcnow() last_week = now - TEMPORARY_BLACKLIST_DURATION - timedelta(days=1) session = self.session cell_key = {'mcc': FRANCE_MCC, 'mnc': 2, 'lac': 3, 'cid': 1} session.add(CellBlacklist(time=last_week, count=1, radio=Radio.gsm, **cell_key)) session.flush() # add a new entry for the previously blacklisted cell obs = dict(lat=PARIS_LAT, lon=PARIS_LON, radio=int(Radio.gsm), **cell_key) insert_measures_cell.delay([obs]).get() self.assertEqual(self.data_queue.size(), 1) update_cell.delay().get() # the cell was inserted again cells = session.query(Cell).all() self.assertEqual(len(cells), 1) # and the creation date was set to the date of the blacklist entry self.assertEqual(cells[0].created, last_week) self.check_statcounter(StatKey.cell, 1) self.check_statcounter(StatKey.unique_cell, 0)
def test_created_from_blocklist_time(self): now = util.utcnow() last_week = now - TEMPORARY_BLOCKLIST_DURATION - timedelta(days=1) obs = CellObservationFactory.build() self.session.add( CellBlocklist(time=last_week, count=1, radio=obs.radio, mcc=obs.mcc, mnc=obs.mnc, lac=obs.lac, cid=obs.cid)) self.session.flush() # add a new entry for the previously blocklisted cell self.data_queue.enqueue([obs]) self.assertEqual(self.data_queue.size(), 1) update_cell.delay().get() # the cell was inserted again cells = self.session.query(Cell).all() self.assertEqual(len(cells), 1) # and the creation date was set to the date of the blocklist entry self.assertEqual(cells[0].created, last_week) self.check_statcounter(StatKey.cell, 1) self.check_statcounter(StatKey.unique_cell, 0)
def test_blacklist(self): now = util.utcnow() session = self.session observations = [dict(mcc=FRANCE_MCC, mnc=2, lac=3, cid=i, psc=5, radio=int(Radio.gsm), lat=PARIS_LAT + i * 0.0000001, lon=PARIS_LON + i * 0.0000001) for i in range(1, 4)] black = CellBlacklist( mcc=FRANCE_MCC, mnc=2, lac=3, cid=1, radio=Radio.gsm, time=now, count=1, ) session.add(black) session.flush() result = insert_measures_cell.delay(observations) self.assertEqual(result.get(), 2) self.assertEqual(self.data_queue.size(), 2) update_cell.delay().get() cells = session.query(Cell).all() self.assertEqual(len(cells), 2) self.check_statcounter(StatKey.cell, 2) self.check_statcounter(StatKey.unique_cell, 2)
def test_insert_observations_invalid_lac(self): time = util.utcnow() - timedelta(days=1) today = util.utcnow().date() cell = CellFactory(total_measures=5) self.session.add(Score(key=ScoreKey.new_cell, userid=1, time=today, value=7)) self.session.flush() obs = dict( radio=int(cell.radio), mcc=cell.mcc, mnc=cell.mnc, psc=cell.psc, created=time, time=time, lat=cell.lat, lon=cell.lon, accuracy=0, altitude=0, altitude_accuracy=0) entries = [ {'lac': constants.MAX_LAC_ALL + 1, 'cid': constants.MAX_CID_ALL + 1, 'asu': 8}, {'lac': None, 'cid': None, 'asu': 8}, ] for entry in entries: entry.update(obs) result = insert_measures_cell.delay(entries, userid=1) self.assertEqual(result.get(), 0) # The incomplete observations never make it into the queue self.assertEqual(self.data_queue.size(), 0) update_cell.delay().get() # Nothing should change in the initially created Cell record self.session.refresh(cell) cells = self.session.query(Cell).all() self.assertEqual(len(cells), 1) self._compare_sets([c.total_measures for c in cells], [5])
def test_stats(self): self.session.add(ApiKey(valid_key='e5444-794', log_submit=True)) self.session.flush() self.add_reports(3, email='secretemail@localhost', ip=self.geoip_data['London']['ip']) self.add_reports(6, api_key='e5444-794') self.add_reports(3, api_key=None) schedule_export_reports.delay().get() update_cell.delay().get() for i in range(16): update_wifi.delay(shard_id='%x' % i).get() self.check_stats(counter=[ ('data.export.batch', 1, 1, ['key:internal']), ('data.report.upload', 2, 3), ('data.report.upload', 1, 3, ['key:test']), ('data.report.upload', 1, 6, ['key:e5444-794']), ('data.observation.insert', 1, 12, ['type:cell']), ('data.observation.upload', 1, 3, ['type:cell', 'key:test']), ('data.observation.upload', 1, 6, ['type:wifi', 'key:test']), ('data.observation.upload', 0, ['type:cell', 'key:no_key']), ('data.observation.upload', 1, 6, ['type:cell', 'key:e5444-794']), ('data.observation.upload', 1, 12, ['type:wifi', 'key:e5444-794']), ]) # we get a variable number of statsd messages and are only # interested in the sum-total insert_msgs = [msg for msg in self.stats_client.msgs if (msg.startswith('data.observation.insert') and 'type:wifi' in msg)] self.assertEqual( sum([int(msg.split(':')[1].split('|')[0]) for msg in insert_msgs]), 24)
def _update_all(self): schedule_export_reports.delay().get() for shard_id in CellShard.shards().keys(): update_cell.delay(shard_id=shard_id).get() for shard_id in WifiShard.shards().keys(): update_wifi.delay(shard_id=shard_id).get()
def _update_all(self): schedule_export_reports.delay().get() for shard_id in CellShard.shards().keys(): update_cell.delay(shard_id=shard_id).get() for shard_id in WifiShard.shards().keys(): update_wifi.delay(shard_id=shard_id).get()
def _queue_and_update_cell(self, obs): sharded_obs = defaultdict(list) for ob in obs: sharded_obs[CellShard.shard_id(ob.cellid)].append(ob) for shard_id, values in sharded_obs.items(): queue = self.celery_app.data_queues['update_cell_' + shard_id] queue.enqueue([value.to_json() for value in values]) update_cell.delay(shard_id=shard_id).get()
def _queue_and_update(self, obs): sharded_obs = defaultdict(list) for ob in obs: sharded_obs[CellShard.shard_id(ob.cellid)].append(ob) for shard_id, values in sharded_obs.items(): queue = self.celery_app.data_queues['update_cell_' + shard_id] queue.enqueue(values) update_cell.delay(shard_id=shard_id).get()
def test_stats(self): self.add_reports(3, api_key='test') schedule_export_reports.delay().get() update_cell.delay().get() update_wifi.delay().get() self.check_stats( counter=[('items.api_log.test.uploaded.reports', 1, 3), ('items.uploaded.reports', 1, 3)])
def test_insert_observations(self): session = self.session time = util.utcnow() - timedelta(days=1) mcc = FRANCE_MCC session.add(Cell(radio=Radio.gsm, mcc=mcc, mnc=2, lac=3, cid=4, psc=5, total_measures=5)) user = User(nickname=u'test') session.add(user) session.flush() obs = dict( created=time, lat=PARIS_LAT, lon=PARIS_LON, time=time, accuracy=0, altitude=0, altitude_accuracy=0, radio=int(Radio.gsm), ) entries = [ # Note that this first entry will be skipped as it does # not include (lac, cid) or (psc) {'mcc': mcc, 'mnc': 2, 'signal': -100}, {'mcc': mcc, 'mnc': 2, 'lac': 3, 'cid': 4, 'psc': 5, 'asu': 8}, {'mcc': mcc, 'mnc': 2, 'lac': 3, 'cid': 4, 'psc': 5, 'asu': 8}, {'mcc': mcc, 'mnc': 2, 'lac': 3, 'cid': 4, 'psc': 5, 'asu': 15}, {'mcc': mcc, 'mnc': 2, 'lac': 3, 'cid': 7, 'psc': 5}, ] for entry in entries: entry.update(obs) result = insert_measures_cell.delay(entries, userid=user.id) self.assertEqual(result.get(), 4) self.assertEqual(self.data_queue.size(), 4) update_cell.delay().get() cells = session.query(Cell).all() self.assertEqual(len(cells), 2) self._compare_sets([c.mcc for c in cells], [mcc]) self._compare_sets([c.mnc for c in cells], [2]) self._compare_sets([c.lac for c in cells], [3]) self._compare_sets([c.cid for c in cells], [4, 7]) self._compare_sets([c.psc for c in cells], [5]) self._compare_sets([c.total_measures for c in cells], [1, 8]) score_queue = self.celery_app.data_queues['update_score'] scores = score_queue.dequeue() self.assertEqual(len(scores), 1) score = scores[0] self.assertEqual(score['hashkey'].userid, user.id) self.assertEqual(score['hashkey'].key, ScoreKey.new_cell) self.assertEqual(score['value'], 1) self.check_statcounter(StatKey.cell, 4) self.check_statcounter(StatKey.unique_cell, 1)
def test_cell_invalid(self): self.add_reports(cell_factor=1, wifi_factor=0, cell_mcc=-2) schedule_export_reports.delay().get() update_cell.delay().get() self.assertEqual(self.session.query(Cell).count(), 0) self.check_stats(counter=[ ('data.report.upload', 1, 1, ['key:test']), ('data.report.drop', 1, 1, ['reason:malformed', 'key:test']), ('data.observation.drop', 1, 1, ['type:cell', 'reason:malformed', 'key:test']), ])
def _update_all(self): update_incoming.delay().get() for shard_id in BlueShard.shards().keys(): update_blue.delay(shard_id=shard_id).get() for shard_id in CellShard.shards().keys(): update_cell.delay(shard_id=shard_id).get() for shard_id in WifiShard.shards().keys(): update_wifi.delay(shard_id=shard_id).get()
def test_insert_observations(self): time = util.utcnow() - timedelta(days=1) cell = CellFactory(radio=Radio.gsm, total_measures=5) user = User(nickname=u'test') self.session.add(user) self.session.flush() obs = dict( radio=int(cell.radio), mcc=cell.mcc, mnc=cell.mnc, created=time, time=time, lat=cell.lat, lon=cell.lon, accuracy=0, altitude=0, altitude_accuracy=0, ) entries = [ # Note that this first entry will be skipped as it does # not include (lac, cid) or (psc) {'signal': -100}, {'lac': cell.lac, 'cid': cell.cid, 'psc': cell.psc, 'asu': 8}, {'lac': cell.lac, 'cid': cell.cid, 'psc': cell.psc, 'asu': 8}, {'lac': cell.lac, 'cid': cell.cid, 'psc': cell.psc, 'asu': 15}, {'lac': cell.lac, 'cid': cell.cid + 1, 'psc': cell.psc}, ] for entry in entries: entry.update(obs) result = insert_measures_cell.delay(entries, userid=user.id) self.assertEqual(result.get(), 4) self.assertEqual(self.data_queue.size(), 4) update_cell.delay().get() self.session.refresh(cell) cells = self.session.query(Cell).all() self.assertEqual(len(cells), 2) self._compare_sets([c.mcc for c in cells], [cell.mcc]) self._compare_sets([c.mnc for c in cells], [cell.mnc]) self._compare_sets([c.lac for c in cells], [cell.lac]) self._compare_sets([c.cid for c in cells], [cell.cid, cell.cid + 1]) self._compare_sets([c.psc for c in cells], [cell.psc]) self._compare_sets([c.total_measures for c in cells], [1, 8]) score_queue = self.celery_app.data_queues['update_score'] scores = score_queue.dequeue() self.assertEqual(len(scores), 1) score = scores[0] self.assertEqual(score['hashkey'].userid, user.id) self.assertEqual(score['hashkey'].key, ScoreKey.new_cell) self.assertEqual(score['value'], 1) self.check_statcounter(StatKey.cell, 4) self.check_statcounter(StatKey.unique_cell, 1)
def test_shard_queues(self): # BBB observations = CellObservationFactory.build_batch(3) data_queues = self.celery_app.data_queues single_queue = data_queues['update_cell'] single_queue.enqueue(observations) update_cell.delay().get() self.assertEqual(single_queue.size(), 0) total = 0 for shard_id in CellShard.shards().keys(): total += data_queues['update_cell_' + shard_id].size() self.assertEqual(total, 3)
def test_shard_queues(self): # BBB observations = CellObservationFactory.build_batch(3) data_queues = self.celery_app.data_queues single_queue = data_queues['update_cell'] single_queue.enqueue(observations) update_cell.delay().get() self.assertEqual(single_queue.size(), 0) total = 0 for shard_id in CellShard.shards().keys(): total += data_queues['update_cell_' + shard_id].size() self.assertEqual(total, 3)
def _update_all(self, session, datamap_only=False): ExportConfigFactory(name="internal", batch=0, schema="internal") session.flush() update_incoming.delay().get() if datamap_only: return for shard_id in BlueShard.shards().keys(): update_blue.delay(shard_id=shard_id).get() for shard_id in CellShard.shards().keys(): update_cell.delay(shard_id=shard_id).get() for shard_id in WifiShard.shards().keys(): update_wifi.delay(shard_id=shard_id).get()
def test_max_min_range_update(self): k1 = dict(radio=Radio.cdma, mcc=1, mnc=2, lac=3, cid=4) self.session.add( Cell(lat=1.001, lon=-1.001, max_lat=1.002, min_lat=1.0, max_lon=-1.0, min_lon=-1.002, total_measures=3, **k1)) observations = [ CellObservation(lat=1.001, lon=-1.003, **k1), CellObservation(lat=1.005, lon=-1.007, **k1), ] self.data_queue.enqueue(observations) self.session.commit() result = update_cell.delay() self.assertEqual(result.get(), (1, 0)) cells = self.session.query(Cell).all() self.assertEqual(len(cells), 1) cell = cells[0] self.assertAlmostEqual(cell.lat, 1.0018, 6) self.assertEqual(cell.max_lat, 1.005) self.assertEqual(cell.min_lat, 1.0) self.assertAlmostEqual(cell.lon, -1.0026, 6) self.assertEqual(cell.max_lon, -1.0) self.assertEqual(cell.min_lon, -1.007) self.assertEqual(cell.range, 605)
def _update_all(self, session, datamap_only=False): ExportConfigFactory(name='internal', batch=0, schema='internal') session.flush() update_incoming.delay().get() if datamap_only: return for shard_id in BlueShard.shards().keys(): update_blue.delay(shard_id=shard_id).get() for shard_id in CellShard.shards().keys(): update_cell.delay(shard_id=shard_id).get() for shard_id in WifiShard.shards().keys(): update_wifi.delay(shard_id=shard_id).get()
def test_max_min_range_update(self): cell = CellFactory(range=150, total_measures=3) cell_lat = cell.lat cell_lon = cell.lon cell.max_lat = cell.lat + 0.001 cell.min_lat = cell.lat - 0.001 cell.max_lon = cell.lon + 0.001 cell.min_lon = cell.lon - 0.001 k1 = cell.hashkey().__dict__ observations = [ CellObservation(lat=cell.lat, lon=cell.lon - 0.002, **k1), CellObservation(lat=cell.lat + 0.004, lon=cell.lon - 0.006, **k1), ] self.data_queue.enqueue(observations) self.session.commit() self.assertEqual(update_cell.delay().get(), (1, 0)) cells = self.session.query(Cell).all() self.assertEqual(len(cells), 1) cell = cells[0] self.assertAlmostEqual(cell.lat, cell_lat + 0.0008) self.assertAlmostEqual(cell.max_lat, cell_lat + 0.004) self.assertAlmostEqual(cell.min_lat, cell_lat - 0.001) self.assertAlmostEqual(cell.lon, cell_lon - 0.0016) self.assertAlmostEqual(cell.max_lon, cell_lon + 0.001) self.assertAlmostEqual(cell.min_lon, cell_lon - 0.006) self.assertEqual(cell.range, 468) self.assertEqual(cell.total_measures, 5)
def add_line_of_cells_and_scan_lac(self): big = 1.0 small = big / 10 keys = dict(radio=Radio.cdma, mcc=1, mnc=1, lac=1) observations = [ CellObservation(lat=ctr + xd, lon=ctr + yd, cid=cell, **keys) for cell in range(10) for ctr in [cell * big] for (xd, yd) in [(small, small), (small, -small), (-small, small), (-small, -small)] ] self.obs_queue.enqueue(observations) cells = [ Cell(lat=ctr, lon=ctr, cid=cell, total_measures=1, **keys) for cell in range(10) for ctr in [cell * big] ] self.session.add_all(cells) self.session.commit() result = update_cell.delay() self.assertEqual(result.get(), (len(cells), 0)) scan_areas.delay().get()
def test_upload_duplicated_cell(self): self.add_reports(cell_factor=1, wifi_factor=0) # duplicate the cell entry inside the report queue = self.celery_app.export_queues['internal'] items = queue.dequeue(queue.queue_key()) report = items[0]['report'] cell = report['cellTowers'][0].copy() report['cellTowers'].append(cell) report['cellTowers'][1]['signalStrength'] += 2 queue.enqueue(items, queue.queue_key()) schedule_export_reports.delay().get() update_cell.delay().get() cells = self.session.query(Cell).all() self.assertEqual(len(cells), 1) self.assertEqual(cells[0].total_measures, 1)
def test_update_cell(self): now = util.utcnow() invalid_key = dict(lac=None, cid=None) observations = [] def obs_factory(**kw): obs = CellObservationFactory.build(**kw) observations.append(obs) cell1 = CellFactory(total_measures=3) lat1, lon1 = (cell1.lat, cell1.lon) key1 = dict(lac=cell1.lac, cid=cell1.cid) obs_factory(lat=lat1, lon=lon1, created=now, **key1) obs_factory(lat=lat1 + 0.004, lon=lon1 + 0.006, created=now, **key1) obs_factory(lat=lat1 + 0.006, lon=lon1 + 0.009, created=now, **key1) # The lac, cid are invalid and should be skipped obs_factory(created=now, **invalid_key) obs_factory(created=now, **invalid_key) cell2 = CellFactory(lat=lat1 + 1.0, lon=lon1 + 1.0, total_measures=3) lat2, lon2 = (cell2.lat, cell2.lon) key2 = dict(lac=cell2.lac, cid=cell2.cid) obs_factory(lat=lat2 + 0.001, lon=lon2 + 0.002, created=now, **key2) obs_factory(lat=lat2 + 0.003, lon=lon2 + 0.006, created=now, **key2) cell3 = CellFactory(total_measures=100000) lat3, lon3 = (cell3.lat, cell3.lon) for i in range(10): obs_factory(lat=lat3 + 1.0, lon=lon3 + 1.0, **dict(lac=cell3.lac, cid=cell3.cid)) self.data_queue.enqueue(observations) self.session.commit() result = update_cell.delay() self.assertEqual(result.get(), (3, 0)) self.check_stats( timer=['task.data.update_cell'], ) cells = self.session.query(Cell).all() self.assertEqual(len(cells), 3) for cell in cells: if cell.hashkey() == cell1.hashkey(): self.assertAlmostEqual(cell.lat, lat1 + 0.001667, 6) self.assertAlmostEqual(cell.lon, lon1 + 0.0025, 6) if cell.hashkey() == cell2.hashkey(): self.assertAlmostEqual(cell.lat, lat2 + 0.0008, 6) self.assertAlmostEqual(cell.lon, lon2 + 0.0016, 6) if cell.hashkey() == cell3.hashkey(): expected_lat = ((lat3 * 1000) + (lat3 + 1.0) * 10) / 1010 expected_lon = ((lon3 * 1000) + (lon3 + 1.0) * 10) / 1010 self.assertAlmostEqual(cell.lat, expected_lat, 7) self.assertAlmostEqual(cell.lon, expected_lon, 7)
def test_upload_cell(self): reports = self.add_reports(cell_factor=1, wifi_factor=0) schedule_export_reports.delay().get() update_cell.delay().get() position = reports[0]['position'] cell_data = reports[0]['cellTowers'][0] cells = self.session.query(Cell).all() self.assertEqual(len(cells), 1) cell = cells[0] self.assertEqual(cell.lat, position['latitude']) self.assertEqual(cell.lon, position['longitude']) self.assertEqual(cell.radio.name, cell_data['radioType']) self.assertEqual(cell.mcc, cell_data['mobileCountryCode']) self.assertEqual(cell.mnc, cell_data['mobileNetworkCode']) self.assertEqual(cell.lac, cell_data['locationAreaCode']) self.assertEqual(cell.cid, cell_data['cellId']) self.assertEqual(cell.psc, cell_data['primaryScramblingCode']) self.assertEqual(cell.total_measures, 1)
def test_blocklist(self): now = util.utcnow() observations = CellObservationFactory.build_batch(3) obs = observations[0] block = CellBlocklist( radio=obs.radio, mcc=obs.mcc, mnc=obs.mnc, lac=obs.lac, cid=obs.cid, time=now, count=1, ) self.session.add(block) self.session.flush() self.data_queue.enqueue(observations) self.assertEqual(self.data_queue.size(), 3) update_cell.delay().get() cells = self.session.query(Cell).all() self.assertEqual(len(cells), 2) self.check_statcounter(StatKey.cell, 2) self.check_statcounter(StatKey.unique_cell, 2)
def test_insert_observations_invalid_lac(self): session = self.session time = util.utcnow() - timedelta(days=1) today = util.utcnow().date() session.add(Cell(radio=Radio.gsm, mcc=FRANCE_MCC, mnc=2, lac=3, cid=4, total_measures=5)) session.add(Score(key=ScoreKey.new_cell, userid=1, time=today, value=7)) session.flush() obs = dict( created=time, lat=PARIS_LAT, lon=PARIS_LON, time=time, accuracy=0, altitude=0, altitude_accuracy=0, radio=int(Radio.gsm)) entries = [ {'mcc': FRANCE_MCC, 'mnc': 2, 'lac': constants.MAX_LAC_ALL + 1, 'cid': constants.MAX_CID_ALL + 1, 'psc': 5, 'asu': 8}, {'mcc': FRANCE_MCC, 'mnc': 2, 'lac': None, 'cid': None, 'psc': 5, 'asu': 8}, ] for entry in entries: entry.update(obs) result = insert_measures_cell.delay(entries, userid=1) self.assertEqual(result.get(), 0) # The incomplete observations never make it into the queue self.assertEqual(self.data_queue.size(), 0) update_cell.delay().get() # Nothing should change in the initially created Cell record cells = session.query(Cell).all() self.assertEqual(len(cells), 1) self._compare_sets([c.total_measures for c in cells], [5])
def test_upload(self): self.session.add(ApiKey(valid_key='e5444-794', log=True)) self.session.flush() self.add_reports(3, email='secretemail@localhost', ip=self.geoip_data['London']['ip']) self.add_reports(6, api_key='e5444-794') self.add_reports(3, api_key=None) schedule_export_reports.delay().get() update_cell.delay().get() update_wifi.delay().get() self.assertEqual(self.session.query(Cell).count(), 12) self.assertEqual(self.session.query(Wifi).count(), 24) self.check_stats(counter=[ ('items.export.internal.batches', 1, 1), ('items.api_log.test.uploaded.cell_observations', 1, 3), ('items.api_log.test.uploaded.wifi_observations', 1, 6), ('items.api_log.no_key.uploaded.cell_observations', 0), ('items.api_log.e5444-794.uploaded.cell_observations', 1, 6), ('items.api_log.e5444-794.uploaded.wifi_observations', 1, 12), ])
def test_scan_areas_race_with_update_cell(self): session = self.session # First batch of cell observations for CID 1 keys = dict(radio=Radio.cdma, mcc=1, mnc=1, lac=1, cid=1) cell = Cell(total_measures=1, **keys) observations = [ CellObservation(lat=1.0, lon=1.0, **keys), CellObservation(lat=1.0, lon=1.0, **keys), CellObservation(lat=1.0, lon=1.0, **keys), CellObservation(lat=1.0, lon=1.0, **keys), ] session.add(cell) self.obs_queue.enqueue(observations) session.commit() # Periodic update_cell runs and updates CID 1 # to have a location, inserts LAC 1 # which will be picked up by the next scan_lac. result = update_cell.delay() self.assertEqual(result.get(), (1, 0)) # Second batch of cell observations for CID 2 keys['cid'] = 2 cell = Cell(total_measures=1, **keys) observations = [ CellObservation(lat=1.0, lon=1.0, **keys), CellObservation(lat=1.0, lon=1.0, **keys), CellObservation(lat=1.0, lon=1.0, **keys), CellObservation(lat=1.0, lon=1.0, **keys), ] session.add(cell) self.obs_queue.enqueue(observations) session.commit() # Periodic LAC scan runs, picking up LAC 1; this could # accidentally pick up CID 2, but it should not since it # has not had its location updated yet. If there's no # exception here, CID 2 is properly ignored. scan_areas.delay()
def test_scan_areas_asymmetric(self): session = self.session big = 0.1 small = big / 10 keys = dict(radio=Radio.cdma, mcc=1, mnc=1, lac=1) observations = [ CellObservation(lat=ctr + xd, lon=ctr + yd, cid=cell, **keys) for cell in range(6) for ctr in [(2 ** cell) * big] for (xd, yd) in [(small, small), (small, -small), (-small, small), (-small, -small)] ] self.obs_queue.enqueue(observations) cells = [ Cell(lat=ctr, lon=ctr, cid=cell, total_measures=1, **keys) for cell in range(6) for ctr in [(2 ** cell) * big] ] session.add_all(cells) session.commit() result = update_cell.delay() self.assertEqual(result.get(), (len(cells), 0)) scan_areas.delay() lac = session.query(CellArea).filter(CellArea.lac == 1).first() # We produced a sequence of 0.02-degree-on-a-side # cell bounding boxes centered at # [0, 0.2, 0.4, 0.8, 1.6, 3.2] degrees. # So the lower-left corner is at (-0.01, -0.01) # and the upper-right corner is at (3.21, 3.21) # we should therefore see a LAC centroid at (1.05, 1.05) # with a range of 339.540m self.assertEqual(lac.lat, 1.05) self.assertEqual(lac.lon, 1.05) self.assertEqual(lac.range, 339540)
def test_upload_invalid_position(self): self.add_reports(1, cell_factor=1, wifi_factor=0, lat=-90.1) self.add_reports(1, cell_factor=1, wifi_factor=0) schedule_export_reports.delay().get() update_cell.delay().get() self.assertEqual(self.session.query(Cell).count(), 1)
def test_upload_invalid_cell(self): self.add_reports(cell_factor=1, wifi_factor=0, cell_mcc=-2) schedule_export_reports.delay().get() update_cell.delay().get() self.assertEqual(self.session.query(Cell).count(), 0)
def test_blacklist_moving_cells(self): now = util.utcnow() k1 = dict(radio=Radio.cdma, mcc=1, mnc=2, lac=3, cid=4) k2 = dict(radio=Radio.cdma, mcc=1, mnc=2, lac=6, cid=8) k3 = dict(radio=Radio.cdma, mcc=1, mnc=2, lac=9, cid=12) k4 = dict(radio=Radio.cdma, mcc=1, mnc=2, lac=12, cid=16) k5 = dict(radio=Radio.cdma, mcc=1, mnc=2, lac=15, cid=20) # keys k2, k3 and k4 are expected to be detected as moving data = [ # a cell with an entry but no prior position Cell(total_measures=0, **k1), CellObservation(lat=1.001, lon=1.001, **k1), CellObservation(lat=1.002, lon=1.005, **k1), CellObservation(lat=1.003, lon=1.009, **k1), # a cell with a prior known position Cell(lat=2.0, lon=2.0, total_measures=1, **k2), CellObservation(lat=2.0, lon=2.0, **k2), CellObservation(lat=4.0, lon=2.0, **k2), # a cell with a very different prior position Cell(lat=1.0, lon=1.0, total_measures=1, **k3), CellObservation(lat=3.0, lon=3.0, **k3), CellObservation(lat=-3.0, lon=3.0, **k3), # another cell with a prior known position (and negative lat) Cell(lat=-4.0, lon=4.0, total_measures=1, **k4), CellObservation(lat=-4.0, lon=4.0, **k4), CellObservation(lat=-6.0, lon=4.0, **k4), # an already blacklisted cell CellBlacklist(time=now, count=1, **k5), CellObservation(lat=5.0, lon=5.0, **k5), CellObservation(lat=8.0, lon=5.0, **k5), ] observations = [] for obj in data: if isinstance(obj, CellObservation): observations.append(obj) else: self.session.add(obj) self.data_queue.enqueue(observations) self.session.commit() result = update_cell.delay() self.assertEqual(result.get(), (4, 3)) moving = [k2, k3, k4, k5] black = self.session.query(CellBlacklist).all() self.assertEqual(set([b.hashkey() for b in black]), set([CellBlacklist.to_hashkey(k) for k in moving])) # test duplicate call result = update_cell.delay() self.assertEqual(result.get(), (0, 0)) self.check_stats( timer=[ # We made duplicate calls ('task.data.update_cell', 2), # One of those would've scheduled a remove_cell task ('task.data.remove_cell', 1) ])