def test_remove(self): area = CellAreaFactory() self.session.flush() self.area_queue.enqueue([area.hashkey()]) self.assertEqual(scan_areas.delay().get(), 1) self.assertEqual(self.session.query(CellArea).count(), 0)
def test_update(self, celery, session): area = CellAreaFactory(radio=Radio.gsm, num_cells=1) area.region = None BlueShardFactory.create_batch(2, region='CA') BlueShardFactory.create_batch(3, region='GB') CellAreaFactory(radio=Radio.gsm, region='DE', num_cells=1) CellAreaFactory(radio=Radio.gsm, region='DE', num_cells=2) CellAreaFactory(radio=Radio.gsm, region='CA', num_cells=2) CellAreaFactory(radio=Radio.wcdma, region='DE', num_cells=3) CellAreaFactory(radio=Radio.lte, region='CA', num_cells=4) WifiShardFactory.create_batch(5, region='DE') WifiShardFactory.create_batch(6, region='US') wifi = WifiShardFactory() wifi.region = None session.add(RegionStat(region='US', blue=1, wifi=2)) session.add(RegionStat(region='TW', wifi=1)) session.flush() update_statregion.delay().get() stats = session.query(RegionStat).all() assert len(stats) == 4 for stat in stats: values = (stat.gsm, stat.wcdma, stat.lte, stat.blue, stat.wifi) if stat.region == 'DE': assert values == (3, 3, 0, 0, 5) elif stat.region == 'CA': assert values == (2, 0, 4, 2, 0) elif stat.region == 'GB': assert values == (0, 0, 0, 3, 0) elif stat.region == 'US': assert values == (0, 0, 0, 0, 6)
def test_update(self): area = CellAreaFactory(radio=Radio.gsm, num_cells=1) area.region = None CellAreaFactory(radio=Radio.gsm, region='DE', num_cells=1) CellAreaFactory(radio=Radio.gsm, region='DE', num_cells=2) CellAreaFactory(radio=Radio.gsm, region='CA', num_cells=2) CellAreaFactory(radio=Radio.wcdma, region='DE', num_cells=3) CellAreaFactory(radio=Radio.lte, region='CA', num_cells=4) WifiShardFactory.create_batch(5, region='DE') WifiShardFactory.create_batch(6, region='US') wifi = WifiShardFactory() wifi.region = None self.session.add(RegionStat(region='US', wifi=2)) self.session.add(RegionStat(region='TW', wifi=1)) self.session.flush() update_statregion.delay().get() stats = self.session.query(RegionStat).all() self.assertEqual(len(stats), 3) for stat in stats: values = (stat.gsm, stat.wcdma, stat.lte, stat.wifi) if stat.region == 'DE': self.assertEqual(values, (3, 3, 0, 5)) elif stat.region == 'CA': self.assertEqual(values, (2, 0, 4, 0)) elif stat.region == 'US': self.assertEqual(values, (0, 0, 0, 6))
def test_smallest_area(self): area = CellAreaFactory(radius=25000) area2 = CellAreaFactory(radius=30000, lat=area.lat + 0.2) self.session.flush() query = self.model_query(cells=[area, area2]) result = self.source.search(query) self.check_model_result(result, area)
def test_scan_areas_remove(self): # create an orphaned lac entry area = CellAreaFactory() self.session.flush() self.area_queue.enqueue([area.hashkey()]) # after scanning the orphaned record gets removed self.assertEqual(scan_areas.delay().get(), 1) areas = self.session.query(CellArea).all() self.assertEqual(areas, [])
def test_smallest_area(self): now = util.utcnow() area = CellAreaFactory(radius=25000, num_cells=8) area2 = CellAreaFactory(radius=30000, lat=area.lat + 0.2, num_cells=6) self.session.flush() query = self.model_query(cells=[area, area2]) results = self.source.search(query) self.check_model_results(results, [area]) self.assertAlmostEqual(results.best().score, area.score(now), 4)
def test_smallest_area(self, geoip_db, http_session, session, source): now = util.utcnow() area = CellAreaFactory(radius=25000, num_cells=8) area2 = CellAreaFactory(radius=30000, lat=area.lat + 0.2, num_cells=6) session.flush() query = self.model_query(geoip_db, http_session, session, cells=[area, area2]) results = source.search(query) self.check_model_results(results, [area]) assert results.best().score == area_score(area, now)
def test_multiple_mcc(self): now = util.utcnow() region = GEOCODER.regions_for_mcc(235, metadata=True)[0] cell = CellAreaFactory(mcc=234, num_cells=6) cell2 = CellAreaFactory(mcc=235, num_cells=8) self.session.flush() query = self.model_query(cells=[cell, cell2]) results = self.source.search(query) self.assertTrue(len(results) > 2) best_result = results.best(query.expected_accuracy) self.assertEqual(best_result.region_code, region.code) self.assertAlmostEqual(best_result.score, 1.25 + cell.score(now), 4)
def test_multiple_mcc(self, geoip_db, http_session, session, source): now = util.utcnow() region = GEOCODER.regions_for_mcc(235, metadata=True)[0] area = CellAreaFactory(mcc=234, num_cells=6) area2 = CellAreaFactory(mcc=235, num_cells=8) session.flush() query = self.model_query(geoip_db, http_session, session, cells=[area, area2]) results = source.search(query) assert len(results) > 2 best_result = results.best() assert best_result.region_code == region.code assert best_result.score == 1.25 + area_score(area, now)
def test_multiple_mcc(self): now = util.utcnow() region = GEOCODER.regions_for_mcc(235, metadata=True)[0] cell = CellAreaFactory(mcc=234, num_cells=6) cell2 = CellAreaFactory(mcc=235, num_cells=8) self.session.flush() query = self.model_query(cells=[cell, cell2]) results = self.source.search(query) self.assertTrue(len(results) > 2) best_result = results.best() self.assertEqual(best_result.region_code, region.code) self.assertAlmostEqual(best_result.score, 1.25 + cell.score(now), 4)
def test_smallest_area(self, geoip_db, http_session, session, source, stats): now = util.utcnow() area = CellAreaFactory(radius=25000, num_cells=8) area2 = CellAreaFactory(radius=30000, lat=area.lat + 0.2, num_cells=6) session.flush() query = self.model_query( geoip_db, http_session, session, stats, cells=[area, area2]) results = source.search(query) self.check_model_results(results, [area]) assert results.best().score == area.score(now)
def test_scan_areas_remove(self): session = self.session redis_client = self.redis_client # create an orphaned lac entry area = CellAreaFactory() session.flush() enqueue_areas(session, redis_client, [area.hashkey()], UPDATE_KEY['cell_lac']) # after scanning the orphaned record gets removed self.assertEqual(scan_areas.delay().get(), 1) areas = session.query(CellArea).all() self.assertEqual(areas, [])
def test_scan_areas_remove(self): session = self.session redis_client = self.redis_client # create an orphaned lac entry area = CellAreaFactory() session.flush() redis_key = self.celery_app.data_queues['cell_area_update'] enqueue_areas(session, redis_client, [area.hashkey()], redis_key) # after scanning the orphaned record gets removed self.assertEqual(scan_areas.delay().get(), 1) areas = session.query(CellArea).all() self.assertEqual(areas, [])
def test_cellarea_with_different_fallback(self, app, session, metricsmock): cell = CellAreaFactory() session.flush() query = self.model_query(cells=[cell]) query["fallbacks"] = {"ipf": True} res = self._call(app, body=query) self.check_model_response(res, cell, fallback="lacf") assert metricsmock.has_record( "incr", "request", value=1, tags=[self.metric_path, "method:post", "status:200"], ) assert metricsmock.has_record( "incr", self.metric_type + ".request", value=1, tags=[self.metric_path, "key:test"], ) assert metricsmock.has_record( "incr", self.metric_type + ".result", value=1, tags=[ "key:test", "region:none", "fallback_allowed:false", "accuracy:low", "status:hit", "source:internal", ], )
def test_cellarea_with_lacf(self, app, session, stats): cell = CellAreaFactory() session.flush() query = self.model_query(cells=[cell]) query['fallbacks'] = {'lacf': True} res = self._call(app, body=query) self.check_model_response(res, cell, fallback='lacf') stats.check(counter=[ ('request', [self.metric_path, 'method:post', 'status:200']), (self.metric_type + '.request', [self.metric_path, 'key:test']), (self.metric_type + '.query', [ 'key:test', 'region:none', 'geoip:false', 'blue:none', 'cell:none', 'wifi:none' ]), (self.metric_type + '.result', [ 'key:test', 'region:none', 'fallback_allowed:false', 'accuracy:low', 'status:hit', 'source:internal' ]), (self.metric_type + '.source', [ 'key:test', 'region:none', 'source:internal', 'accuracy:low', 'status:hit' ]), ])
def test_cellarea_without_lacf(self, app, data_queues, session, metricsmock): cell = CellAreaFactory() session.flush() query = self.model_query(cells=[cell]) query["fallbacks"] = {"lacf": False} res = self._call(app, body=query, status=self.not_found.code) self.check_response(data_queues, res, "not_found") assert metricsmock.has_record( "incr", "request", value=1, tags=[ self.metric_path, "method:post", "status:%s" % self.not_found.code ], ) assert metricsmock.has_record( "incr", self.metric_type + ".request", value=1, tags=[self.metric_path, "key:test"], )
def test_update(self): area = CellAreaFactory(num_cells=2, range=500, avg_cell_range=100) area_key = area.hashkey() cell = CellFactory( lat=area.lat, lon=area.lon, range=200, **area_key.__dict__) self.session.commit() self.area_queue.enqueue([area_key]) self.assertEqual(scan_areas.delay().get(), 1) area = self.session.query(CellArea).one() self.assertAlmostEqual(area.lat, cell.lat) self.assertAlmostEqual(area.lon, cell.lon) self.assertEqual(area.range, 0) self.assertEqual(area.num_cells, 1) self.assertEqual(area.avg_cell_range, 200)
def test_ambiguous_mcc(self, geoip_db, http_session, session, source, stats): now = util.utcnow() regions = GEOCODER.regions_for_mcc(234, metadata=True) area = CellAreaFactory(mcc=234, num_cells=10) session.flush() query = self.model_query(geoip_db, http_session, session, stats, cells=[area]) results = source.search(query) self.check_model_results(results, regions) assert results.best().region_code == 'GB' for result in results: score = 0.25 if result.region_code == 'GB': score += area_score(area, now) assert result.score == score stats.check(counter=[ (self.api_type + '.source', [ 'key:test', 'region:none', 'source:internal', 'accuracy:low', 'status:hit' ]), ])
def test_cell_area_region_no_fallback(self): cell = CellAreaFactory.build() cell_query = self.cell_model_query([cell]) query = Query(cell=cell_query, api_type='region', fallback={'lacf': False}) assert query.expected_accuracy is DataAccuracy.none
def test_ambiguous_mcc(self, geoip_db, http_session, session, source, metricsmock): now = util.utcnow() regions = GEOCODER.regions_for_mcc(234, metadata=True) area = CellAreaFactory(mcc=234, num_cells=10) session.flush() query = self.model_query(geoip_db, http_session, session, cells=[area]) results = source.search(query) self.check_model_results(results, regions) assert results.best().region_code == "GB" for result in results: score = 0.25 if result.region_code == "GB": score += area_score(area, now) assert result.score == score assert metricsmock.has_record( "incr", self.api_type + ".source", value=1, tags=[ "key:test", "region:none", "source:internal", "accuracy:low", "status:hit", ], )
def test_cell_area_no_fallback(self): cell = CellAreaFactory.build() cell_query = self.cell_model_query([cell]) query = Query(cell=cell_query, fallback={'lacf': False}) self.assertEqual(len(query.cell), 0) self.assertEqual(len(query.cell_area), 0) self.assertEqual(query.expected_accuracy, DataAccuracy.none)
def test_update_incomplete_cell(self): area = CellAreaFactory(range=500) area_key = area.hashkey() cell = CellFactory( lat=area.lat + 0.0002, lon=area.lon, **area_key.__dict__) CellFactory(lat=None, lon=None, **area_key.__dict__) CellFactory(lat=area.lat, lon=area.lon, max_lat=None, min_lon=None, **area_key.__dict__) self.session.commit() self.area_queue.enqueue([area_key]) self.assertEqual(scan_areas.delay().get(), 1) area = self.session.query(CellArea).one() self.assertAlmostEqual(area.lat, cell.lat - 0.0001) self.assertAlmostEqual(area.lon, cell.lon) self.assertEqual(area.num_cells, 2)
def test_cell_area(self): cell = CellAreaFactory.build() cell_query = self.cell_model_query([cell]) query = Query(cell=cell_query) self.assertEqual(len(query.cell), 0) self.assertEqual(len(query.cell_area), 1) self.assertEqual(query.expected_accuracy, DataAccuracy.low)
def test_cell_area_no_fallback(self): cell = CellAreaFactory.build() cell_query = self.cell_model_query([cell]) query = Query(cell=cell_query, fallback={'lacf': False}) assert len(query.cell) == 0 assert len(query.cell_area) == 0 assert query.expected_accuracy is DataAccuracy.none
def test_cell_area(self): cell = CellAreaFactory.build() cell_query = self.cell_model_query([cell]) query = Query(cell=cell_query) assert len(query.cell) == 0 assert len(query.cell_area) == 1 assert query.expected_accuracy is DataAccuracy.low
def test_cell_area(self): cell = CellAreaFactory.build() cell_query = self.cell_model_query([cell]) query = Query(cell=cell_query) assert len(query.cell) == 0 assert len(query.cell_area) == 1 assert query.expected_accuracy is DataAccuracy.low assert query.geoip_only is False
def test_invalid_mcc(self, geoip_db, http_session, session, source, stats): area = CellAreaFactory.build(mcc=235, num_cells=10) area.mcc = 999 query = self.model_query( geoip_db, http_session, session, stats, cells=[area]) results = source.search(query) self.check_model_results(results, None)
def test_invalid_mcc(self, geoip_db, http_session, session, source, stats): area = CellAreaFactory.build(mcc=235, num_cells=10) area.mcc = 999 query = self.model_query(geoip_db, http_session, session, stats, cells=[area]) results = source.search(query) self.check_model_results(results, None)
def test_minimum_range_returned(self): areas = CellAreaFactory.create_batch(2) areas[0].range = LAC_MIN_ACCURACY - 2000 areas[1].range = LAC_MIN_ACCURACY + 3000 areas[1].lat = areas[0].lat + 0.2 self.session.flush() query = self.model_query(cells=areas) location = self.provider.locate(query) self.check_model_location(location, areas[0], accuracy=LAC_MIN_ACCURACY)
def test_minimum_radius(self): areas = CellAreaFactory.create_batch(2) areas[0].radius = CELLAREA_MIN_ACCURACY - 2000 areas[1].radius = CELLAREA_MIN_ACCURACY + 3000 areas[1].lat = areas[0].lat + 0.2 self.session.flush() query = self.model_query(cells=areas) results = self.source.search(query) self.check_model_results( results, [areas[0]], accuracy=CELLAREA_MIN_ACCURACY)
def test_ambiguous_mcc(self): now = util.utcnow() regions = GEOCODER.regions_for_mcc(234, metadata=True) cell = CellAreaFactory(mcc=234, num_cells=10) self.session.flush() query = self.model_query(cells=[cell]) results = self.source.search(query) self.check_model_results(results, regions) self.assertEqual(results.best().region_code, 'GB') for result in results: score = 0.25 if result.region_code == 'GB': score += cell.score(now) self.assertAlmostEqual(result.score, score, 4) self.check_stats(counter=[ (self.api_type + '.source', ['key:test', 'region:none', 'source:internal', 'accuracy:low', 'status:hit']), ])
def test_minimum_radius(self, geoip_db, http_session, session, source): areas = CellAreaFactory.create_batch(2) areas[0].radius = CELLAREA_MIN_ACCURACY - 2000 areas[1].radius = CELLAREA_MIN_ACCURACY + 3000 areas[1].lat = areas[0].lat + 0.2 session.flush() query = self.model_query(geoip_db, http_session, session, cells=areas) results = source.search(query) self.check_model_results(results, [areas[0]], accuracy=CELLAREA_MIN_ACCURACY)
def test_incomplete_keys(self): cells = CellAreaFactory.build_batch(4) cells[0].radio = None cells[1].mcc = None cells[2].mnc = None cells[3].lac = None with self.db_call_checker() as check_db_calls: query = self.model_query(cells=cells) results = self.source.result_list() self.assertFalse(self.source.should_search(query, results)) check_db_calls(rw=0, ro=0)
def test_minimum_radius(self): areas = CellAreaFactory.create_batch(2) areas[0].radius = CELLAREA_MIN_ACCURACY - 2000 areas[1].radius = CELLAREA_MIN_ACCURACY + 3000 areas[1].lat = areas[0].lat + 0.2 self.session.flush() query = self.model_query(cells=areas) result = self.source.search(query) self.check_model_result( result, areas[0], accuracy=CELLAREA_MIN_ACCURACY)
def test_incomplete_keys(self): cells = CellAreaFactory.build_batch(4) cells[0].radio = None cells[1].mcc = None cells[2].mnc = None cells[3].lac = None with self.db_call_checker() as check_db_calls: query = self.model_query(cells=cells) results = self.source.result_type().new_list() self.assertFalse(self.source.should_search(query, results)) check_db_calls(rw=0, ro=0)
def test_incomplete_keys(self, geoip_db, http_session, session_tracker, session, source): cells = CellAreaFactory.build_batch(4) cells[0].radio = None cells[1].mcc = None cells[2].mnc = None cells[3].lac = None query = self.model_query(geoip_db, http_session, session, cells=cells) results = source.result_list() assert not source.should_search(query, results) session_tracker(0)
def test_minimum_range(self): areas = CellAreaFactory.create_batch(2) areas[0].range = LAC_MIN_ACCURACY - 2000 areas[1].range = LAC_MIN_ACCURACY + 3000 areas[1].lat = areas[0].lat + 0.2 self.session.flush() query = self.model_query(cells=areas) result = self.source.search(query) self.check_model_result( result, areas[0], accuracy=LAC_MIN_ACCURACY)
def test_incomplete_keys(self, geoip_db, http_session, session_tracker, session, source, stats): cells = CellAreaFactory.build_batch(4) cells[0].radio = None cells[1].mcc = None cells[2].mnc = None cells[3].lac = None query = self.model_query( geoip_db, http_session, session, stats, cells=cells) results = source.result_list() assert not source.should_search(query, results) session_tracker(0)
def test_cellarea_without_lacf(self, app, data_queues, session, stats): cell = CellAreaFactory() session.flush() query = self.model_query(cells=[cell]) query['fallbacks'] = {'lacf': False} res = self._call(app, body=query, status=self.not_found.code) self.check_response(data_queues, res, 'not_found') stats.check(counter=[ ('request', [self.metric_path, 'method:post', 'status:%s' % self.not_found.code]), (self.metric_type + '.request', [self.metric_path, 'key:test']), ])
def test_from_mcc(self): region = GEOCODER.regions_for_mcc(235, metadata=True)[0] cell = CellAreaFactory(mcc=235, num_cells=10) self.session.flush() query = self.model_query(cells=[cell]) results = self.source.search(query) self.check_model_results(results, [region]) self.assertAlmostEqual(results[0].score, 1.0, 4) self.check_stats(counter=[ (self.api_type + '.source', ['key:test', 'region:none', 'source:internal', 'accuracy:low', 'status:hit']), ])
def test_from_mcc(self, geoip_db, http_session, session, source, metricsmock): region = GEOCODER.regions_for_mcc(235, metadata=True)[0] area = CellAreaFactory(mcc=235, num_cells=10) session.flush() query = self.model_query(geoip_db, http_session, session, cells=[area]) results = source.search(query) self.check_model_results(results, [region]) assert results[0].score == 1.0 metricsmock.assert_incr_once( self.api_type + ".source", tags=["key:test", "source:internal", "accuracy:low", "status:hit"], )
def test_cellarea_with_different_fallback(self): cell = CellAreaFactory() self.session.flush() query = self.model_query(cells=[cell]) query['fallbacks'] = {'ipf': True} res = self._call(body=query) self.check_model_response(res, cell, fallback='lacf') self.check_stats(counter=[ ('request', [self.metric_path, 'method:post', 'status:200']), (self.metric_type + '.request', [self.metric_path, 'key:test']), (self.metric_type + '.result', ['key:test', 'region:none', 'fallback_allowed:false', 'accuracy:low', 'status:hit', 'source:internal']), ])
def test_cellarea(self, app, session, metricsmock, fallback, logs): """ A unknown cell in a known cell area can be a hit, with fallback enabled. The cell location area fallback (lacf) is on by default, or can be explicitly enabled. """ cell = CellAreaFactory() session.flush() query = self.model_query(cells=[cell]) if fallback == "explicit": query["fallbacks"] = {"lacf": True} elif fallback == "ipf": # Enabling IP fallback leaves lac fallback at default query["fallbacks"] = {"ipf": True} res = self._call(app, body=query) self.check_model_response(res, cell, fallback="lacf") metricsmock.assert_incr_once( "request", tags=[self.metric_path, "method:post", "status:200"]) metricsmock.assert_incr_once(self.metric_type + ".request", tags=[self.metric_path, "key:test"]) metricsmock.assert_incr_once( self.metric_type + ".query", tags=[ "key:test", "geoip:false", "blue:none", "cell:none", "wifi:none" ], ) metricsmock.assert_incr_once( self.metric_type + ".result", tags=[ "key:test", "fallback_allowed:false", "accuracy:low", "status:hit", "source:internal", ], ) metricsmock.assert_incr_once( self.metric_type + ".source", tags=["key:test", "source:internal", "accuracy:low", "status:hit"], ) assert logs.only_entry["cell"] == 1 assert logs.only_entry["cell_valid"] == 0
def test_cellarea_without_lacf(self, app, data_queues, session, metricsmock, logs): """The cell location area fallback can be disabled.""" cell = CellAreaFactory() session.flush() query = self.model_query(cells=[cell]) query["fallbacks"] = {"lacf": False} res = self._call(app, body=query, status=404) self.check_response(data_queues, res, "not_found") metricsmock.assert_incr_once( "request", tags=[self.metric_path, "method:post", "status:404"]) metricsmock.assert_incr_once(self.metric_type + ".request", tags=[self.metric_path, "key:test"]) assert logs.only_entry["cell"] == 1 assert logs.only_entry["cell_valid"] == 0
def test_cell_area_region(self): cell = CellAreaFactory.build() cell_query = self.cell_model_query([cell]) query = Query(cell=cell_query, api_type='region') self.assertEqual(query.expected_accuracy, DataAccuracy.low)
def test_invalid_mcc(self): cell = CellAreaFactory.build(mcc=235, num_cells=10) cell.mcc = 999 query = self.model_query(cells=[cell]) results = self.source.search(query) self.check_model_results(results, None)