def test_database_error(self, db_errors=0): cells = [ CellShardFactory.build(radio=Radio.gsm), CellShardFactory.build(radio=Radio.wcdma), CellShardFactory.build(radio=Radio.lte), ] wifis = WifiShardFactory.build_batch(2) for model in (CellArea, CellOCID, CellAreaOCID): self.session.execute(text("drop table %s;" % model.__tablename__)) for name in set([cell.__tablename__ for cell in cells]): self.session.execute(text("drop table %s;" % name)) for name in set([wifi.__tablename__ for wifi in wifis]): self.session.execute(text("drop table %s;" % name)) query = self.model_query(cells=cells, wifis=wifis) res = self._call(body=query, ip=self.test_ip) self.check_response(res, "ok") self.check_stats( counter=[("request", [self.metric_path, "method:post", "status:200"])], timer=[("request", [self.metric_path, "method:post"])], ) if self.apikey_metrics: self.check_stats( counter=[ ( self.metric_type + ".result", ["key:test", "region:GB", "fallback_allowed:false", "accuracy:high", "status:miss"], ) ] ) self.check_raven([("ProgrammingError", db_errors)])
def test_database_error(self, app, data_queues, raven, session, stats, restore_db): cells = [ CellShardFactory.build(radio=Radio.gsm), CellShardFactory.build(radio=Radio.wcdma), CellShardFactory.build(radio=Radio.lte), ] wifis = WifiShardFactory.build_batch(2) for model in (CellArea, ): session.execute(text('drop table %s;' % model.__tablename__)) for name in set([cell.__tablename__ for cell in cells]): session.execute(text('drop table %s;' % name)) for name in set([wifi.__tablename__ for wifi in wifis]): session.execute(text('drop table %s;' % name)) query = self.model_query(cells=cells, wifis=wifis) res = self._call(app, body=query, ip=self.test_ip) self.check_response(data_queues, res, 'ok', fallback='ipf') self.check_queue(data_queues, 0) stats.check(counter=[ ('request', [self.metric_path, 'method:post', 'status:200']), ], timer=[ ('request', [self.metric_path, 'method:post']), ]) if self.apikey_metrics: stats.check(counter=[ (self.metric_type + '.result', ['key:test', 'region:GB', 'fallback_allowed:false', 'accuracy:high', 'status:miss']), ]) raven.check([('ProgrammingError', 3)])
def test_database_error(self, db_errors=0): cells = [ CellShardFactory.build(radio=Radio.gsm), CellShardFactory.build(radio=Radio.wcdma), CellShardFactory.build(radio=Radio.lte), ] wifis = WifiShardFactory.build_batch(2) for model in (CellArea, CellOCID, CellAreaOCID): self.session.execute(text('drop table %s;' % model.__tablename__)) for name in set([cell.__tablename__ for cell in cells]): self.session.execute(text('drop table %s;' % name)) for name in set([wifi.__tablename__ for wifi in wifis]): self.session.execute(text('drop table %s;' % name)) query = self.model_query(cells=cells, wifis=wifis) res = self._call(body=query, ip=self.test_ip) self.check_response(res, 'ok') self.check_stats(counter=[ ('request', [self.metric_path, 'method:post', 'status:200']), ], timer=[ ('request', [self.metric_path, 'method:post']), ]) if self.apikey_metrics: self.check_stats(counter=[ (self.metric_type + '.result', ['key:test', 'region:GB', 'fallback_allowed:false', 'accuracy:high', 'status:miss']), ]) self.check_raven([('ProgrammingError', db_errors)])
def test_cell(self): cell = CellShardFactory.build(radio=Radio.lte) query = Query(cell=[{ 'radioType': cell.radio, 'mobileCountryCode': cell.mcc, 'mobileNetworkCode': cell.mnc, 'locationAreaCode': cell.lac, 'cellId': cell.cid, 'asu': 17, 'primaryScramblingCode': 5, 'signalStrength': -70, 'timingAdvance': 15, 'unknown_field': 'foo' }]) data = self._call(query.json()) assert (data == { 'cells': [{ 'radio': cell.radio.name, 'mcc': cell.mcc, 'mnc': cell.mnc, 'lac': cell.lac, 'cid': cell.cid, 'asu': 17, 'psc': 5, 'signal': -70, 'tA': 15, }], 'fallbacks': { 'lacf': True }, 'token': None, })
def test_cell(self): cell = CellShardFactory.build(radio=Radio.lte) query = Query(cell=[{ 'radioType': cell.radio, 'mobileCountryCode': cell.mcc, 'mobileNetworkCode': cell.mnc, 'locationAreaCode': cell.lac, 'cellId': cell.cid, 'age': 1200, 'signalStrength': -70, 'timingAdvance': 15, 'unknown_field': 'foo' }]) data = self._call(query.json()) assert (data == { 'cellTowers': [{ 'radioType': cell.radio.name, 'mobileCountryCode': cell.mcc, 'mobileNetworkCode': cell.mnc, 'locationAreaCode': cell.lac, 'cellId': cell.cid, 'age': 1200, 'signalStrength': -70, 'timingAdvance': 15, }], 'considerIp': False, })
def test_cell(self): cell = CellShardFactory.build(radio=Radio.lte) cell_query = self.cell_model_query([cell]) query = Query(cell=cell_query) self.assertEqual(len(query.cell), 1) self.assertEqual(query.expected_accuracy, DataAccuracy.medium) query_cell = query.cell[0] for key, value in cell_query[0].items(): query_value = getattr(query_cell, key, None) if key == 'radio': self.assertEqual(query_value, cell.radio) else: self.assertEqual(query_value, value) self.assertEqual(len(query.cell_area), 1) query_area = query.cell_area[0] for key, value in cell_query[0].items(): query_value = getattr(query_area, key, None) if key == 'radio': self.assertEqual(query_value, cell.radio) elif key in ('cid', 'psc'): pass else: self.assertEqual(query_value, value)
def test_cell(self): cell = CellShardFactory.build(radio=Radio.lte) query = Query(cell=[ {'radioType': cell.radio, 'mobileCountryCode': cell.mcc, 'mobileNetworkCode': cell.mnc, 'locationAreaCode': cell.lac, 'cellId': cell.cid, 'age': 1200, 'signalStrength': -70, 'timingAdvance': 15, 'unknown_field': 'foo'}]) data = self._call(query.json()) assert (data == { 'cellTowers': [{ 'radioType': cell.radio.name, 'mobileCountryCode': cell.mcc, 'mobileNetworkCode': cell.mnc, 'locationAreaCode': cell.lac, 'cellId': cell.cid, 'age': 1200, 'signalStrength': -70, 'timingAdvance': 15, }], 'considerIp': False, })
def test_cell(self): cell = CellShardFactory.build(radio=Radio.lte) query = Query(cell=[{ 'radio': cell.radio, 'mcc': cell.mcc, 'mnc': cell.mnc, 'lac': cell.lac, 'cid': cell.cid, 'age': 1200, 'psc': 5, 'signal': -70, 'ta': 15, 'unknown_field': 'foo' }]) data = self.schema.deserialize(query.internal_query()) self.assertEqual( data, { 'cellTowers': [{ 'radioType': cell.radio.name, 'mobileCountryCode': cell.mcc, 'mobileNetworkCode': cell.mnc, 'locationAreaCode': cell.lac, 'cellId': cell.cid, 'primaryScramblingCode': 5, 'age': 1200, 'signalStrength': -70, 'timingAdvance': 15, }], 'fallbacks': { 'lacf': True }, })
def test_cell(self): cell = CellShardFactory.build(radio=Radio.lte) query = Query( cell=[ { "radioType": cell.radio, "mobileCountryCode": cell.mcc, "mobileNetworkCode": cell.mnc, "locationAreaCode": cell.lac, "cellId": cell.cid, "age": 1200, "signalStrength": -70, "timingAdvance": 15, "unknown_field": "foo", } ] ) data = self._call(query.json()) assert data == { "cellTowers": [ { "radioType": cell.radio.name, "mobileCountryCode": cell.mcc, "mobileNetworkCode": cell.mnc, "locationAreaCode": cell.lac, "cellId": cell.cid, "age": 1200, "signalStrength": -70, "timingAdvance": 15, } ], "considerIp": False, }
def test_cell(self): cell = CellShardFactory.build(radio=Radio.lte) query = Query(cell=[{ "radioType": cell.radio, "mobileCountryCode": cell.mcc, "mobileNetworkCode": cell.mnc, "locationAreaCode": cell.lac, "cellId": cell.cid, "asu": 17, "primaryScramblingCode": 5, "signalStrength": -70, "timingAdvance": 15, "unknown_field": "foo", }]) data = self._call(query.json()) assert data == { "cells": [{ "radio": cell.radio.name, "mcc": cell.mcc, "mnc": cell.mnc, "lac": cell.lac, "cid": cell.cid, "asu": 17, "psc": 5, "signal": -70, "tA": 15, }], "fallbacks": { "lacf": True }, "token": None, }
def test_dont_recache(self, geoip_db, http_session, session, source, stats): cell = CellShardFactory.build() mock_redis_client = self._mock_redis_client() mock_redis_client.mget.return_value = [self.fallback_cached_result] with requests_mock.Mocker() as mock_request: mock_request.register_uri('POST', requests_mock.ANY, json=self.fallback_result) with mock.patch.object(source.cache, 'redis_client', mock_redis_client): query = self.model_query(geoip_db, http_session, session, stats, cells=[cell]) results = source.search(query) self.check_model_results(results, [self.fallback_model]) assert mock_redis_client.mget.called assert not mock_redis_client.mset.called stats.check(counter=[ ('locate.fallback.cache', ['status:hit']), ])
def test_cache_single_cell(self): cell = CellShardFactory.build() with requests_mock.Mocker() as mock_request: mock_request.register_uri( 'POST', requests_mock.ANY, json=self.fallback_result) query = self.model_query(cells=[cell]) query.cell[0].signal = -77 result = self.source.search(query) self.check_model_result(result, self.fallback_model) self.assertEqual(mock_request.call_count, 1) self.check_stats(counter=[ ('locate.fallback.cache', ['status:miss']), ('locate.fallback.lookup', ['status:200']), ], timer=[ 'locate.fallback.lookup', ]) # vary the signal strength, not part of cache key query.cell[0].signal = -82 result = self.source.search(query) self.check_model_result(result, self.fallback_model) self.assertEqual(mock_request.call_count, 1) self.check_stats(counter=[ ('locate.fallback.cache', ['status:hit']), ('locate.fallback.lookup', ['status:200']), ], timer=[ 'locate.fallback.lookup', ])
def test_set_cache_redis_failure(self): cell = CellShardFactory.build() mock_redis_client = self._mock_redis_client() mock_redis_client.mget.return_value = [] mock_redis_client.mset.side_effect = RedisError() mock_redis_client.expire.side_effect = RedisError() mock_redis_client.execute.side_effect = RedisError() with requests_mock.Mocker() as mock_request: mock_request.register_uri( 'POST', requests_mock.ANY, json=self.fallback_result) with mock.patch.object(self.source.cache, 'redis_client', mock_redis_client): query = self.model_query(cells=[cell]) result = self.source.search(query) self.check_model_result(result, self.fallback_model) self.assertTrue(mock_redis_client.mget.called) self.assertTrue(mock_redis_client.mset.called) self.assertTrue(mock_request.called) self.check_stats(counter=[ ('locate.fallback.cache', ['status:miss']), ])
def test_success(self): cell = CellShardFactory.build() with requests_mock.Mocker() as mock_request: mock_request.register_uri( 'POST', requests_mock.ANY, json=self.fallback_result) query = self.model_query( cells=[cell], fallback={ 'lacf': True, 'ipf': False, }, ) result = self.source.search(query) self.check_model_result(result, self.fallback_model) request_json = mock_request.request_history[0].json() self.assertEqual(request_json['fallbacks'], {'lacf': True}) self.check_stats(counter=[ ('locate.fallback.lookup', ['status:200']), ], timer=[ 'locate.fallback.lookup', ])
def test_cell(self): cell = CellShardFactory.build(radio=Radio.lte) query = Query(cell=[{ 'radioType': cell.radio, 'mobileCountryCode': cell.mcc, 'mobileNetworkCode': cell.mnc, 'locationAreaCode': cell.lac, 'cellId': cell.cid, 'age': 1200, 'asu': None, 'primaryScramblingCode': 5, 'signalStrength': -70, 'timingAdvance': 15, 'unknown_field': 'foo' }]) data = OUTBOUND_SCHEMA.deserialize(query.json()) assert (data == { 'cellTowers': [{ 'radioType': cell.radio.name, 'mobileCountryCode': cell.mcc, 'mobileNetworkCode': cell.mnc, 'locationAreaCode': cell.lac, 'cellId': cell.cid, 'primaryScramblingCode': 5, 'age': 1200, 'signalStrength': -70, 'timingAdvance': 15, }], 'fallbacks': { 'lacf': True }, })
def test_cell_not_found(self, app, data_queues, stats): cell = CellShardFactory.build() query = self.model_query(cells=[cell]) res = self._call(app, body=query, status=self.not_found.code) self.check_response(data_queues, res, 'not_found') stats.check(counter=[ ('request', [ self.metric_path, 'method:post', 'status:%s' % self.not_found.code ]), (self.metric_type + '.request', [self.metric_path, 'key:test']), (self.metric_type + '.query', [ 'key:test', 'region:none', 'geoip:false', 'blue:none', 'cell:one', 'wifi:none' ]), (self.metric_type + '.result', [ 'key:test', 'region:none', 'fallback_allowed:false', 'accuracy:medium', 'status:miss' ]), (self.metric_type + '.source', [ 'key:test', 'region:none', 'source:internal', 'accuracy:medium', 'status:miss' ]), ], timer=[ ('request', [self.metric_path, 'method:post']), ])
def test_set_cache_redis_failure( self, geoip_db, http_session, raven, session, source, metricsmock ): cell = CellShardFactory.build() mock_redis_client = _mock_redis_client() mock_redis_client.mget.return_value = [] mock_redis_client.mset.side_effect = RedisError() mock_redis_client.expire.side_effect = RedisError() mock_redis_client.execute.side_effect = RedisError() with requests_mock.Mocker() as mock_request: mock_request.register_uri( "POST", requests_mock.ANY, json=self.fallback_result ) with mock.patch.object( source.caches[DEFAULT_SCHEMA], "redis_client", mock_redis_client ): query = self.model_query(geoip_db, http_session, session, cells=[cell]) results = source.search(query) self.check_model_results(results, [self.fallback_model]) assert mock_redis_client.mget.called assert mock_redis_client.mset.called assert mock_request.called raven.check([("RedisError", 1)]) assert metricsmock.has_record( "incr", "locate.fallback.cache", value=1, tags=[self.fallback_tag, "status:miss"], )
def test_cell_not_found(self, app, data_queues, metricsmock, logs): """A failed cell-based lookup emits several metrics.""" cell = CellShardFactory.build() query = self.model_query(cells=[cell]) res = self._call(app, body=query, status=404) self.check_response(data_queues, res, "not_found") metricsmock.assert_incr_once( "request", tags=[self.metric_path, "method:post", "status:404"] ) metricsmock.assert_incr_once( self.metric_type + ".request", tags=[self.metric_path, "key:test"] ) metricsmock.assert_incr_once( self.metric_type + ".query", tags=["key:test", "geoip:false", "blue:none", "cell:one", "wifi:none"], ) metricsmock.assert_incr_once( self.metric_type + ".result", tags=[ "key:test", "fallback_allowed:false", "accuracy:medium", "status:miss", ], ) metricsmock.assert_incr_once( self.metric_type + ".source", tags=["key:test", "source:internal", "accuracy:medium", "status:miss"], ) assert logs.only_entry["cell_valid"] == 1
def test_cell(self): cell = CellShardFactory.build() cell_query = self.cell_model_query([cell]) query = Query(cell=cell_query) self.assertEqual(len(query.cell), 1) self.assertEqual(query.expected_accuracy, DataAccuracy.medium) query_cell = query.cell[0] for key, value in cell_query[0].items(): query_value = getattr(query_cell, key, None) if key == 'radio': self.assertEqual(query_value, cell.radio) else: self.assertEqual(query_value, value) self.assertEqual(len(query.cell_area), 1) query_area = query.cell_area[0] for key, value in cell_query[0].items(): query_value = getattr(query_area, key, None) if key == 'radio': self.assertEqual(query_value, cell.radio) elif key in ('cid', 'psc'): pass else: self.assertEqual(query_value, value)
def test_dont_recache(self, geoip_db, http_session, session, source, stats): cell = CellShardFactory.build() mock_redis_client = _mock_redis_client() mock_redis_client.mget.return_value = [self.fallback_cached_result] with requests_mock.Mocker() as mock_request: mock_request.register_uri( 'POST', requests_mock.ANY, json=self.fallback_result) with mock.patch.object(source.caches[DEFAULT_SCHEMA], 'redis_client', mock_redis_client): query = self.model_query( geoip_db, http_session, session, stats, cells=[cell]) results = source.search(query) self.check_model_results(results, [self.fallback_model]) assert mock_redis_client.mget.called assert not mock_redis_client.mset.called stats.check(counter=[ ('locate.fallback.cache', [self.fallback_tag, 'status:hit']), ])
def test_cell(self): cell = CellShardFactory.build(radio=Radio.lte) query = Query(cell=[ {'radioType': cell.radio, 'mobileCountryCode': cell.mcc, 'mobileNetworkCode': cell.mnc, 'locationAreaCode': cell.lac, 'cellId': cell.cid, 'asu': 17, 'primaryScramblingCode': 5, 'signalStrength': -70, 'timingAdvance': 15, 'unknown_field': 'foo'}]) data = self._call(query.json()) assert (data == { 'cells': [{ 'radio': cell.radio.name, 'mcc': cell.mcc, 'mnc': cell.mnc, 'lac': cell.lac, 'cid': cell.cid, 'asu': 17, 'psc': 5, 'signal': -70, 'tA': 15, }], 'fallbacks': {'lacf': True}, 'token': None, })
def test_set_cache_redis_failure(self, geoip_db, http_session, raven, session, source, stats): cell = CellShardFactory.build() mock_redis_client = _mock_redis_client() mock_redis_client.mget.return_value = [] mock_redis_client.mset.side_effect = RedisError() mock_redis_client.expire.side_effect = RedisError() mock_redis_client.execute.side_effect = RedisError() with requests_mock.Mocker() as mock_request: mock_request.register_uri( 'POST', requests_mock.ANY, json=self.fallback_result) with mock.patch.object(source.caches[DEFAULT_SCHEMA], 'redis_client', mock_redis_client): query = self.model_query( geoip_db, http_session, session, stats, cells=[cell]) results = source.search(query) self.check_model_results(results, [self.fallback_model]) assert mock_redis_client.mget.called assert mock_redis_client.mset.called assert mock_request.called raven.check([('RedisError', 1)]) stats.check(counter=[ ('locate.fallback.cache', [self.fallback_tag, 'status:miss']), ])
def test_success(self): cell = CellShardFactory.build() with requests_mock.Mocker() as mock_request: mock_request.register_uri('POST', requests_mock.ANY, json=self.fallback_result) query = self.model_query( cells=[cell], fallback={ 'lacf': True, 'ipf': False, }, ) results = self.source.search(query) self.check_model_results(results, [self.fallback_model]) self.assertAlmostEqual(results.best().score, 5.0, 4) request_json = mock_request.request_history[0].json() self.assertEqual(request_json['fallbacks'], {'lacf': True}) self.check_stats(counter=[ ('locate.fallback.lookup', ['fallback_name:fall', 'status:200']), ], timer=[ ('locate.fallback.lookup', ['fallback_name:fall' ]), ])
def add_reports(self, num=1, blue_factor=0, cell_factor=1, wifi_factor=2, api_key='test', nickname=None, blue_key=None, cell_mcc=None, wifi_key=None, lat=None, lon=None): reports = [] for i in range(num): pos = CellShardFactory.build() report = { 'timestamp': time.time() * 1000.0, 'position': {}, 'bluetoothBeacons': [], 'cellTowers': [], 'wifiAccessPoints': [], } report['position']['latitude'] = lat or pos.lat report['position']['longitude'] = lon or pos.lon report['position']['accuracy'] = 17.0 + i blues = BlueShardFactory.build_batch(blue_factor, lat=pos.lat, lon=pos.lon) for blue in blues: blue_data = { 'macAddress': blue_key or blue.mac, 'signalStrength': -100 + i, } report['bluetoothBeacons'].append(blue_data) cells = CellShardFactory.build_batch(cell_factor, lat=pos.lat, lon=pos.lon) for cell in cells: cell_data = { 'radioType': cell.radio.name, 'mobileCountryCode': cell_mcc or cell.mcc, 'mobileNetworkCode': cell.mnc, 'locationAreaCode': cell.lac, 'cellId': cell.cid, 'primaryScramblingCode': cell.psc, 'signalStrength': -110 + i, } report['cellTowers'].append(cell_data) wifis = WifiShardFactory.build_batch(wifi_factor, lat=pos.lat, lon=pos.lon) for wifi in wifis: wifi_data = { 'macAddress': wifi_key or wifi.mac, 'signalStrength': -90 + i, 'ssid': 'my-wifi', } report['wifiAccessPoints'].append(wifi_data) reports.append(report) items = [{'api_key': api_key, 'nickname': nickname, 'report': rep} for rep in reports] self.incoming_queue.enqueue(items) update_incoming.delay().get() return reports
def test_success(self, geoip_db, http_session, session, source, metricsmock): cell = CellShardFactory.build() with requests_mock.Mocker() as mock_request: mock_request.register_uri( "POST", requests_mock.ANY, json=self.fallback_result ) query = self.model_query( geoip_db, http_session, session, cells=[cell], fallback={"lacf": True, "ipf": False}, ) results = source.search(query) self.check_model_results(results, [self.fallback_model]) assert results.best().score == 5.0 request_json = mock_request.request_history[0].json() self._check_success_fallbacks(request_json) assert metricsmock.has_record( "incr", "locate.fallback.cache", value=1, tags=[self.fallback_tag, "status:miss"], ) assert metricsmock.has_record( "timing", "locate.fallback.lookup.timing", tags=[self.fallback_tag] )
def test_dont_recache(self, geoip_db, http_session, session, source, metricsmock): cell = CellShardFactory.build() mock_redis_client = _mock_redis_client() mock_redis_client.mget.return_value = [self.fallback_cached_result] with requests_mock.Mocker() as mock_request: mock_request.register_uri( "POST", requests_mock.ANY, json=self.fallback_result ) with mock.patch.object( source.caches[DEFAULT_SCHEMA], "redis_client", mock_redis_client ): query = self.model_query(geoip_db, http_session, session, cells=[cell]) results = source.search(query) self.check_model_results(results, [self.fallback_model]) assert mock_redis_client.mget.called assert not mock_redis_client.mset.called assert metricsmock.has_record( "incr", "locate.fallback.cache", value=1, tags=[self.fallback_tag, "status:hit"], )
def test_cell(self): cell = CellShardFactory.build(radio=Radio.lte) cell_query = self.cell_model_query([cell]) query = Query(cell=cell_query) assert len(query.cell) == 1 assert query.expected_accuracy is DataAccuracy.medium assert query.geoip_only is False query_cell = query.cell[0] for key, value in cell_query[0].items(): query_value = getattr(query_cell, key, None) if key == 'radioType': assert query_value is cell.radio else: assert query_value == value assert len(query.cell_area) == 1 query_area = query.cell_area[0] for key, value in cell_query[0].items(): query_value = getattr(query_area, key, None) if key == 'radioType': assert query_value is cell.radio elif key in ('cellId', 'primaryScramblingCode'): pass else: assert query_value == value
def test_success(self, geoip_db, http_session, session, source, stats): cell = CellShardFactory.build() with requests_mock.Mocker() as mock_request: mock_request.register_uri( 'POST', requests_mock.ANY, json=self.fallback_result) query = self.model_query( geoip_db, http_session, session, stats, cells=[cell], fallback={ 'lacf': True, 'ipf': False, }, ) results = source.search(query) self.check_model_results(results, [self.fallback_model]) assert results.best().score == 5.0 request_json = mock_request.request_history[0].json() self._check_success_fallbacks(request_json) stats.check(counter=[ ('locate.fallback.lookup', [self.fallback_tag, 'status:200']), ], timer=[ ('locate.fallback.lookup', [self.fallback_tag]), ])
def test_cache_empty_result(self): cell = CellShardFactory.build() with requests_mock.Mocker() as mock_request: mock_request.register_uri( 'POST', requests_mock.ANY, json=LocationNotFound.json_body(), status_code=404 ) query = self.model_query(cells=[cell]) result = self.source.search(query) self.check_model_result(result, None) self.assertEqual(mock_request.call_count, 1) self.check_stats(counter=[ ('locate.fallback.cache', ['status:miss']), ('locate.fallback.lookup', ['status:404']), ]) query = self.model_query(cells=[cell]) result = self.source.search(query) self.check_model_result(result, None) self.assertEqual(mock_request.call_count, 1) self.check_stats(counter=[ ('locate.fallback.cache', ['status:hit']), ('locate.fallback.lookup', ['status:404']), ])
def test_set_cache_redis_failure(self, geoip_db, http_session, raven, session, source, stats): cell = CellShardFactory.build() mock_redis_client = self._mock_redis_client() mock_redis_client.mget.return_value = [] mock_redis_client.mset.side_effect = RedisError() mock_redis_client.expire.side_effect = RedisError() mock_redis_client.execute.side_effect = RedisError() with requests_mock.Mocker() as mock_request: mock_request.register_uri('POST', requests_mock.ANY, json=self.fallback_result) with mock.patch.object(source.cache, 'redis_client', mock_redis_client): query = self.model_query(geoip_db, http_session, session, stats, cells=[cell]) results = source.search(query) self.check_model_results(results, [self.fallback_model]) assert mock_redis_client.mget.called assert mock_redis_client.mset.called assert mock_request.called raven.check([('RedisError', 1)]) stats.check(counter=[ ('locate.fallback.cache', ['status:miss']), ])
def test_success(self, geoip_db, http_session, session, source, stats): cell = CellShardFactory.build() with requests_mock.Mocker() as mock_request: mock_request.register_uri('POST', requests_mock.ANY, json=self.fallback_result) query = self.model_query( geoip_db, http_session, session, stats, cells=[cell], fallback={ 'lacf': True, 'ipf': False, }, ) results = source.search(query) self.check_model_results(results, [self.fallback_model]) assert results.best().score == 5.0 request_json = mock_request.request_history[0].json() assert request_json['fallbacks'] == {'lacf': True} stats.check(counter=[ ('locate.fallback.lookup', ['fallback_name:fall', 'status:200']), ], timer=[ ('locate.fallback.lookup', ['fallback_name:fall']), ])
def test_cache_empty_result(self): cell = CellShardFactory.build() with requests_mock.Mocker() as mock_request: mock_request.register_uri('POST', requests_mock.ANY, json=LocationNotFound.json_body(), status_code=404) query = self.model_query(cells=[cell]) results = self.source.search(query) self.check_model_results(results, None) self.assertEqual(mock_request.call_count, 1) self.check_stats(counter=[ ('locate.fallback.cache', ['status:miss']), ('locate.fallback.lookup', ['fallback_name:fall', 'status:404']), ]) query = self.model_query(cells=[cell]) results = self.source.search(query) self.check_model_results(results, None) self.assertEqual(mock_request.call_count, 1) self.check_stats(counter=[ ('locate.fallback.cache', ['status:hit']), ('locate.fallback.lookup', ['fallback_name:fall', 'status:404']), ])
def test_cache_single_cell( self, geoip_db, http_session, session, source, metricsmock ): cell = CellShardFactory.build() with requests_mock.Mocker() as mock_request: mock_request.register_uri( "POST", requests_mock.ANY, json=self.fallback_result ) query = self.model_query(geoip_db, http_session, session, cells=[cell]) query.cell[0].signalStrength = -77 results = source.search(query) self.check_model_results(results, [self.fallback_model]) assert results.best().score == 5.0 assert mock_request.call_count == 1 assert metricsmock.has_record( "incr", "locate.fallback.cache", value=1, tags=["fallback_name:fall", "status:miss"], ) assert metricsmock.has_record( "incr", "locate.fallback.lookup", value=1, tags=["fallback_name:fall", "status:200"], ) assert metricsmock.has_record( "timing", "locate.fallback.lookup.timing", tags=["fallback_name:fall"] ) metricsmock.clear_records() # vary the signal strength, not part of cache key query.cell[0].signalStrength = -82 results = source.search(query) self.check_model_results(results, [self.fallback_model]) assert results.best().score == 5.0 assert mock_request.call_count == 1 assert metricsmock.has_record( "incr", "locate.fallback.cache", value=1, tags=["fallback_name:fall", "status:hit"], ) assert metricsmock.has_record( "incr", "locate.source", value=1, tags=[ "key:test", "region:none", "source:fallback", "accuracy:medium", "status:hit", ], )
def test_local_export(self, celery, session): now = util.utcnow() today = now.date() long_ago = now - timedelta(days=367) cell_fixture_fields = ("radio", "cid", "lat", "lon", "mnc", "mcc", "lac") base_cell = CellShardFactory.build(radio=Radio.wcdma) cell_key = { "radio": Radio.wcdma, "mcc": base_cell.mcc, "mnc": base_cell.mnc, "lac": base_cell.lac, } cells = set() for cid in range(190, 200): cell = dict(cid=cid, lat=base_cell.lat, lon=base_cell.lon, **cell_key) CellShardFactory(**cell) cell["lat"] = "%.7f" % cell["lat"] cell["lon"] = "%.7f" % cell["lon"] cell["radio"] = "UMTS" cell_strings = [(field, str(value)) for (field, value) in cell.items()] cell_tuple = tuple(sorted(cell_strings)) cells.add(cell_tuple) # add one incomplete / unprocessed cell CellShardFactory(cid=210, lat=None, lon=None, **cell_key) # add one really old cell CellShardFactory( cid=220, created=long_ago, modified=long_ago, last_seen=long_ago.date(), **cell_key, ) session.commit() with util.selfdestruct_tempdir() as temp_dir: path = os.path.join(temp_dir, "export.csv.gz") write_stations_to_csv(session, path, today) with util.gzip_open(path, "r") as gzip_wrapper: with gzip_wrapper as gzip_file: reader = csv.DictReader(gzip_file, CELL_FIELDS) header = next(reader) assert "area" in header.values() exported_cells = set() for exported_cell in reader: exported_cell_filtered = [ (field, value) for (field, value) in exported_cell.items() if field in cell_fixture_fields ] exported_cell = tuple(sorted(exported_cell_filtered)) exported_cells.add(exported_cell) assert cells == exported_cells
def test_cell_area_duplicated(self): cell = CellShardFactory.build() cell_query = self.cell_model_query([cell, cell, cell]) cell_query[1]['cid'] += 2 cell_query[2]['cid'] += 1 query = Query(cell=cell_query) self.assertEqual(len(query.cell), 3) self.assertEqual(len(query.cell_area), 1)
def test_check_invalid_cell(self, geoip_db, http_session, session, source): malformed_cell = CellShardFactory.build() malformed_cell.mcc = 99999 query = self.model_query( geoip_db, http_session, session, cells=[malformed_cell] ) self.check_should_search(source, query, False)
def _one_cell_query(self, radio=True): cell = CellShardFactory.build() query = {'lat': cell.lat, 'lon': cell.lon, 'cell': [{'mcc': cell.mcc, 'mnc': cell.mnc, 'lac': cell.lac, 'cid': cell.cid}]} if radio: query['cell'][0]['radio'] = cell.radio.name return (cell, query)
def test_cell_area_duplicated(self): cell = CellShardFactory.build() cell_query = self.cell_model_query([cell, cell, cell]) cell_query[1]['cellId'] += 2 cell_query[2]['cellId'] += 1 query = Query(cell=cell_query) assert len(query.cell) == 3 assert len(query.cell_area) == 1
def test_cell_duplicated(self): cell = CellShardFactory.build() cell_query = self.cell_model_query([cell, cell, cell]) cell_query[0]['signalStrength'] = -95 cell_query[1]['signalStrength'] = -90 cell_query[2]['signalStrength'] = -92 query = Query(cell=cell_query) assert len(query.cell) == 1 assert query.cell[0].signalStrength == -90
def test_no_api_key_shortname(self): api_key = ApiKeyFactory.build(shortname=None, log_locate=True) cell = CellShardFactory.build() self._make_query(api_key=api_key, cell=[cell]) self.check_stats(counter=[ ('locate.query', ['key:%s' % api_key.valid_key, 'region:none', 'geoip:false', 'cell:one', 'wifi:none']), ])
def test_cell_radio(self): cell = CellShardFactory.build() data = SUBMIT_V0_SCHEMA.deserialize({'items': [ {'lat': cell.lat, 'lon': cell.lon, 'cell': [{ 'radio': 'UMTS', 'mcc': cell.mcc, 'mnc': cell.mnc, 'lac': cell.lac, 'cid': cell.cid, }]} ]}) assert data['items'][0]['cellTowers'][0]['radioType'] == 'wcdma' cell = CellShardFactory.build() data = SUBMIT_V0_SCHEMA.deserialize({'items': [ {'lat': cell.lat, 'lon': cell.lon, 'cell': [{ 'radio': 'foo', 'mcc': cell.mcc, 'mnc': cell.mnc, 'lac': cell.lac, 'cid': cell.cid, }]} ]}) assert 'radioType' not in data['items'][0]['cellTowers'][0]
def test_cell_geoip_mismatch(self, app, session): # UK GeoIP with ambiguous US mcc uk_cell = CellShardFactory.build(mcc=234) us_cell = CellShardFactory(mcc=310) session.flush() query = self.model_query(cells=[us_cell]) res = self._call(app, body=query, ip=self.test_ip) self.check_model_response(res, uk_cell, region="GB", fallback="ipf")
def test_cell_duplicated(self): cell = CellShardFactory.build() cell_query = self.cell_model_query([cell, cell, cell]) cell_query[0]['signal'] = -95 cell_query[1]['signal'] = -90 cell_query[2]['signal'] = -92 query = Query(cell=cell_query) self.assertEqual(len(query.cell), 1) self.assertEqual(query.cell[0].signal, -90)
def test_cell_geoip_mismatch(self, app, session): # UK GeoIP with ambiguous US mcc uk_cell = CellShardFactory.build(mcc=234) us_cell = CellShardFactory(mcc=310) session.flush() query = self.model_query(cells=[us_cell]) res = self._call(app, body=query, ip=self.test_ip) self.check_model_response(res, uk_cell, region='GB', fallback='ipf')
def test_cell_invalid_lac(self, app, data_queues, logs): """A valid CID with and invalid LAC is not an error.""" cell = CellShardFactory.build(radio=Radio.wcdma, lac=0, cid=1) query = self.model_query(cells=[cell]) res = self._call(app, body=query, status=404) self.check_response(data_queues, res, "not_found") assert logs.only_entry["cell"] == 1 assert logs.only_entry["cell_valid"] == 0
def test_check_invalid_cell(self, geoip_db, http_session, session, source, stats): malformed_cell = CellShardFactory.build() malformed_cell.mcc = 99999 query = self.model_query( geoip_db, http_session, session, stats, cells=[malformed_cell]) self.check_should_search(source, query, False)
def test_invalid_json(self): cell = CellShardFactory.build() with requests_mock.Mocker() as mock_request: mock_request.register_uri( 'POST', requests_mock.ANY, json=['invalid json']) query = self.model_query(cells=[cell]) results = self.source.search(query) self.check_model_results(results, None)
def test_malformed_json(self): cell = CellShardFactory.build() with requests_mock.Mocker() as mock_request: mock_request.register_uri( 'POST', requests_mock.ANY, content=b'[invalid json') query = self.model_query(cells=[cell]) result = self.source.search(query) self.check_model_result(result, None)
def test_ambiguous_mcc(self): regions = GEOCODER.regions_for_mcc(234, metadata=True) cell = CellShardFactory.build(mcc=234) query = self.model_query(cells=[cell]) results = self.source.search(query) self.check_model_result(results, regions) self.check_stats(counter=[ (self.api_type + '.source', ['key:test', 'region:none', 'source:internal', 'accuracy:low', 'status:hit']), ])
def test_import_local_ocid(self, celery, redis, session): self.import_csv( celery, redis, session, CellShardFactory.build(radio=Radio.wcdma)) cells = session.query(CellShardOCID.shards()['wcdma']).all() assert len(cells) == 9 areaids = set([cell.areaid for cell in cells]) assert session.query(CellAreaOCID).count() == len(areaids) update_statcounter.delay().get() self.check_stat(session, StatKey.unique_cell_ocid, 9)
def test_set_cell_not_found(self): cell = CellShardFactory.build() query = Query(cell=self.cell_model_query([cell])) result = ExternalResult(None, None, None, None) self.cache.set(query, result) keys = self.redis_client.keys('cache:fallback:cell:*') self.assertEqual(len(keys), 1) self.assertEqual(self.redis_client.get(keys[0]), b'"404"') self.assertEqual(self.cache.get(query), result) self.check_stats(counter=[ ('locate.fallback.cache', 1, 1, ['status:hit']), ])
def test_cell(self, app, celery): now = util.utcnow() today = now.replace(hour=0, minute=0, second=0) cell = CellShardFactory.build(radio=Radio.umts) res = self._post(app, [{ 'lat': cell.lat, 'lon': cell.lon, 'time': today.strftime('%Y-%m-%d'), 'accuracy': 10.6, 'altitude': 123.1, 'altitude_accuracy': 7.0, 'heading': 45.2, 'pressure': 1020.23, 'speed': 3.6, 'source': 'gnss', 'radio': cell.radio.name, 'cell': [{ 'radio': 'umts', 'mcc': cell.mcc, 'mnc': cell.mnc, 'lac': cell.lac, 'cid': cell.cid, 'age': 1000, 'asu': 3, 'psc': 7, 'serving': 1, 'signal': -85, 'ta': 2}], }], api_key='test') assert res.body == b'' assert self.queue(celery).size() == 1 item = self.queue(celery).dequeue()[0] assert item['api_key'] == 'test' report = item['report'] timestamp = datetime.utcfromtimestamp(report['timestamp'] / 1000.0) timestamp = timestamp.replace(microsecond=0, tzinfo=pytz.UTC) assert timestamp == today position = report['position'] assert position['latitude'] == cell.lat assert position['longitude'] == cell.lon assert position['accuracy'] == 10.6 assert position['altitude'] == 123.1 assert position['altitudeAccuracy'] == 7.0 assert position['heading'] == 45.2 assert position['pressure'] == 1020.23 assert position['speed'] == 3.6 assert position['source'] == 'gnss' cells = report['cellTowers'] assert cells[0]['radioType'] == 'wcdma' assert cells[0]['mobileCountryCode'] == cell.mcc assert cells[0]['mobileNetworkCode'] == cell.mnc assert cells[0]['locationAreaCode'] == cell.lac assert cells[0]['cellId'] == cell.cid assert cells[0]['age'] == 1000 assert cells[0]['asu'] == 3 assert cells[0]['primaryScramblingCode'] == 7 assert cells[0]['serving'] == 1 assert cells[0]['signalStrength'] == -85 assert cells[0]['timingAdvance'] == 2
def test_set_cell(self): cell = CellShardFactory.build() query = Query(cell=self.cell_model_query([cell])) result = ExternalResult(cell.lat, cell.lon, cell.radius, None) self.cache.set(query, result) keys = self.redis_client.keys('cache:fallback:cell:*') self.assertEqual(len(keys), 1) self.assertTrue(500 < self.redis_client.ttl(keys[0]) <= 600) self.assertEqual(self.cache.get(query), result) self.check_stats(counter=[ ('locate.fallback.cache', 1, 1, ['status:hit']), ])