def test_no_log(self, geoip_db, metricsmock):
api_key = KeyFactory(valid_key=None)
self._make_query(geoip_db,
self._make_result(),
api_key=api_key,
api_type="locate")
assert len(metricsmock.get_records()) == 0
def test_store_sample(self):
key = KeyFactory(store_sample_locate=None, store_sample_submit=None)
assert key.store_sample('locate') is False
assert key.store_sample('submit') is False
assert key.store_sample('region') is False
key = KeyFactory(store_sample_locate=0, store_sample_submit=100)
assert key.store_sample('locate') is False
assert key.store_sample('submit') is True
key = KeyFactory(store_sample_locate=50)
results = []
for i in range(20):
results.append(key.store_sample('locate'))
assert True in results
assert False in results
class TestGoogleMapsV1Fallback(BaseFallbackTest, BaseSourceTest):
api_key = KeyFactory(fallback_name="googlemaps",
fallback_schema=GOOGLEMAPS_V1_SCHEMA)
def _check_success_fallbacks(self, request_json):
assert "fallbacks" not in request_json
assert request_json["considerIp"] is False
def test_api_key_disallows(self, geoip_db, http_session, session, source):
api_key = KeyFactory(allow_fallback=False)
cells = CellShardFactory.build_batch(2)
wifis = WifiShardFactory.build_batch(2)
query = self.model_query(
geoip_db, http_session, session, cells=cells, wifis=wifis, api_key=api_key
)
self.check_should_search(source, query, False)
def _make_query(self, geoip_db, stats,
source, results, api_key=None, api_type='locate',
blue=(), cell=(), wifi=(), **kw):
query = Query(
api_key=api_key or KeyFactory(valid_key='test'),
api_type=api_type,
blue=self.blue_model_query(blue),
cell=self.cell_model_query(cell),
wifi=self.wifi_model_query(wifi),
geoip_db=geoip_db,
stats_client=stats,
**kw)
query.emit_source_stats(source, results)
return query
def _make_query(
self, geoip_db, api_key=None, api_type="locate", blue=(), cell=(), wifi=(), **kw
):
query = Query(
api_key=api_key or KeyFactory(valid_key="test"),
api_type=api_type,
blue=self.blue_model_query(blue),
cell=self.cell_model_query(cell),
wifi=self.wifi_model_query(wifi),
geoip_db=geoip_db,
**kw,
)
query.emit_query_stats()
return query
def _search(self, data_queues, geoip_db, raven, redis, session, klass, **kw):
query = Query(
api_key=KeyFactory(valid_key="test"),
api_type="locate",
session=session,
**kw,
)
searcher = klass(
geoip_db=geoip_db,
raven_client=raven,
redis_client=redis,
data_queues=data_queues,
)
return searcher.search(query)
def test_store_sample_mock_random(self, mock_random):
key = KeyFactory(store_sample_locate=50)
mock_random.return_value = 0.1
assert key.store_sample("locate") is True
mock_random.return_value = 0.5
assert key.store_sample("locate") is True
mock_random.return_value = 0.51
assert key.store_sample("locate") is False
mock_random.return_value = 0.9
assert key.store_sample("locate") is False
def test_store_sample(self):
key = KeyFactory(store_sample_locate=None, store_sample_submit=None)
assert key.store_sample('locate') is False
assert key.store_sample('submit') is False
assert key.store_sample('region') is False
key = KeyFactory(store_sample_locate=0, store_sample_submit=100)
assert key.store_sample('locate') is False
assert key.store_sample('submit') is True
key = KeyFactory(store_sample_locate=50)
results = []
for i in range(20):
results.append(key.store_sample('locate'))
assert True in results
assert False in results
def test_store_sample_mock_random_with_global_rate(
self, mock_random, global_rate, q1, q2, q3, q4
):
assert 0.0 < (q3 - q2) < 0.1
key = KeyFactory(store_sample_locate=50)
mock_random.return_value = q1
assert key.store_sample("locate", global_rate) is True
mock_random.return_value = q2
assert key.store_sample("locate", global_rate) is True
mock_random.return_value = q3
assert key.store_sample("locate", global_rate) is False
mock_random.return_value = q4
assert key.store_sample("locate", global_rate) is False
def _make_query(self, geoip_db, stats, **kw):
return Query(api_key=KeyFactory(valid_key='test'),
geoip_db=geoip_db,
stats_client=stats,
**kw)
def one(**kw):
return KeyFactory(**kw)
def _make_query(self, geoip_db, **kw):
return Query(api_key=KeyFactory(valid_key="test"),
geoip_db=geoip_db,
**kw)
class BaseFallbackTest(object):
api_key = KeyFactory(valid_key='test', allow_fallback=True)
fallback_model = DummyModel(lat=51.5366, lon=0.03989, radius=1500.0)
fallback_not_found_status = 404
Source = FallbackPositionSource
@property
def fallback_tag(self):
return 'fallback_name:%s' % self.api_key.fallback_name
@property
def fallback_result(self):
return {
'location': {
'lat': self.fallback_model.lat,
'lng': self.fallback_model.lon,
},
'accuracy': float(self.fallback_model.radius),
'fallback': 'lacf',
}
@property
def fallback_not_found(self):
return LocationNotFound.json_body()
def _check_success_fallbacks(self, request_json):
assert 'considerIp' not in request_json
assert request_json['fallbacks'] == {'lacf': True, 'ipf': False}
def test_success(self, geoip_db, http_session, session, source, stats):
cell = CellShardFactory.build()
with requests_mock.Mocker() as mock_request:
mock_request.register_uri('POST',
requests_mock.ANY,
json=self.fallback_result)
query = self.model_query(
geoip_db,
http_session,
session,
stats,
cells=[cell],
fallback={
'lacf': True,
'ipf': False,
},
)
results = source.search(query)
self.check_model_results(results, [self.fallback_model])
assert results.best().score == 5.0
request_json = mock_request.request_history[0].json()
self._check_success_fallbacks(request_json)
stats.check(counter=[
('locate.fallback.lookup', [self.fallback_tag, 'status:200']),
],
timer=[
('locate.fallback.lookup', [self.fallback_tag]),
])
def test_cache_empty_result(self, geoip_db, http_session, session, source,
stats):
cell = CellShardFactory.build()
with requests_mock.Mocker() as mock_request:
mock_request.register_uri(
'POST',
requests_mock.ANY,
json=self.fallback_not_found,
status_code=self.fallback_not_found_status)
query = self.model_query(geoip_db,
http_session,
session,
stats,
cells=[cell])
results = source.search(query)
self.check_model_results(results, None)
assert mock_request.call_count == 1
stats.check(counter=[
('locate.fallback.cache', [self.fallback_tag, 'status:miss']),
('locate.fallback.lookup', [
self.fallback_tag,
'status:%s' % self.fallback_not_found_status
]),
])
query = self.model_query(geoip_db,
http_session,
session,
stats,
cells=[cell])
results = source.search(query)
self.check_model_results(results, None)
assert mock_request.call_count == 1
stats.check(counter=[
('locate.fallback.cache', [self.fallback_tag, 'status:hit']),
('locate.fallback.lookup', [
self.fallback_tag,
'status:%s' % self.fallback_not_found_status
]),
])
def test_no_log(self, geoip_db, stats):
api_key = KeyFactory(valid_key=None)
self._make_query(
geoip_db, stats,
self._make_result(), api_key=api_key, api_type='locate')
stats.check(total=0)
def test_api_key(self):
api_key = KeyFactory()
query = Query(api_key=api_key)
assert query.api_key.valid_key == api_key.valid_key
assert query.api_key == api_key
class BaseSourceTest(object):
api_key = KeyFactory(valid_key="test", allow_fallback=True)
api_type = "locate"
Source = None
def make_query(self, geoip_db, http_session, session, **kw):
api_key = kw.pop("api_key", self.api_key)
return Query(
api_key=api_key,
api_type=self.api_type,
session=session,
http_session=http_session,
geoip_db=geoip_db,
**kw,
)
def model_query(
self, geoip_db, http_session, session, blues=(), cells=(), wifis=(), **kw
):
query_blue = []
if blues:
for blue in blues:
query_blue.append({"macAddress": blue.mac})
query_cell = []
if cells:
for cell in cells:
cell_query = {
"radioType": cell.radio,
"mobileCountryCode": cell.mcc,
"mobileNetworkCode": cell.mnc,
"locationAreaCode": cell.lac,
}
if getattr(cell, "cid", None) is not None:
cell_query["cellId"] = cell.cid
query_cell.append(cell_query)
query_wifi = []
if wifis:
for wifi in wifis:
query_wifi.append({"macAddress": wifi.mac})
return self.make_query(
geoip_db,
http_session,
session,
blue=query_blue,
cell=query_cell,
wifi=query_wifi,
**kw,
)
def check_should_search(self, source, query, should, results=None):
if results is None:
results = source.result_list()
assert source.should_search(query, results) is should
def check_model_results(self, results, models, **kw):
type_ = self.Source.result_type
if not models:
assert len(results) == 0
return
expected = []
if type_ is Position:
for model in models:
expected.append(
{
"lat": kw.get("lat", model.lat),
"lon": kw.get("lon", model.lon),
"accuracy": bound_model_accuracy(
model, kw.get("accuracy", model.radius)
),
}
)
# don't test ordering of results
expected = sorted(expected, key=operator.itemgetter("lat", "lon"))
results = sorted(results, key=operator.attrgetter("lat", "lon"))
elif type_ is Region:
for model in models:
expected.append({"region_code": model.code, "region_name": model.name})
# don't test ordering of results
expected = sorted(expected, key=operator.itemgetter("region_code"))
results = sorted(results, key=operator.attrgetter("region_code"))
for expect, result in zip(expected, results):
assert type(result) is type_
for key, value in expect.items():
assert getattr(result, key) == value
from ichnaea.api.locate.query import Query
from ichnaea.api.locate.result import Position, PositionResultList
from ichnaea.api.locate.tests.base import BaseSourceTest, DummyModel
from ichnaea.api.locate.tests.test_query import QueryTest
from ichnaea.models import Radio
from ichnaea.tests.factories import (
BlueShardFactory,
CellShardFactory,
KeyFactory,
WifiShardFactory,
)
UNWIREDLABS_KEY = KeyFactory(
fallback_name="labs",
fallback_schema=UNWIREDLABS_V1_SCHEMA,
fallback_url="http://127.0.0.1:9/process.php#my_secret_token",
)
def _mock_redis_client():
client = mock.Mock()
client.pipeline.return_value = client
client.__enter__ = mock.Mock(return_value=client)
client.__exit__ = mock.Mock(return_value=None)
client.expire.return_value = mock.Mock()
client.get.return_value = mock.Mock()
client.mget.return_value = mock.Mock()
client.set.return_value = mock.Mock()
client.mset.return_value = mock.Mock()
return client
class BaseSourceTest(object):
api_key = KeyFactory(valid_key='test', allow_fallback=True)
api_type = 'locate'
Source = None
def make_query(self, geoip_db, http_session, session, stats, **kw):
api_key = kw.pop('api_key', self.api_key)
return Query(
api_key=api_key,
api_type=self.api_type,
session=session,
http_session=http_session,
geoip_db=geoip_db,
stats_client=stats,
**kw)
def model_query(self, geoip_db, http_session, session, stats,
blues=(), cells=(), wifis=(), **kw):
query_blue = []
if blues:
for blue in blues:
query_blue.append({'macAddress': blue.mac})
query_cell = []
if cells:
for cell in cells:
cell_query = {
'radioType': cell.radio,
'mobileCountryCode': cell.mcc,
'mobileNetworkCode': cell.mnc,
'locationAreaCode': cell.lac,
}
if getattr(cell, 'cid', None) is not None:
cell_query['cellId'] = cell.cid
query_cell.append(cell_query)
query_wifi = []
if wifis:
for wifi in wifis:
query_wifi.append({'macAddress': wifi.mac})
return self.make_query(
geoip_db, http_session, session, stats,
blue=query_blue,
cell=query_cell,
wifi=query_wifi,
**kw)
def check_should_search(self, source, query, should, results=None):
if results is None:
results = source.result_list()
assert source.should_search(query, results) is should
def check_model_results(self, results, models, **kw):
type_ = self.Source.result_type
if not models:
assert len(results) == 0
return
expected = []
if type_ is Position:
for model in models:
expected.append({
'lat': kw.get('lat', model.lat),
'lon': kw.get('lon', model.lon),
'accuracy': bound_model_accuracy(
model, kw.get('accuracy', model.radius)),
})
# don't test ordering of results
expected = sorted(expected, key=operator.itemgetter('lat', 'lon'))
results = sorted(results, key=operator.attrgetter('lat', 'lon'))
elif type_ is Region:
for model in models:
expected.append({
'region_code': model.code,
'region_name': model.name,
})
# don't test ordering of results
expected = sorted(expected, key=operator.itemgetter('region_code'))
results = sorted(results, key=operator.attrgetter('region_code'))
for expect, result in zip(expected, results):
assert type(result) is type_
for key, value in expect.items():
assert getattr(result, key) == value
class TestCombainV1Fallback(BaseFallbackTest, BaseSourceTest):
api_key = KeyFactory(fallback_name="combain", fallback_schema=COMBAIN_V1_SCHEMA)
def test_store_sample(self):
key = KeyFactory(store_sample_locate=None, store_sample_submit=None)
assert key.store_sample("locate") is False
assert key.store_sample("submit") is False
assert key.store_sample("region") is False
key = KeyFactory(store_sample_locate=0, store_sample_submit=100)
assert key.store_sample("locate") is False
assert key.store_sample("submit") is True
# A global_locate_sample_rate can turn off samples
assert key.store_sample("locate", global_locate_sample_rate=0.0) is False
# And can raise a sample rate
key = KeyFactory(store_sample_locate=50, store_sample_submit=None)
assert key.store_sample("locate", global_locate_sample_rate=200.0) is True
class BaseFallbackTest(object):
api_key = KeyFactory(valid_key="test", allow_fallback=True)
fallback_model = DummyModel(lat=51.5366, lon=0.03989, radius=1500.0)
fallback_not_found_status = 404
Source = FallbackPositionSource
@property
def fallback_tag(self):
return "fallback_name:%s" % self.api_key.fallback_name
@property
def fallback_result(self):
return {
"location": {
"lat": self.fallback_model.lat,
"lng": self.fallback_model.lon,
},
"accuracy": float(self.fallback_model.radius),
"fallback": "lacf",
}
@property
def fallback_not_found(self):
return LocationNotFound().json_body()
def _check_success_fallbacks(self, request_json):
assert "considerIp" not in request_json
assert request_json["fallbacks"] == {"lacf": True, "ipf": False}
def test_success(self, geoip_db, http_session, session, source, metricsmock):
cell = CellShardFactory.build()
with requests_mock.Mocker() as mock_request:
mock_request.register_uri(
"POST", requests_mock.ANY, json=self.fallback_result
)
query = self.model_query(
geoip_db,
http_session,
session,
cells=[cell],
fallback={"lacf": True, "ipf": False},
)
results = source.search(query)
self.check_model_results(results, [self.fallback_model])
assert results.best().score == 5.0
request_json = mock_request.request_history[0].json()
self._check_success_fallbacks(request_json)
assert metricsmock.has_record(
"incr",
"locate.fallback.cache",
value=1,
tags=[self.fallback_tag, "status:miss"],
)
assert metricsmock.has_record(
"timing", "locate.fallback.lookup.timing", tags=[self.fallback_tag]
)
def test_cache_empty_result(
self, geoip_db, http_session, session, source, metricsmock
):
cell = CellShardFactory.build()
with requests_mock.Mocker() as mock_request:
mock_request.register_uri(
"POST",
requests_mock.ANY,
json=self.fallback_not_found,
status_code=self.fallback_not_found_status,
)
query = self.model_query(geoip_db, http_session, session, cells=[cell])
results = source.search(query)
self.check_model_results(results, None)
assert mock_request.call_count == 1
assert metricsmock.has_record(
"incr",
"locate.fallback.cache",
value=1,
tags=[self.fallback_tag, "status:miss"],
)
assert metricsmock.has_record(
"incr",
"locate.fallback.lookup",
value=1,
tags=[self.fallback_tag, "status:%s" % self.fallback_not_found_status],
)
metricsmock.clear_records()
query = self.model_query(geoip_db, http_session, session, cells=[cell])
results = source.search(query)
self.check_model_results(results, None)
assert mock_request.call_count == 1
assert metricsmock.has_record(
"incr",
"locate.fallback.cache",
value=1,
tags=[self.fallback_tag, "status:hit"],
)
def _query(self, **kwargs):
if "api_key" not in kwargs:
kwargs["api_key"] = KeyFactory(fallback_cache_expire=60)
return Query(**kwargs)
