def process_single(request):
stats_client = get_stats_client()
locate_data, locate_errors = preprocess_request(
request,
schema=GeoLocateSchema(),
extra_checks=(geolocate_validator, ),
response=JSONParseError,
accept_empty=True,
)
data, errors = preprocess_request(
request,
schema=GeoSubmitSchema(),
extra_checks=(geosubmit_validator, ),
response=None,
)
data = {'items': [data]}
nickname = request.headers.get('X-Nickname', u'')
email = request.headers.get('X-Email', u'')
upload_items = flatten_items(data)
errors = process_upload(nickname, email, upload_items)
if errors is not SENTINEL and errors: # pragma: no cover
stats_client.incr('geosubmit.upload.errors', len(errors))
first_item = data['items'][0]
if first_item['latitude'] == -255 or first_item['longitude'] == -255:
data = map_data(data['items'][0])
session = request.db_slave_session
result = search_all_sources(
session,
'geosubmit',
data,
client_addr=request.client_addr,
geoip_db=request.registry.geoip_db,
api_key_log=getattr(request, 'api_key_log', False),
api_key_name=getattr(request, 'api_key_name', None))
else:
result = {
'lat': first_item['latitude'],
'lon': first_item['longitude'],
'accuracy': first_item['accuracy']
}
if result is None:
stats_client.incr('geosubmit.miss')
result = HTTPNotFound()
result.content_type = 'application/json'
result.body = NOT_FOUND
return result
return {
"location": {
"lat": result['lat'],
"lng": result['lon'],
},
"accuracy": float(result['accuracy']),
}
def test_cell_miss_lac_hit(self):
session = self.db_slave_session
lat = PARIS_LAT
lon = PARIS_LON
key = dict(mcc=FRANCE_MCC, mnc=2, lac=3)
umts = RADIO_TYPE['umts']
data = [
Cell(lat=lat, lon=lon, radio=umts, cid=4, **key),
Cell(lat=lat + 0.002, lon=lon + 0.004, radio=umts, cid=5, **key),
Cell(lat=lat + 0.006, lon=lon + 0.006, radio=umts, cid=6, **key),
Cell(lat=lat + 0.0026666, lon=lon + 0.0033333,
radio=umts, cid=CELLID_LAC,
range=500000, **key),
]
session.add_all(data)
session.flush()
result = locate.search_all_sources(
session, 'm',
{"cell": [dict(radio="umts", cid=7, **key)]},
api_key_log=True, api_key_name='test')
self.assertEqual(result,
{'lat': PARIS_LAT + 0.0026666,
'lon': PARIS_LON + 0.0033333,
'accuracy': 500000})
self.check_stats(
counter=[
'm.no_cell_found',
'm.cell_lac_found',
'm.cell_lac_hit',
'm.api_log.test.cell_lac_hit',
],
)
def geolocate_view(request):
data, errors = preprocess_request(
request,
schema=GeoLocateSchema(),
response=JSONParseError,
accept_empty=True,
)
data = map_data(data)
session = request.db_slave_session
result = search_all_sources(session,
'geolocate',
data,
client_addr=request.client_addr,
geoip_db=request.registry.geoip_db,
api_key_log=getattr(request, 'api_key_log',
False),
api_key_name=getattr(request, 'api_key_name',
None))
if not result:
result = HTTPNotFound()
result.content_type = 'application/json'
result.body = NOT_FOUND
return result
return {
"location": {
"lat": result['lat'],
"lng": result['lon'],
},
"accuracy": float(result['accuracy']),
}
def test_cell_ignore_invalid_lac_cid(self):
session = self.db_slave_session
lat = PARIS_LAT
lon = PARIS_LON
gsm = RADIO_TYPE['gsm']
lte = RADIO_TYPE['lte']
key = dict(mcc=FRANCE_MCC, mnc=2, lac=3)
ignored_key = dict(mcc=FRANCE_MCC, mnc=2, lac=-1, cid=-1)
data = [
Cell(lat=lat, lon=lon, radio=gsm, cid=4, **key),
Cell(lat=lat + 0.002, lon=lon + 0.004, radio=gsm, cid=5, **key),
Cell(lat=lat, lon=lon, radio=gsm, **ignored_key),
Cell(lat=lat + 0.002, lon=lon + 0.004, radio=lte, **ignored_key),
]
session.add_all(data)
session.flush()
result = locate.search_all_sources(
session, 'm',
{"cell": [
dict(radio="gsm", cid=4, **key),
dict(radio="gsm", cid=5, **key),
dict(radio="gsm", cid=5, mcc=FRANCE_MCC, mnc=2, lac=-1),
dict(radio="gsm", cid=-1, mcc=FRANCE_MCC, mnc=2, lac=3),
]})
self.assertEqual(result,
{'lat': PARIS_LAT + 0.001,
'lon': PARIS_LON + 0.002,
'accuracy': CELL_MIN_ACCURACY})
def search_view(request):
data, errors = preprocess_request(
request,
schema=SearchSchema(),
extra_checks=(check_cell_or_wifi, ),
accept_empty=True,
)
session = request.db_slave_session
result = search_all_sources(
session, 'search', data,
client_addr=request.client_addr,
geoip_db=request.registry.geoip_db,
api_key_log=getattr(request, 'api_key_log', False),
api_key_name=getattr(request, 'api_key_name', None))
if not result:
return {'status': 'not_found'}
return {
'status': 'ok',
'lat': result['lat'],
'lon': result['lon'],
'accuracy': result['accuracy'],
}
def test_cell(self):
session = self.db_slave_session
cell_key = {
'radio': RADIO_TYPE['gsm'], 'mcc': GB_MCC, 'mnc': 1, 'lac': 1,
}
session.add(Cell(
lat=GB_LAT, lon=GB_LON, range=6000, cid=1, **cell_key))
session.add(Cell(
lat=GB_LAT, lon=GB_LON, range=9000, cid=CELLID_LAC, **cell_key))
session.flush()
result = locate.search_all_sources(
session, 'm',
{'cell': [dict(cid=1, **cell_key)]},
client_addr=GB_IP, geoip_db=self.geoip_db,
api_key_log=True, api_key_name='test')
self.assertEqual(result,
{'lat': GB_LAT,
'lon': GB_LON,
'accuracy': 6000})
self.check_stats(
counter=[
'm.cell_found',
'm.cell_hit',
'm.cell_lac_found',
'm.api_log.test.cell_hit',
],
)
def test_lac_disagrees_with_country(self):
# This test checks that when a LAC is at a lat/lon that
# is not in the country determined by mcc (say) we reject
# the query. Really we should start filtering these out
# on ingress as well, but this is a double-check.
session = self.db_slave_session
key = dict(mcc=BRAZIL_MCC, mnc=VIVO_MNC, lac=12345)
data = [
Cell(lat=PARIS_LAT,
lon=PARIS_LON,
radio=RADIO_TYPE['gsm'], cid=CELLID_LAC, **key),
]
session.add_all(data)
session.flush()
result = locate.search_all_sources(
session, 'm',
{"cell": [dict(radio="gsm", cid=6789, **key)]})
self.assertTrue(result is None)
self.check_stats(
counter=[
('m.anomaly.cell_lac_country_mismatch', 1),
('m.country_from_mcc', 1),
('m.no_cell_found', 1),
('m.cell_lac_found', 1),
]
)
def test_cell_multiple_lac_lower_range_wins(self):
session = self.db_slave_session
lat = PARIS_LAT
lon = PARIS_LON
gsm = RADIO_TYPE['gsm']
key = dict(mcc=FRANCE_MCC, mnc=2, lac=3)
key2 = dict(mcc=FRANCE_MCC, mnc=2, lac=4)
data = [
Cell(lat=lat + 0.2, lon=lon + 0.2, radio=gsm,
cid=CELLID_LAC, range=10000, **key),
Cell(lat=lat + 0.2, lon=lon + 0.4, radio=gsm,
cid=4, range=4000, **key),
Cell(lat=lat, lon=lon, radio=gsm,
cid=CELLID_LAC, range=20000, **key2),
Cell(lat=lat + 0.02, lon=lon + 0.02, radio=gsm,
cid=4, range=2000, **key2),
]
session.add_all(data)
session.flush()
# We have two lacs with each one known cell.
# The lac with the smallest cell wins.
result = locate.search_all_sources(
session, 'm',
{"cell": [
dict(radio="gsm", cid=4, **key),
dict(radio="gsm", cid=4, **key2),
]})
self.assertEqual(result,
{'lat': PARIS_LAT + 0.02,
'lon': PARIS_LON + 0.02,
'accuracy': CELL_MIN_ACCURACY})
def test_wifi_only_use_top_five_signals_in_noisy_cluster(self):
session = self.db_slave_session
# all these should wind up in the same cluster since
# clustering threshold is 500m and the 10 wifis are
# spaced in increments of (+1m, +1.2m)
wifis = [Wifi(key=("0%X".lower() % i) * 6,
lat=1 + i * 0.000010,
lon=1 + i * 0.000012)
for i in range(1, 11)]
session.add_all(wifis)
session.commit()
measures = [dict(key=("0%X" % i) * 6,
signal=-80)
for i in range(6, 11)]
measures += [
dict(key="010101010101", signal=-75),
dict(key="020202020202", signal=-74),
dict(key="030303030303", signal=-73),
dict(key="040404040404", signal=-72),
dict(key="050505050505", signal=-71),
]
random.shuffle(measures)
result = locate.search_all_sources(
session, 'm',
{'wifi': measures})
self.assertEqual(result,
{'lat': 1.00003,
'lon': 1.000036,
'accuracy': WIFI_MIN_ACCURACY})
def test_wifi(self):
session = self.db_slave_session
wifis = [{'key': '001122334455'}, {'key': '112233445566'}]
session.add(Wifi(
key=wifis[0]['key'], lat=GB_LAT, lon=GB_LON, range=200))
session.add(Wifi(
key=wifis[1]['key'], lat=GB_LAT, lon=GB_LON + 0.00001, range=300))
session.flush()
result = locate.search_all_sources(
session, 'm',
{'wifi': wifis},
client_addr=GB_IP, geoip_db=self.geoip_db,
api_key_log=True, api_key_name='test')
self.assertEqual(result,
{'lat': GB_LAT,
'lon': GB_LON + 0.000005,
'accuracy': WIFI_MIN_ACCURACY})
self.check_stats(
counter=[
'm.wifi_found',
'm.wifi_hit',
'm.api_log.test.wifi_hit',
],
)
def test_wifi_prefer_cluster_with_better_signals(self):
session = self.db_slave_session
wifis = [
Wifi(key="a1" * 6, lat=1.0, lon=1.0),
Wifi(key="b2" * 6, lat=1.001, lon=1.002),
Wifi(key="c3" * 6, lat=1.002, lon=1.004),
Wifi(key="d4" * 6, lat=2.0, lon=2.0),
Wifi(key="e5" * 6, lat=2.001, lon=2.002),
Wifi(key="f6" * 6, lat=2.002, lon=2.004),
]
session.add_all(wifis)
session.flush()
result = locate.search_all_sources(
session, 'm',
{'wifi': [
{"key": "A1" * 6, "signal": -100},
{"key": "D4" * 6, "signal": -80},
{"key": "B2" * 6, "signal": -100},
{"key": "E5" * 6, "signal": -90},
{"key": "C3" * 6, "signal": -100},
{"key": "F6" * 6, "signal": -54},
]})
self.assertEqual(result,
{'lat': 2.001,
'lon': 2.002,
'accuracy': 248.51819})
def test_wifi_prefer_larger_cluster_over_high_signal(self):
session = self.db_slave_session
wifis = [Wifi(key=("0%X" % i).lower() * 6,
lat=1 + i * 0.000010,
lon=1 + i * 0.000012)
for i in range(1, 6)]
wifis += [
Wifi(key="d4" * 6, lat=2.0, lon=2.0),
Wifi(key="e5" * 6, lat=2.001, lon=2.002),
Wifi(key="f6" * 6, lat=2.002, lon=2.004),
]
session.add_all(wifis)
session.flush()
measures = [dict(key=("0%X" % i) * 6,
signal=-80)
for i in range(1, 6)]
measures += [
dict(key="D4" * 6, signal=-75),
dict(key="E5" * 6, signal=-74),
dict(key="F6" * 6, signal=-73)
]
random.shuffle(measures)
result = locate.search_all_sources(
session, 'm',
{'wifi': measures})
self.assertEqual(result,
{'lat': 1.00003,
'lon': 1.000036,
'accuracy': WIFI_MIN_ACCURACY})
def geolocate_view(request):
data, errors = preprocess_request(
request,
schema=GeoLocateSchema(),
extra_checks=(geolocate_validator, ),
response=JSONParseError,
accept_empty=True,
)
data = map_data(data)
session = request.db_slave_session
result = search_all_sources(
session, 'geolocate', data,
client_addr=request.client_addr,
geoip_db=request.registry.geoip_db,
api_key_log=getattr(request, 'api_key_log', False),
api_key_name=getattr(request, 'api_key_name', None))
if not result:
result = HTTPNotFound()
result.content_type = 'application/json'
result.body = NOT_FOUND
return result
return {
"location": {
"lat": result['lat'],
"lng": result['lon'],
},
"accuracy": float(result['accuracy']),
}
def test_cell_multiple_radio_mixed_cell_lac_hit(self):
session = self.db_slave_session
lat = PARIS_LAT
lon = PARIS_LON
gsm = RADIO_TYPE['gsm']
lte = RADIO_TYPE['lte']
key = dict(mcc=FRANCE_MCC, mnc=3, lac=4)
key2 = dict(mcc=FRANCE_MCC, mnc=2, lac=3)
data = [
Cell(lat=lat + 0.2, lon=lon + 0.2, radio=gsm,
cid=CELLID_LAC, range=3000, **key),
Cell(lat=lat + 0.2, lon=lon + 0.4, radio=gsm,
cid=5, range=500, **key),
Cell(lat=lat, lon=lon, radio=lte,
cid=CELLID_LAC, range=10000, **key2),
Cell(lat=lat + 0.01, lon=lon + 0.02, radio=lte,
cid=4, range=2000, **key2),
]
session.add_all(data)
session.flush()
# GSM lac-only hit (cid 9 instead of 5) and a LTE cell hit
result = locate.search_all_sources(
session, 'm',
{"cell": [
dict(radio="gsm", cid=9, **key),
dict(radio="lte", cid=4, **key2),
]})
self.assertEqual(result,
{'lat': PARIS_LAT + 0.01,
'lon': PARIS_LON + 0.02,
'accuracy': CELL_MIN_ACCURACY})
def test_wifi_not_found_cell_fallback(self):
session = self.db_slave_session
lat = PARIS_LAT
lon = PARIS_LON
umts = RADIO_TYPE['umts']
key = dict(mcc=FRANCE_MCC, mnc=2, lac=3)
data = [
Wifi(key="a0a0a0a0a0a0", lat=3, lon=3),
Cell(lat=lat, lon=lon, radio=umts, cid=4, **key),
Cell(lat=lat + 0.002, lon=lon + 0.004, radio=umts, cid=5, **key),
]
session.add_all(data)
session.flush()
result = locate.search_all_sources(
session, 'm',
{"cell": [
dict(radio="umts", cid=4, **key),
dict(radio="umts", cid=5, **key),
], "wifi": [{"key": "101010101010"},
{"key": "202020202020"}]})
self.assertEqual(result,
{'lat': PARIS_LAT + 0.001,
'lon': PARIS_LON + 0.002,
'accuracy': CELL_MIN_ACCURACY})
def test_cell_multiple_country_codes_from_mcc(self):
session = self.db_slave_session
cell_key = {
'radio': RADIO_TYPE['gsm'], 'mcc': GB_MCC, 'mnc': 1, 'lac': 1,
}
session.add(Cell(
lat=GB_LAT, lon=GB_LON, range=6000, cid=1, **cell_key))
session.add(Cell(
lat=GB_LAT, lon=GB_LON, range=9000, cid=CELLID_LAC, **cell_key))
session.flush()
result = locate.search_all_sources(
session, 'm',
{'cell': [dict(cid=1, **cell_key)]},
client_addr=None, geoip_db=self.geoip_db)
# Without a GeoIP, the mcc results in 4 different equally common
# mcc values, GB not being the first one. We need to make sure
# that we accept any of the country codes as a possible match
# and don't discard otherwise good cell data based on this.
self.assertEqual(result,
{'lat': GB_LAT,
'lon': GB_LON,
'accuracy': 6000})
self.check_stats(
counter=[
'm.cell_found',
'm.cell_hit',
'm.cell_lac_found',
],
)
def test_geoip_mcc_multiple_unknown_mismatching_cell(self):
session = self.db_slave_session
gsm = RADIO_TYPE['gsm']
cell_key = {'radio': gsm, 'mnc': 1, 'lac': 1, 'cid': 1}
cells = [
dict(mcc=GB_MCC, **cell_key),
dict(mcc=USA_MCC, **cell_key),
]
# Only add the matching cell to the DB
session.add(Cell(**cells[0]))
session.flush()
result = locate.search_all_sources(
session, 'm',
{'cell': cells},
client_addr=GB_IP, geoip_db=self.geoip_db)
self.assertEqual(result,
{'lat': GB_LAT,
'lon': GB_LON,
'accuracy': GB_RADIUS})
self.check_stats(
counter=[
'm.anomaly.multiple_mccs',
'm.country_from_geoip',
],
)
def process_single(request):
stats_client = get_stats_client()
locate_data, locate_errors = preprocess_request(
request,
schema=GeoLocateSchema(),
extra_checks=(geolocate_validator, ),
response=JSONParseError,
accept_empty=True,
)
data, errors = preprocess_request(
request,
schema=GeoSubmitSchema(),
extra_checks=(geosubmit_validator,),
response=None,
)
data = {'items': [data]}
nickname = request.headers.get('X-Nickname', u'')
email = request.headers.get('X-Email', u'')
upload_items = flatten_items(data)
errors = process_upload(nickname, email, upload_items)
if errors is not SENTINEL and errors: # pragma: no cover
stats_client.incr('geosubmit.upload.errors', len(errors))
first_item = data['items'][0]
if first_item['latitude'] == -255 or first_item['longitude'] == -255:
data = map_data(data['items'][0])
session = request.db_slave_session
result = search_all_sources(
session, 'geosubmit', data,
client_addr=request.client_addr,
geoip_db=request.registry.geoip_db,
api_key_log=getattr(request, 'api_key_log', False),
api_key_name=getattr(request, 'api_key_name', None))
else:
result = {'lat': first_item['latitude'],
'lon': first_item['longitude'],
'accuracy': first_item['accuracy']}
if result is None:
stats_client.incr('geosubmit.miss')
result = HTTPNotFound()
result.content_type = 'application/json'
result.body = NOT_FOUND
return result
return {
"location": {
"lat": result['lat'],
"lng": result['lon'],
},
"accuracy": float(result['accuracy']),
}
def test_no_data(self):
result = locate.search_all_sources(
self.db_slave_session, 'm', {},
client_addr=None, geoip_db=self.geoip_db)
self.assertTrue(result is None)
self.check_stats(
counter=[
'm.no_country',
'm.no_geoip_found',
],
)
def test_geoip_unknown(self):
result = locate.search_all_sources(
self.db_slave_session, 'm', {},
client_addr='127.0.0.1', geoip_db=self.geoip_db,
api_key_log=True, api_key_name='test')
self.assertTrue(result is None)
self.check_stats(
counter=[
'm.no_country',
'm.no_geoip_found',
'm.api_log.test.geoip_miss',
],
)
def test_wifi_not_closeby(self):
session = self.db_slave_session
wifis = [
Wifi(key="A1", lat=1.0, lon=1.0),
Wifi(key="B2", lat=1.001, lon=1.002),
Wifi(key="C3", lat=2.002, lon=2.004),
Wifi(key="D4", lat=2.0, lon=2.0),
]
session.add_all(wifis)
session.flush()
result = locate.search_all_sources(
session, 'm',
{'wifi': [{"key": "A1"}, {"key": "C3"}]})
self.assertTrue(result is None)
def test_geoip_city(self):
result = locate.search_all_sources(
self.db_slave_session, 'm', {},
client_addr=FREMONT_IP, geoip_db=self.geoip_db)
self.assertEqual(result,
{'lat': FREMONT_LAT,
'lon': FREMONT_LON,
'accuracy': GEOIP_CITY_ACCURACY})
self.check_stats(
counter=[
'm.country_from_geoip',
'm.geoip_city_found',
'm.geoip_hit',
],
)
def test_wifi_agrees_with_cell_and_lac(self):
# This test checks that when a wifi is at a lat/lon that
# is inside its enclosing LAC and cell, we accept it and
# tighten our accuracy accordingly.
session = self.db_slave_session
key = dict(mcc=BRAZIL_MCC, mnc=VIVO_MNC, lac=12345)
wifi1 = dict(key="1234567890ab")
wifi2 = dict(key="1234890ab567")
wifi3 = dict(key="4321890ab567")
lat = SAO_PAULO_LAT + 0.002
lon = SAO_PAULO_LON + 0.002
data = [
Wifi(lat=lat, lon=lon, **wifi1),
Wifi(lat=lat, lon=lon, **wifi2),
Wifi(lat=lat, lon=lon, **wifi3),
Cell(lat=SAO_PAULO_LAT,
lon=SAO_PAULO_LON,
radio=RADIO_TYPE['gsm'], cid=6789, **key),
Cell(lat=SAO_PAULO_LAT,
lon=SAO_PAULO_LON,
radio=RADIO_TYPE['gsm'], cid=CELLID_LAC, **key),
]
session.add_all(data)
session.flush()
result = locate.search_all_sources(
session, 'm',
{"cell": [dict(radio="gsm", cid=6789, **key)],
"wifi": [wifi1, wifi2, wifi3]},
)
self.assertEqual(result,
{'lat': SAO_PAULO_LAT + 0.002,
'lon': SAO_PAULO_LON + 0.002,
'accuracy': WIFI_MIN_ACCURACY})
self.check_stats(
counter=[
('m.country_from_mcc', 1),
('m.wifi_found', 1),
('m.cell_found', 1),
('m.cell_lac_found', 1),
('m.wifi_hit', 1),
]
)
def test_geoip_country(self):
result = locate.search_all_sources(
self.db_slave_session, 'm', {},
client_addr=GB_IP, geoip_db=self.geoip_db)
self.assertEqual(result,
{'lat': GB_LAT,
'lon': GB_LON,
'accuracy': GB_RADIUS})
self.check_stats(
counter=[
'm.country_from_geoip',
'm.geoip_country_found',
'm.geoip_hit',
],
)
def test_wifi_disagrees_with_cell(self):
# This test checks that when a wifi is at a lat/lon that
# is not in the cell associated with our query, we drop back
# to the cell.
session = self.db_slave_session
key = dict(mcc=BRAZIL_MCC, mnc=VIVO_MNC, lac=12345)
wifi1 = dict(key="1234567890ab")
wifi2 = dict(key="1234890ab567")
wifi3 = dict(key="4321890ab567")
lat = PORTO_ALEGRE_LAT
lon = PORTO_ALEGRE_LON
data = [
Wifi(lat=lat, lon=lon, **wifi1),
Wifi(lat=lat, lon=lon, **wifi2),
Wifi(lat=lat, lon=lon, **wifi3),
Cell(lat=SAO_PAULO_LAT,
lon=SAO_PAULO_LON,
radio=RADIO_TYPE['gsm'], cid=6789, **key),
]
session.add_all(data)
session.flush()
result = locate.search_all_sources(
session, 'm',
{"cell": [dict(radio="gsm", cid=6789, **key)],
"wifi": [wifi1, wifi2, wifi3]},
)
self.assertEqual(result,
{'lat': SAO_PAULO_LAT,
'lon': SAO_PAULO_LON,
'accuracy': CELL_MIN_ACCURACY})
self.check_stats(
counter=[
('m.anomaly.wifi_cell_mismatch', 1),
('m.country_from_mcc', 1),
('m.no_cell_lac_found', 1),
('m.cell_found', 1),
('m.wifi_found', 1),
('m.cell_hit', 1),
]
)
def country_view(request):
client_addr = request.client_addr
geoip_db = request.registry.geoip_db
if request.body == EMPTY_REQUEST and client_addr and geoip_db is not None:
# Optimize common case of geoip-only request
country = geoip_db.country_lookup(client_addr)
if country:
result = HTTPOk()
result.content_type = 'application/json'
result.text = '{"country_code": "%s", "country_name": "%s"}' % (
country.code, country.name)
return result
else:
result = HTTPNotFound()
result.content_type = 'application/json'
result.body = NOT_FOUND
return result
data, errors = preprocess_request(
request,
schema=GeoLocateSchema(),
response=JSONParseError,
accept_empty=True,
)
data = map_data(data)
session = request.db_slave_session
result = search_all_sources(session,
'country',
data,
client_addr=client_addr,
geoip_db=geoip_db,
api_key_log=False,
api_key_name=None,
result_type='country')
if not result:
result = HTTPNotFound()
result.content_type = 'application/json'
result.body = NOT_FOUND
return result
return result
def test_geoip_city(self):
result = locate.search_all_sources(
self.db_slave_session, 'm', {},
client_addr=FREMONT_IP, geoip_db=self.geoip_db,
api_key_log=True, api_key_name='test')
self.assertEqual(result,
{'lat': FREMONT_LAT,
'lon': FREMONT_LON,
'accuracy': GEOIP_CITY_ACCURACY})
self.check_stats(
counter=[
'm.country_from_geoip',
'm.geoip_city_found',
'm.geoip_hit',
'm.api_log.test.geoip_hit',
],
)
def test_wifi_too_few_candidates(self):
session = self.db_slave_session
wifis = [
Wifi(key="001122334455", lat=1.0, lon=1.0),
Wifi(key="112233445566", lat=1.001, lon=1.002),
]
session.add_all(wifis)
session.flush()
result = locate.search_all_sources(
session, 'm',
{'wifi': [{"key": "001122334455"}]})
self.assertTrue(result is None)
self.check_stats(
counter=[
'm.miss',
'm.no_wifi_found',
],
)
def test_wifi_disagrees_with_country(self):
# This test checks that when a wifi is at a lat/lon that
# is not in the country determined by geoip, we still
# trust the wifi position over the geoip result
session = self.db_slave_session
# This lat/lon is Paris, France
(lat, lon) = (PARIS_LAT, PARIS_LON)
wifi1 = dict(key="1234567890ab")
wifi2 = dict(key="1234890ab567")
wifi3 = dict(key="4321890ab567")
data = [
Wifi(lat=lat, lon=lon, **wifi1),
Wifi(lat=lat, lon=lon, **wifi2),
Wifi(lat=lat, lon=lon, **wifi3),
]
session.add_all(data)
session.flush()
result = locate.search_all_sources(
session, 'm',
{"wifi": [wifi1, wifi2, wifi3]},
client_addr=FREMONT_IP, geoip_db=self.geoip_db
)
self.assertEqual(result,
{'lat': PARIS_LAT,
'lon': PARIS_LON,
'accuracy': WIFI_MIN_ACCURACY})
self.check_stats(
counter=[
('m.anomaly.wifi_country_mismatch', 1),
('m.country_from_geoip', 1),
('m.geoip_city_found', 1),
('m.wifi_found', 1),
('m.wifi_hit', 1),
]
)
def test_cell_multiple_lac_hit(self):
session = self.db_slave_session
lat = PARIS_LAT
lon = PARIS_LON
gsm = RADIO_TYPE['gsm']
key = dict(mcc=FRANCE_MCC, mnc=2, lac=3)
key2 = dict(mcc=FRANCE_MCC, mnc=2, lac=4)
data = [
Cell(lat=lat + 0.2, lon=lon + 0.2, radio=gsm,
cid=CELLID_LAC, range=20000, **key),
Cell(lat=lat + 0.2, lon=lon + 0.4, radio=gsm,
cid=5, range=1000, **key),
Cell(lat=lat, lon=lon, radio=gsm,
cid=CELLID_LAC, range=30000, **key2),
Cell(lat=lat + 0.02, lon=lon + 0.02, radio=gsm,
cid=4, range=2000, **key2),
Cell(lat=lat + 0.04, lon=lon + 0.04, radio=gsm,
cid=5, range=3000, **key2),
]
session.add_all(data)
session.flush()
# We have two lacs, both with two cells, but only know about
# one cell in one of them and two in the other.
# The lac with two known cells wins and we use both their
# positions to calculate the final result.
result = locate.search_all_sources(
session, 'm',
{"cell": [
dict(radio="gsm", cid=4, **key),
dict(radio="gsm", cid=9, **key),
dict(radio="gsm", cid=4, **key2),
dict(radio="gsm", cid=5, **key2),
]})
self.assertEqual(result,
{'lat': PARIS_LAT + 0.03,
'lon': PARIS_LON + 0.03,
'accuracy': CELL_MIN_ACCURACY})
def test_geoip_mcc_mismatch_unknown_cell(self):
session = self.db_slave_session
gsm = RADIO_TYPE['gsm']
# We do not add the cell to the DB on purpose
cell = {'radio': gsm, 'mcc': USA_MCC, 'mnc': 1, 'lac': 1, 'cid': 1}
result = locate.search_all_sources(
session, 'm',
{'cell': [cell]},
client_addr=GB_IP, geoip_db=self.geoip_db)
self.assertEqual(result,
{'lat': GB_LAT,
'lon': GB_LON,
'accuracy': GB_RADIUS})
self.check_stats(
counter=[
'm.anomaly.geoip_mcc_mismatch',
],
)
def test_wifi_too_similar_bssids_by_hamming_distance(self):
session = self.db_slave_session
wifis = [
Wifi(key="000000000058", lat=1.0, lon=1.0),
Wifi(key="00000000005c", lat=1.0, lon=1.0),
]
session.add_all(wifis)
session.flush()
result = locate.search_all_sources(
session, 'm',
{'wifi': [{"key": "000000000058"},
{"key": "00000000005c"}]})
self.assertTrue(result is None)
self.check_stats(
counter=[
'm.miss',
'm.no_wifi_found',
],
)
def test_wifi_too_few_matches(self):
session = self.db_slave_session
wifis = [
Wifi(key="001122334455", lat=1.0, lon=1.0),
Wifi(key="112233445566", lat=1.001, lon=1.002),
Wifi(key="223344556677", lat=None, lon=None),
]
session.add_all(wifis)
session.flush()
result = locate.search_all_sources(
session, 'm',
{'wifi': [{"key": "001122334455"}, {"key": "223344556677"}]})
self.assertTrue(result is None)
self.check_stats(
counter=[
'm.miss',
'm.no_wifi_found',
],
)
def test_lac_miss(self):
session = self.db_slave_session
key = dict(mcc=FRANCE_MCC, mnc=2, lac=3)
lat = PARIS_LAT
lon = PARIS_LON
gsm = RADIO_TYPE['gsm']
data = [
Cell(lat=lat, lon=lon, radio=gsm, cid=4, **key),
Cell(lat=lat + 0.002, lon=lon + 0.004, radio=gsm, cid=5, **key),
Cell(lat=1.006, lon=1.006, radio=gsm, cid=6, **key),
Cell(lat=1.0026666, lon=1.0033333, radio=gsm, cid=CELLID_LAC,
range=50000, **key),
]
session.add_all(data)
session.flush()
result = locate.search_all_sources(
session, 'm',
{"cell": [dict(radio="gsm", mcc=FRANCE_MCC, mnc=2, lac=4, cid=5)]})
self.assertTrue(result is None)
def test_wifi_similar_bssids_but_enough_found_clusters(self):
session = self.db_slave_session
wifis = [
Wifi(key="00000000001f", lat=1.0, lon=1.0),
Wifi(key="000000000024", lat=1.00004, lon=1.00004),
]
session.add_all(wifis)
session.flush()
result = locate.search_all_sources(
session, 'm',
{'wifi': [{"key": "00000000001f"},
{"key": "000000000020"},
{"key": "000000000021"},
{"key": "000000000022"},
{"key": "000000000023"},
{"key": "000000000024"}]})
self.assertEqual(result,
{'lat': 1.00002,
'lon': 1.00002,
'accuracy': 100.0})
def test_wifi_too_few_candidates(self):
session = self.db_slave_session
wifis = [
Wifi(key="001122334455", lat=1.0, lon=1.0),
Wifi(key="112233445566", lat=1.001, lon=1.002),
]
session.add_all(wifis)
session.flush()
result = locate.search_all_sources(
session, 'm',
{'wifi': [{"key": "001122334455"}]},
api_key_log=True, api_key_name='test')
self.assertTrue(result is None)
self.check_stats(
counter=[
'm.miss',
'm.no_wifi_found',
'm.api_log.test.geoip_miss',
],
)
def test_wifi_ignore_outlier(self):
session = self.db_slave_session
wifis = [
Wifi(key="001122334455", lat=1.0, lon=1.0),
Wifi(key="112233445566", lat=1.001, lon=1.002),
Wifi(key="223344556677", lat=1.002, lon=1.004),
Wifi(key="334455667788", lat=2.0, lon=2.0),
]
session.add_all(wifis)
session.flush()
result = locate.search_all_sources(
session, 'm',
{'wifi': [
{"key": "001122334455"}, {"key": "112233445566"},
{"key": "223344556677"}, {"key": "334455667788"},
]})
self.assertEqual(result,
{'lat': 1.001,
'lon': 1.002,
'accuracy': 248.6090897})
def test_cell_disagrees_with_lac(self):
# This test checks that when a cell is at a lat/lon that
# is not in the LAC associated with it, we drop back
# to the LAC. This likely represents some kind of internal
# database consistency error, but it might also just be a
# new cell that hasn't been integrated yet or something.
session = self.db_slave_session
key = dict(mcc=BRAZIL_MCC, mnc=VIVO_MNC, lac=12345)
data = [
Cell(lat=PORTO_ALEGRE_LAT,
lon=PORTO_ALEGRE_LON,
radio=RADIO_TYPE['gsm'], cid=6789, **key),
Cell(lat=SAO_PAULO_LAT,
lon=SAO_PAULO_LON,
radio=RADIO_TYPE['gsm'], cid=CELLID_LAC, **key),
]
session.add_all(data)
session.flush()
result = locate.search_all_sources(
session, 'm',
{"cell": [dict(radio="gsm", cid=6789, **key)]},
)
self.assertEqual(result,
{'lat': SAO_PAULO_LAT,
'lon': SAO_PAULO_LON,
'accuracy': LAC_MIN_ACCURACY})
self.check_stats(
counter=[
('m.anomaly.cell_cell_lac_mismatch', 1),
('m.country_from_mcc', 1),
('m.cell_lac_found', 1),
('m.cell_found', 1),
('m.cell_lac_hit', 1),
]
)
def test_geoip_mcc_match(self):
session = self.db_slave_session
gsm = RADIO_TYPE['gsm']
cell = {'radio': gsm, 'mcc': GB_MCC, 'mnc': 1, 'lac': 1, 'cid': 1}
session.add(Cell(**cell))
session.flush()
result = locate.search_all_sources(
session, 'm',
{'cell': [cell]},
client_addr=GB_IP, geoip_db=self.geoip_db)
self.assertEqual(result,
{'lat': GB_LAT,
'lon': GB_LON,
'accuracy': GB_RADIUS})
self.check_stats(
counter=[
'm.country_from_geoip',
'm.geoip_country_found',
],
)
