def add_reports(self, num=1, blue_factor=0, cell_factor=1, wifi_factor=2,
api_key='test', nickname=None,
blue_key=None, cell_mcc=None, wifi_key=None,
lat=None, lon=None):
reports = []
for i in range(num):
pos = CellShardFactory.build()
report = {
'timestamp': time.time() * 1000.0,
'position': {},
'bluetoothBeacons': [],
'cellTowers': [],
'wifiAccessPoints': [],
}
report['position']['latitude'] = lat or pos.lat
report['position']['longitude'] = lon or pos.lon
report['position']['accuracy'] = 17.0 + i
blues = BlueShardFactory.build_batch(blue_factor,
lat=pos.lat, lon=pos.lon)
for blue in blues:
blue_data = {
'macAddress': blue_key or blue.mac,
'signalStrength': -100 + i,
}
report['bluetoothBeacons'].append(blue_data)
cells = CellShardFactory.build_batch(cell_factor,
lat=pos.lat, lon=pos.lon)
for cell in cells:
cell_data = {
'radioType': cell.radio.name,
'mobileCountryCode': cell_mcc or cell.mcc,
'mobileNetworkCode': cell.mnc,
'locationAreaCode': cell.lac,
'cellId': cell.cid,
'primaryScramblingCode': cell.psc,
'signalStrength': -110 + i,
}
report['cellTowers'].append(cell_data)
wifis = WifiShardFactory.build_batch(wifi_factor,
lat=pos.lat, lon=pos.lon)
for wifi in wifis:
wifi_data = {
'macAddress': wifi_key or wifi.mac,
'signalStrength': -90 + i,
'ssid': 'my-wifi',
}
report['wifiAccessPoints'].append(wifi_data)
reports.append(report)
items = [{'api_key': api_key,
'nickname': nickname,
'report': rep} for rep in reports]
self.incoming_queue.enqueue(items)
update_incoming.delay().get()
return reports
def test_database_error(self, app, data_queues,
raven, session, stats, restore_db):
cells = [
CellShardFactory.build(radio=Radio.gsm),
CellShardFactory.build(radio=Radio.wcdma),
CellShardFactory.build(radio=Radio.lte),
]
wifis = WifiShardFactory.build_batch(2)
for model in (CellArea, ):
session.execute(text('drop table %s;' % model.__tablename__))
for name in set([cell.__tablename__ for cell in cells]):
session.execute(text('drop table %s;' % name))
for name in set([wifi.__tablename__ for wifi in wifis]):
session.execute(text('drop table %s;' % name))
query = self.model_query(cells=cells, wifis=wifis)
res = self._call(app, body=query, ip=self.test_ip)
self.check_response(data_queues, res, 'ok', fallback='ipf')
self.check_queue(data_queues, 0)
stats.check(counter=[
('request', [self.metric_path, 'method:post', 'status:200']),
], timer=[
('request', [self.metric_path, 'method:post']),
])
if self.apikey_metrics:
stats.check(counter=[
(self.metric_type + '.result',
['key:test', 'region:GB', 'fallback_allowed:false',
'accuracy:high', 'status:miss']),
])
raven.check([('ProgrammingError', 3)])
def test_export_full(self, celery, session):
now = util.utcnow()
long_ago = now - timedelta(days=367)
CellShardFactory.create_batch(10, radio=Radio.gsm)
CellShardFactory(
radio=Radio.gsm,
created=long_ago,
modified=long_ago,
last_seen=long_ago.date(),
)
session.commit()
pattern = re.compile(
r"MLS-full-cell-export-\d+-\d+-\d+T000000\.csv\.gz")
mock_conn = mock.MagicMock()
mock_bucket = mock.MagicMock(name="bucket")
mock_obj = mock.MagicMock()
mock_conn.return_value.Bucket.return_value = mock_bucket
mock_bucket.Object.return_value = mock_obj
with mock.patch.object(boto3, "resource", mock_conn):
cell_export_full(_bucket="bucket")
s3_key = mock_bucket.Object.call_args[0][0]
assert pattern.search(s3_key)
tmp_file = mock_obj.upload_file.call_args[0][0]
assert pattern.search(tmp_file)
def test_database_error(self, db_errors=0):
cells = [
CellShardFactory.build(radio=Radio.gsm),
CellShardFactory.build(radio=Radio.wcdma),
CellShardFactory.build(radio=Radio.lte),
]
wifis = WifiShardFactory.build_batch(2)
for model in (CellArea, CellOCID, CellAreaOCID):
self.session.execute(text("drop table %s;" % model.__tablename__))
for name in set([cell.__tablename__ for cell in cells]):
self.session.execute(text("drop table %s;" % name))
for name in set([wifi.__tablename__ for wifi in wifis]):
self.session.execute(text("drop table %s;" % name))
query = self.model_query(cells=cells, wifis=wifis)
res = self._call(body=query, ip=self.test_ip)
self.check_response(res, "ok")
self.check_stats(
counter=[("request", [self.metric_path, "method:post", "status:200"])],
timer=[("request", [self.metric_path, "method:post"])],
)
if self.apikey_metrics:
self.check_stats(
counter=[
(
self.metric_type + ".result",
["key:test", "region:GB", "fallback_allowed:false", "accuracy:high", "status:miss"],
)
]
)
self.check_raven([("ProgrammingError", db_errors)])
def test_export_full(self, celery, session):
now = util.utcnow()
long_ago = now - timedelta(days=367)
CellShardFactory.create_batch(10, radio=Radio.gsm)
CellShardFactory(
radio=Radio.gsm, created=long_ago,
modified=long_ago, last_seen=long_ago.date())
session.commit()
pattern = re.compile(
r'MLS-full-cell-export-\d+-\d+-\d+T000000\.csv\.gz')
mock_conn = mock.MagicMock()
mock_bucket = mock.MagicMock(name='bucket')
mock_obj = mock.MagicMock()
mock_conn.return_value.Bucket.return_value = mock_bucket
mock_bucket.Object.return_value = mock_obj
with mock.patch.object(boto3, 'resource', mock_conn):
cell_export_full(_bucket='bucket')
s3_key = mock_bucket.Object.call_args[0][0]
assert pattern.search(s3_key)
tmp_file = mock_obj.upload_file.call_args[0][0]
assert pattern.search(tmp_file)
def test_max_min_radius_update(self):
cell = CellShardFactory(radius=150, samples=3, weight=3.0)
cell_lat = cell.lat
cell_lon = cell.lon
cell.max_lat = cell.lat + 0.001
cell.min_lat = cell.lat - 0.001
cell.max_lon = cell.lon + 0.001
cell.min_lon = cell.lon - 0.001
k1 = dict(radio=cell.radio, mcc=cell.mcc, mnc=cell.mnc,
lac=cell.lac, cid=cell.cid)
obs_factory = CellObservationFactory
obs = [
obs_factory(lat=cell.lat, lon=cell.lon - 0.002, **k1),
obs_factory(lat=cell.lat + 0.004, lon=cell.lon - 0.006, **k1),
]
self.session.commit()
self._queue_and_update_cell(obs)
shard = CellShard.shard_model(cell.cellid)
cells = self.session.query(shard).all()
self.assertEqual(len(cells), 1)
cell = cells[0]
self.assertAlmostEqual(cell.lat, cell_lat + 0.0008)
self.assertAlmostEqual(cell.max_lat, cell_lat + 0.004)
self.assertAlmostEqual(cell.min_lat, cell_lat - 0.001)
self.assertAlmostEqual(cell.lon, cell_lon - 0.0016)
self.assertAlmostEqual(cell.max_lon, cell_lon + 0.001)
self.assertAlmostEqual(cell.min_lon, cell_lon - 0.006)
self.assertEqual(cell.radius, 468)
self.assertEqual(cell.samples, 5)
self.assertAlmostEqual(cell.weight, 5.0, 2)
def test_multiple_cells(self, geoip_db, http_session, session, source):
now = util.utcnow()
cell = CellShardFactory(samples=100)
cell2 = CellShardFactory(
radio=cell.radio,
mcc=cell.mcc,
mnc=cell.mnc,
lac=cell.lac,
cid=cell.cid + 1,
lat=cell.lat + 1.0,
lon=cell.lon + 1.0,
samples=10,
)
session.flush()
query = self.model_query(geoip_db,
http_session,
session,
cells=[cell, cell2])
results = source.search(query)
self.check_model_results(
results,
[cell],
lat=cell.lat + 0.3333333,
lon=cell.lon + 0.3333333,
accuracy=CELL_MAX_ACCURACY,
)
assert results.best().score == station_score(
cell, now) + station_score(cell2, now)
def test_database_error(self, db_errors=0):
cells = [
CellShardFactory.build(radio=Radio.gsm),
CellShardFactory.build(radio=Radio.wcdma),
CellShardFactory.build(radio=Radio.lte),
]
wifis = WifiShardFactory.build_batch(2)
for model in (CellArea, CellOCID, CellAreaOCID):
self.session.execute(text('drop table %s;' % model.__tablename__))
for name in set([cell.__tablename__ for cell in cells]):
self.session.execute(text('drop table %s;' % name))
for name in set([wifi.__tablename__ for wifi in wifis]):
self.session.execute(text('drop table %s;' % name))
query = self.model_query(cells=cells, wifis=wifis)
res = self._call(body=query, ip=self.test_ip)
self.check_response(res, 'ok')
self.check_stats(counter=[
('request', [self.metric_path, 'method:post', 'status:200']),
], timer=[
('request', [self.metric_path, 'method:post']),
])
if self.apikey_metrics:
self.check_stats(counter=[
(self.metric_type + '.result',
['key:test', 'region:GB', 'fallback_allowed:false',
'accuracy:high', 'status:miss']),
])
self.check_raven([('ProgrammingError', db_errors)])
def test_local_export(self, celery, session):
now = util.utcnow()
today = now.date()
long_ago = now - timedelta(days=367)
cell_fixture_fields = ("radio", "cid", "lat", "lon", "mnc", "mcc", "lac")
base_cell = CellShardFactory.build(radio=Radio.wcdma)
cell_key = {
"radio": Radio.wcdma,
"mcc": base_cell.mcc,
"mnc": base_cell.mnc,
"lac": base_cell.lac,
}
cells = set()
for cid in range(190, 200):
cell = dict(cid=cid, lat=base_cell.lat, lon=base_cell.lon, **cell_key)
CellShardFactory(**cell)
cell["lat"] = "%.7f" % cell["lat"]
cell["lon"] = "%.7f" % cell["lon"]
cell["radio"] = "UMTS"
cell_strings = [(field, str(value)) for (field, value) in cell.items()]
cell_tuple = tuple(sorted(cell_strings))
cells.add(cell_tuple)
# add one incomplete / unprocessed cell
CellShardFactory(cid=210, lat=None, lon=None, **cell_key)
# add one really old cell
CellShardFactory(
cid=220,
created=long_ago,
modified=long_ago,
last_seen=long_ago.date(),
**cell_key,
)
session.commit()
with util.selfdestruct_tempdir() as temp_dir:
path = os.path.join(temp_dir, "export.csv.gz")
write_stations_to_csv(session, path, today)
with util.gzip_open(path, "r") as gzip_wrapper:
with gzip_wrapper as gzip_file:
reader = csv.DictReader(gzip_file, CELL_FIELDS)
header = next(reader)
assert "area" in header.values()
exported_cells = set()
for exported_cell in reader:
exported_cell_filtered = [
(field, value)
for (field, value) in exported_cell.items()
if field in cell_fixture_fields
]
exported_cell = tuple(sorted(exported_cell_filtered))
exported_cells.add(exported_cell)
assert cells == exported_cells
def test_cell(self):
now = util.utcnow()
cell = CellShardFactory(samples=10)
self.session.flush()
query = self.model_query(cells=[cell])
results = self.source.search(query)
self.check_model_results(results, [cell])
self.assertAlmostEqual(results.best().score, cell.score(now), 4)
def test_cell_geoip_mismatch(self, app, session):
# UK GeoIP with ambiguous US mcc
uk_cell = CellShardFactory.build(mcc=234)
us_cell = CellShardFactory(mcc=310)
session.flush()
query = self.model_query(cells=[us_cell])
res = self._call(app, body=query, ip=self.test_ip)
self.check_model_response(res, uk_cell, region="GB", fallback="ipf")
def test_update(self):
now = util.utcnow()
invalid_key = dict(lac=None, cid=None)
observations = []
def obs_factory(**kw):
obs = CellObservationFactory.build(**kw)
if obs is not None:
observations.append(obs)
cell1 = CellShardFactory(samples=3, weight=3.0)
lat1, lon1 = (cell1.lat, cell1.lon)
key1 = dict(radio=cell1.radio, lac=cell1.lac, cid=cell1.cid)
obs_factory(lat=lat1, lon=lon1, created=now, **key1)
obs_factory(lat=lat1 + 0.004, lon=lon1 + 0.006, created=now, **key1)
obs_factory(lat=lat1 + 0.006, lon=lon1 + 0.009, created=now, **key1)
# The lac, cid are invalid and should be skipped
obs_factory(created=now, **invalid_key)
obs_factory(created=now, **invalid_key)
cell2 = CellShardFactory(lat=lat1 + 1.0,
lon=lon1 + 1.0,
samples=3,
weight=3.0)
lat2, lon2 = (cell2.lat, cell2.lon)
key2 = dict(radio=cell2.radio, lac=cell2.lac, cid=cell2.cid)
obs_factory(lat=lat2 + 0.001, lon=lon2 + 0.002, created=now, **key2)
obs_factory(lat=lat2 + 0.003, lon=lon2 + 0.006, created=now, **key2)
cell3 = CellShardFactory(samples=100000, weight=100000.0)
lat3, lon3 = (cell3.lat, cell3.lon)
key3 = dict(radio=cell3.radio, lac=cell3.lac, cid=cell3.cid)
for i in range(10):
obs_factory(lat=lat3 + 0.5, lon=lon3 + 0.5, **key3)
self.session.commit()
self._queue_and_update_cell(observations)
shard = CellShard.shard_model(cell1.cellid)
found = (self.session.query(shard).filter(
shard.cellid == cell1.cellid)).one()
self.assertAlmostEqual(found.lat, lat1 + 0.001667, 6)
self.assertAlmostEqual(found.lon, lon1 + 0.0025, 6)
shard = CellShard.shard_model(cell2.cellid)
found = (self.session.query(shard).filter(
shard.cellid == cell2.cellid)).one()
self.assertAlmostEqual(found.lat, lat2 + 0.0008, 6)
self.assertAlmostEqual(found.lon, lon2 + 0.0016, 6)
shard = CellShard.shard_model(cell3.cellid)
found = (self.session.query(shard).filter(
shard.cellid == cell3.cellid)).one()
expected_lat = ((lat3 * 10000) + (lat3 + 0.5) * 10) / 10010
expected_lon = ((lon3 * 10000) + (lon3 + 0.5) * 10) / 10010
self.assertAlmostEqual(found.lat, expected_lat, 7)
self.assertAlmostEqual(found.lon, expected_lon, 7)
def test_cells_over_geoip(self, app, session):
# UK GeoIP with multiple US cells
us_cell1 = CellShardFactory(radio=Radio.gsm, mcc=310, samples=100)
us_cell2 = CellShardFactory(radio=Radio.lte, mcc=311, samples=100)
session.flush()
query = self.model_query(cells=[us_cell1, us_cell2])
res = self._call(app, body=query, ip=self.test_ip)
self.check_model_response(res, us_cell1, region="US")
def add_reports(self, num=1, blue_factor=0, cell_factor=1, wifi_factor=2,
blue_key=None, cell_mcc=None, wifi_key=None,
api_key='test', lat=None, lon=None):
reports = []
for i in range(num):
pos = CellShardFactory.build()
report = {
'timestamp': time.time() * 1000.0,
'position': {},
'bluetoothBeacons': [],
'cellTowers': [],
'wifiAccessPoints': [],
}
report['position']['latitude'] = lat or pos.lat
report['position']['longitude'] = lon or pos.lon
report['position']['accuracy'] = 17.0 + i
blues = BlueShardFactory.build_batch(blue_factor,
lat=pos.lat, lon=pos.lon)
for blue in blues:
blue_data = {
'macAddress': blue_key or blue.mac,
'signalStrength': -100 + i,
}
report['bluetoothBeacons'].append(blue_data)
cells = CellShardFactory.build_batch(cell_factor,
lat=pos.lat, lon=pos.lon)
for cell in cells:
cell_data = {
'radioType': cell.radio.name,
'mobileCountryCode': cell_mcc or cell.mcc,
'mobileNetworkCode': cell.mnc,
'locationAreaCode': cell.lac,
'cellId': cell.cid,
'primaryScramblingCode': cell.psc,
'signalStrength': -110 + i,
}
report['cellTowers'].append(cell_data)
wifis = WifiShardFactory.build_batch(wifi_factor,
lat=pos.lat, lon=pos.lon)
for wifi in wifis:
wifi_data = {
'macAddress': wifi_key or wifi.mac,
'signalStrength': -90 + i,
'ssid': 'my-wifi',
}
report['wifiAccessPoints'].append(wifi_data)
reports.append(report)
items = [{'api_key': api_key, 'report': rep} for rep in reports]
self.queue.enqueue(items)
return reports
def test_cell_geoip_mismatch(self):
# UK GeoIP with ambiguous US mcc
uk_cell = CellShardFactory.build(mcc=234)
us_cell = CellShardFactory(mcc=310)
self.session.flush()
query = self.model_query(cells=[us_cell])
res = self._call(body=query, ip=self.test_ip)
self.check_model_response(res, uk_cell, region='GB', fallback='ipf')
self.check_db_calls(rw=0, ro=2)
def test_export_full(self):
CellShardFactory.create_batch(10, radio=Radio.gsm)
self.session.commit()
with mock_s3() as mock_key:
cell_export_full(_bucket='localhost.bucket')
pat = r'MLS-full-cell-export-\d+-\d+-\d+T000000\.csv\.gz'
self.assertRegex(mock_key.key, pat)
method = mock_key.set_contents_from_filename
self.assertRegex(method.call_args[0][0], pat)
def test_export_full(self):
CellShardFactory.create_batch(10, radio=Radio.gsm)
self.session.commit()
with mock_s3() as mock_key:
cell_export_full(_bucket='localhost.bucket')
pat = r'MLS-full-cell-export-\d+-\d+-\d+T000000\.csv\.gz'
self.assertRegex(mock_key.key, pat)
method = mock_key.set_contents_from_filename
self.assertRegex(method.call_args[0][0], pat)
def test_cell(self, geoip_db, http_session, session, source, stats):
now = util.utcnow()
cell = CellShardFactory(samples=10)
session.flush()
query = self.model_query(
geoip_db, http_session, session, stats,
cells=[cell])
results = source.search(query)
self.check_model_results(results, [cell])
assert results.best().score == cell.score(now)
def test_export_diff(self, celery, session):
CellShardFactory.create_batch(10, radio=Radio.gsm)
session.commit()
pattern = re.compile(
r'MLS-diff-cell-export-\d+-\d+-\d+T\d+0000\.csv\.gz')
with mock_s3() as mock_key:
cell_export_diff(_bucket='localhost.bucket')
assert pattern.search(mock_key.key)
method = mock_key.set_contents_from_filename
assert pattern.search(method.call_args[0][0])
def test_cdma_cell(self, app, session):
"""A CDMA radio is not an error, but the information is ignored."""
cell = CellShardFactory(radio=Radio.gsm, radius=15000)
cell2 = CellShardFactory(
radio=Radio.gsm, radius=35000, lat=cell.lat + 0.0002, lon=cell.lon
)
cell2.radio = Radio.cdma
session.flush()
query = self.model_query(cells=[cell, cell2])
res = self._call(app, body=query)
self.check_model_response(res, cell)
def test_cdma_cell(self):
# Specifying a CDMA radio type works,
# but the information is ignored.
cell = CellShardFactory(radio=Radio.gsm, radius=15000)
cell2 = CellShardFactory(radio=Radio.gsm, radius=35000,
lat=cell.lat + 0.0002, lon=cell.lon)
cell2.radio = Radio.cdma
self.session.flush()
query = self.model_query(cells=[cell, cell2])
res = self._call(body=query)
self.check_model_response(res, cell)
def test_local_export(self):
cell_fixture_fields = ('radio', 'cid', 'lat', 'lon', 'mnc', 'mcc',
'lac')
base_cell = CellShardFactory.build(radio=Radio.wcdma)
cell_key = {
'radio': Radio.wcdma,
'mcc': base_cell.mcc,
'mnc': base_cell.mnc,
'lac': base_cell.lac
}
cells = set()
for cid in range(190, 200):
cell = dict(cid=cid,
lat=base_cell.lat,
lon=base_cell.lon,
**cell_key)
CellShardFactory(**cell)
cell['lat'] = '%.7f' % cell['lat']
cell['lon'] = '%.7f' % cell['lon']
cell['radio'] = 'UMTS'
cell_strings = [(field, str(value))
for (field, value) in cell.items()]
cell_tuple = tuple(sorted(cell_strings))
cells.add(cell_tuple)
# add one incomplete / unprocessed cell
CellShardFactory(cid=210, lat=None, lon=None, **cell_key)
self.session.commit()
with util.selfdestruct_tempdir() as temp_dir:
path = os.path.join(temp_dir, 'export.csv.gz')
write_stations_to_csv(self.session, path)
with util.gzip_open(path, 'r') as gzip_wrapper:
with gzip_wrapper as gzip_file:
reader = csv.DictReader(gzip_file, CELL_FIELDS)
header = six.next(reader)
self.assertTrue('area' in header.values())
exported_cells = set()
for exported_cell in reader:
exported_cell_filtered = [
(field, value)
for (field, value) in exported_cell.items()
if field in cell_fixture_fields
]
exported_cell = tuple(sorted(exported_cell_filtered))
exported_cells.add(exported_cell)
self.assertEqual(cells, exported_cells)
def test_local_export(self, celery, session):
now = util.utcnow()
today = now.date()
long_ago = now - timedelta(days=367)
cell_fixture_fields = (
'radio', 'cid', 'lat', 'lon', 'mnc', 'mcc', 'lac')
base_cell = CellShardFactory.build(radio=Radio.wcdma)
cell_key = {'radio': Radio.wcdma, 'mcc': base_cell.mcc,
'mnc': base_cell.mnc, 'lac': base_cell.lac}
cells = set()
for cid in range(190, 200):
cell = dict(cid=cid, lat=base_cell.lat,
lon=base_cell.lon, **cell_key)
CellShardFactory(**cell)
cell['lat'] = '%.7f' % cell['lat']
cell['lon'] = '%.7f' % cell['lon']
cell['radio'] = 'UMTS'
cell_strings = [
(field, str(value)) for (field, value) in cell.items()]
cell_tuple = tuple(sorted(cell_strings))
cells.add(cell_tuple)
# add one incomplete / unprocessed cell
CellShardFactory(cid=210, lat=None, lon=None, **cell_key)
# add one really old cell
CellShardFactory(cid=220, created=long_ago, modified=long_ago,
last_seen=long_ago.date(), **cell_key)
session.commit()
with util.selfdestruct_tempdir() as temp_dir:
path = os.path.join(temp_dir, 'export.csv.gz')
write_stations_to_csv(session, path, today)
with util.gzip_open(path, 'r') as gzip_wrapper:
with gzip_wrapper as gzip_file:
reader = csv.DictReader(gzip_file, CELL_FIELDS)
header = six.next(reader)
assert 'area' in header.values()
exported_cells = set()
for exported_cell in reader:
exported_cell_filtered = [
(field, value) for (field, value)
in exported_cell.items()
if field in cell_fixture_fields]
exported_cell = tuple(sorted(exported_cell_filtered))
exported_cells.add(exported_cell)
assert cells == exported_cells
def test_multiple_cells(self):
cell = CellShardFactory()
cell2 = CellShardFactory(radio=cell.radio, mcc=cell.mcc, mnc=cell.mnc,
lac=cell.lac, cid=cell.cid + 1,
lat=cell.lat + 1.0, lon=cell.lon + 1.0)
self.session.flush()
query = self.model_query(cells=[cell, cell2])
result = self.source.search(query)
self.check_model_result(
result, cell,
lat=cell.lat + 0.5, lon=cell.lon + 0.5,
accuracy=CELL_MAX_ACCURACY)
def test_cdma_cell(self):
# Specifying a CDMA radio type works,
# but the information is ignored.
cell = CellShardFactory(radio=Radio.gsm, radius=15000)
cell2 = CellShardFactory(radio=Radio.gsm,
radius=35000,
lat=cell.lat + 0.0002,
lon=cell.lon)
cell2.radio = Radio.cdma
self.session.flush()
query = self.model_query(cells=[cell, cell2])
res = self._call(body=query)
self.check_model_response(res, cell)
def test_inconsistent_cell_radio_type(self, app, session):
cell = CellShardFactory(
radio=Radio.wcdma, radius=15000, samples=10)
cell2 = CellShardFactory(
radio=Radio.gsm, radius=35000, samples=5,
lat=cell.lat + 0.0002, lon=cell.lon)
session.flush()
query = self.model_query(cells=[cell, cell2])
query['radioType'] = Radio.lte.name
query['cellTowers'][0]['radio'] = 'lte'
res = self._call(app, body=query)
self.check_model_response(res, cell)
def test_get_cell_multi(self, cache, stats):
cells = CellShardFactory.build_batch(2)
query = self._query(cell=self.cell_model_query(cells))
assert cache.get(query) is None
stats.check(counter=[
('locate.fallback.cache', 1, 1, ['status:bypassed']),
])
def test_cell(self):
cell = CellShardFactory.build(radio=Radio.lte)
query = Query(cell=[{
'radioType': cell.radio,
'mobileCountryCode': cell.mcc,
'mobileNetworkCode': cell.mnc,
'locationAreaCode': cell.lac,
'cellId': cell.cid,
'age': 1200,
'asu': None,
'primaryScramblingCode': 5,
'signalStrength': -70,
'timingAdvance': 15,
'unknown_field': 'foo'
}])
data = OUTBOUND_SCHEMA.deserialize(query.json())
assert (data == {
'cellTowers': [{
'radioType': cell.radio.name,
'mobileCountryCode': cell.mcc,
'mobileNetworkCode': cell.mnc,
'locationAreaCode': cell.lac,
'cellId': cell.cid,
'primaryScramblingCode': 5,
'age': 1200,
'signalStrength': -70,
'timingAdvance': 15,
}],
'fallbacks': {
'lacf': True
},
})
def test_cell(self):
cell = CellShardFactory.build(radio=Radio.lte)
cell_query = self.cell_model_query([cell])
query = Query(cell=cell_query)
assert len(query.cell) == 1
assert query.expected_accuracy is DataAccuracy.medium
assert query.geoip_only is False
query_cell = query.cell[0]
for key, value in cell_query[0].items():
query_value = getattr(query_cell, key, None)
if key == 'radioType':
assert query_value is cell.radio
else:
assert query_value == value
assert len(query.cell_area) == 1
query_area = query.cell_area[0]
for key, value in cell_query[0].items():
query_value = getattr(query_area, key, None)
if key == 'radioType':
assert query_value is cell.radio
elif key in ('cellId', 'primaryScramblingCode'):
pass
else:
assert query_value == value
def test_get_cell_multi(self):
cells = CellShardFactory.build_batch(2)
query = Query(cell=self.cell_model_query(cells))
self.assertEqual(self.cache.get(query), None)
self.check_stats(counter=[
('locate.fallback.cache', 1, 1, ['status:bypassed']),
])
def test_dont_recache(self, geoip_db, http_session, session, source,
stats):
cell = CellShardFactory.build()
mock_redis_client = self._mock_redis_client()
mock_redis_client.mget.return_value = [self.fallback_cached_result]
with requests_mock.Mocker() as mock_request:
mock_request.register_uri('POST',
requests_mock.ANY,
json=self.fallback_result)
with mock.patch.object(source.cache, 'redis_client',
mock_redis_client):
query = self.model_query(geoip_db,
http_session,
session,
stats,
cells=[cell])
results = source.search(query)
self.check_model_results(results, [self.fallback_model])
assert mock_redis_client.mget.called
assert not mock_redis_client.mset.called
stats.check(counter=[
('locate.fallback.cache', ['status:hit']),
])
def test_cell_geoip_mismatch(self):
# UK GeoIP with US mcc
cell = CellShardFactory.create(mcc=310)
query = self.model_query(cells=[cell])
res = self._call(body=query, ip=self.test_ip)
self.check_model_response(res, cell, region='US')
self.check_db_calls(rw=0, ro=0)
def test_api_key_disallows(self):
api_key = ApiKeyFactory.build(allow_fallback=False)
cells = CellShardFactory.build_batch(2)
wifis = WifiShardFactory.build_batch(2)
query = self.model_query(cells=cells, wifis=wifis, api_key=api_key)
self.check_should_search(query, False)
def test_cache_empty_result(self):
cell = CellShardFactory.build()
with requests_mock.Mocker() as mock_request:
mock_request.register_uri('POST',
requests_mock.ANY,
json=LocationNotFound.json_body(),
status_code=404)
query = self.model_query(cells=[cell])
results = self.source.search(query)
self.check_model_results(results, None)
self.assertEqual(mock_request.call_count, 1)
self.check_stats(counter=[
('locate.fallback.cache', ['status:miss']),
('locate.fallback.lookup',
['fallback_name:fall', 'status:404']),
])
query = self.model_query(cells=[cell])
results = self.source.search(query)
self.check_model_results(results, None)
self.assertEqual(mock_request.call_count, 1)
self.check_stats(counter=[
('locate.fallback.cache', ['status:hit']),
('locate.fallback.lookup',
['fallback_name:fall', 'status:404']),
])
def test_get_cell_multi(self):
cells = CellShardFactory.build_batch(2)
query = self._query(cell=self.cell_model_query(cells))
self.assertEqual(self.cache.get(query), None)
self.check_stats(counter=[
('locate.fallback.cache', 1, 1, ['status:bypassed']),
])
def test_success(self):
cell = CellShardFactory.build()
with requests_mock.Mocker() as mock_request:
mock_request.register_uri('POST',
requests_mock.ANY,
json=self.fallback_result)
query = self.model_query(
cells=[cell],
fallback={
'lacf': True,
'ipf': False,
},
)
results = self.source.search(query)
self.check_model_results(results, [self.fallback_model])
self.assertAlmostEqual(results.best().score, 5.0, 4)
request_json = mock_request.request_history[0].json()
self.assertEqual(request_json['fallbacks'], {'lacf': True})
self.check_stats(counter=[
('locate.fallback.lookup', ['fallback_name:fall', 'status:200']),
],
timer=[
('locate.fallback.lookup', ['fallback_name:fall'
]),
])
def test_success(self):
cell = CellShardFactory.build()
with requests_mock.Mocker() as mock_request:
mock_request.register_uri(
'POST', requests_mock.ANY, json=self.fallback_result)
query = self.model_query(
cells=[cell],
fallback={
'lacf': True,
'ipf': False,
},
)
result = self.source.search(query)
self.check_model_result(result, self.fallback_model)
request_json = mock_request.request_history[0].json()
self.assertEqual(request_json['fallbacks'], {'lacf': True})
self.check_stats(counter=[
('locate.fallback.lookup', ['status:200']),
], timer=[
'locate.fallback.lookup',
])
def test_api_key_disallows(self):
api_key = ApiKeyFactory.build(allow_fallback=False)
cells = CellShardFactory.build_batch(2)
wifis = WifiShardFactory.build_batch(2)
query = self.model_query(cells=cells, wifis=wifis, api_key=api_key)
self.check_should_search(query, False)
def test_set_cache_redis_failure(self):
cell = CellShardFactory.build()
mock_redis_client = self._mock_redis_client()
mock_redis_client.mget.return_value = []
mock_redis_client.mset.side_effect = RedisError()
mock_redis_client.expire.side_effect = RedisError()
mock_redis_client.execute.side_effect = RedisError()
with requests_mock.Mocker() as mock_request:
mock_request.register_uri(
'POST', requests_mock.ANY, json=self.fallback_result)
with mock.patch.object(self.source.cache, 'redis_client',
mock_redis_client):
query = self.model_query(cells=[cell])
result = self.source.search(query)
self.check_model_result(result, self.fallback_model)
self.assertTrue(mock_redis_client.mget.called)
self.assertTrue(mock_redis_client.mset.called)
self.assertTrue(mock_request.called)
self.check_stats(counter=[
('locate.fallback.cache', ['status:miss']),
])
def test_cache_single_cell(self):
cell = CellShardFactory.build()
with requests_mock.Mocker() as mock_request:
mock_request.register_uri(
'POST', requests_mock.ANY, json=self.fallback_result)
query = self.model_query(cells=[cell])
query.cell[0].signal = -77
result = self.source.search(query)
self.check_model_result(result, self.fallback_model)
self.assertEqual(mock_request.call_count, 1)
self.check_stats(counter=[
('locate.fallback.cache', ['status:miss']),
('locate.fallback.lookup', ['status:200']),
], timer=[
'locate.fallback.lookup',
])
# vary the signal strength, not part of cache key
query.cell[0].signal = -82
result = self.source.search(query)
self.check_model_result(result, self.fallback_model)
self.assertEqual(mock_request.call_count, 1)
self.check_stats(counter=[
('locate.fallback.cache', ['status:hit']),
('locate.fallback.lookup', ['status:200']),
], timer=[
'locate.fallback.lookup',
])
def test_cell_ambiguous(self):
# cell with ambiguous mcc to region mapping
cell = CellShardFactory.create(mcc=234)
query = self.model_query(cells=[cell])
res = self._call(body=query)
self.check_model_response(res, cell, region='GB')
self.check_db_calls(rw=0, ro=0)
def test_set_cache_redis_failure(self, geoip_db, http_session, raven,
session, source, stats):
cell = CellShardFactory.build()
mock_redis_client = self._mock_redis_client()
mock_redis_client.mget.return_value = []
mock_redis_client.mset.side_effect = RedisError()
mock_redis_client.expire.side_effect = RedisError()
mock_redis_client.execute.side_effect = RedisError()
with requests_mock.Mocker() as mock_request:
mock_request.register_uri('POST',
requests_mock.ANY,
json=self.fallback_result)
with mock.patch.object(source.cache, 'redis_client',
mock_redis_client):
query = self.model_query(geoip_db,
http_session,
session,
stats,
cells=[cell])
results = source.search(query)
self.check_model_results(results, [self.fallback_model])
assert mock_redis_client.mget.called
assert mock_redis_client.mset.called
assert mock_request.called
raven.check([('RedisError', 1)])
stats.check(counter=[
('locate.fallback.cache', ['status:miss']),
])
def test_success(self, geoip_db, http_session, session, source, stats):
cell = CellShardFactory.build()
with requests_mock.Mocker() as mock_request:
mock_request.register_uri('POST',
requests_mock.ANY,
json=self.fallback_result)
query = self.model_query(
geoip_db,
http_session,
session,
stats,
cells=[cell],
fallback={
'lacf': True,
'ipf': False,
},
)
results = source.search(query)
self.check_model_results(results, [self.fallback_model])
assert results.best().score == 5.0
request_json = mock_request.request_history[0].json()
assert request_json['fallbacks'] == {'lacf': True}
stats.check(counter=[
('locate.fallback.lookup', ['fallback_name:fall', 'status:200']),
],
timer=[
('locate.fallback.lookup', ['fallback_name:fall']),
])
def test_cache_empty_result(self):
cell = CellShardFactory.build()
with requests_mock.Mocker() as mock_request:
mock_request.register_uri(
'POST',
requests_mock.ANY,
json=LocationNotFound.json_body(),
status_code=404
)
query = self.model_query(cells=[cell])
result = self.source.search(query)
self.check_model_result(result, None)
self.assertEqual(mock_request.call_count, 1)
self.check_stats(counter=[
('locate.fallback.cache', ['status:miss']),
('locate.fallback.lookup', ['status:404']),
])
query = self.model_query(cells=[cell])
result = self.source.search(query)
self.check_model_result(result, None)
self.assertEqual(mock_request.call_count, 1)
self.check_stats(counter=[
('locate.fallback.cache', ['status:hit']),
('locate.fallback.lookup', ['status:404']),
])
def test_export_full(self, celery, session):
now = util.utcnow()
long_ago = now - timedelta(days=367)
CellShardFactory.create_batch(10, radio=Radio.gsm)
CellShardFactory(
radio=Radio.gsm, created=long_ago,
modified=long_ago, last_seen=long_ago.date())
session.commit()
pattern = re.compile(
r'MLS-full-cell-export-\d+-\d+-\d+T000000\.csv\.gz')
with mock_s3() as mock_key:
cell_export_full(_bucket='localhost.bucket')
assert pattern.search(mock_key.key)
method = mock_key.set_contents_from_filename
assert pattern.search(method.call_args[0][0])
def test_cell(self):
cell = CellShardFactory.build(radio=Radio.lte)
cell_query = self.cell_model_query([cell])
query = Query(cell=cell_query)
self.assertEqual(len(query.cell), 1)
self.assertEqual(query.expected_accuracy, DataAccuracy.medium)
query_cell = query.cell[0]
for key, value in cell_query[0].items():
query_value = getattr(query_cell, key, None)
if key == 'radio':
self.assertEqual(query_value, cell.radio)
else:
self.assertEqual(query_value, value)
self.assertEqual(len(query.cell_area), 1)
query_area = query.cell_area[0]
for key, value in cell_query[0].items():
query_value = getattr(query_area, key, None)
if key == 'radio':
self.assertEqual(query_value, cell.radio)
elif key in ('cid', 'psc'):
pass
else:
self.assertEqual(query_value, value)
def test_cell_geoip_mismatch(self):
# UK GeoIP with US mcc
cell = CellShardFactory.create(mcc=310)
query = self.model_query(cells=[cell])
res = self._call(body=query, ip=self.test_ip)
self.check_model_response(res, cell, region='US')
self.check_db_calls(rw=0, ro=0)
def test_blocklist(self):
now = util.utcnow()
today = now.date()
observations = CellObservationFactory.build_batch(3)
obs = observations[0]
CellShardFactory(
radio=obs.radio,
mcc=obs.mcc,
mnc=obs.mnc,
lac=obs.lac,
cid=obs.cid,
created=now,
block_first=today - timedelta(days=10),
block_last=today,
block_count=1,
)
self.session.commit()
self._queue_and_update(observations)
blocks = []
for obs in observations:
shard = CellShard.shard_model(obs.cellid)
cell = (self.session.query(shard).filter(
shard.cellid == obs.cellid)).one()
if cell.blocked():
blocks.append(cell)
self.assertEqual(len(blocks), 1)
self.check_statcounter(StatKey.cell, 2)
self.check_statcounter(StatKey.unique_cell, 2)
def test_set_cache_redis_failure(self, geoip_db, http_session,
raven, session, source, stats):
cell = CellShardFactory.build()
mock_redis_client = _mock_redis_client()
mock_redis_client.mget.return_value = []
mock_redis_client.mset.side_effect = RedisError()
mock_redis_client.expire.side_effect = RedisError()
mock_redis_client.execute.side_effect = RedisError()
with requests_mock.Mocker() as mock_request:
mock_request.register_uri(
'POST', requests_mock.ANY, json=self.fallback_result)
with mock.patch.object(source.caches[DEFAULT_SCHEMA],
'redis_client',
mock_redis_client):
query = self.model_query(
geoip_db, http_session, session, stats,
cells=[cell])
results = source.search(query)
self.check_model_results(results, [self.fallback_model])
assert mock_redis_client.mget.called
assert mock_redis_client.mset.called
assert mock_request.called
raven.check([('RedisError', 1)])
stats.check(counter=[
('locate.fallback.cache', [self.fallback_tag, 'status:miss']),
])
def test_cell_ambiguous(self):
# cell with ambiguous mcc to region mapping
cell = CellShardFactory.create(mcc=234)
query = self.model_query(cells=[cell])
res = self._call(body=query)
self.check_model_response(res, cell, region='GB')
self.check_db_calls(rw=0, ro=0)
def test_cell(self):
cell = CellShardFactory.build(radio=Radio.lte)
query = Query(cell=[
{'radioType': cell.radio,
'mobileCountryCode': cell.mcc,
'mobileNetworkCode': cell.mnc,
'locationAreaCode': cell.lac,
'cellId': cell.cid,
'age': 1200,
'signalStrength': -70,
'timingAdvance': 15,
'unknown_field': 'foo'}])
data = self._call(query.json())
assert (data == {
'cellTowers': [{
'radioType': cell.radio.name,
'mobileCountryCode': cell.mcc,
'mobileNetworkCode': cell.mnc,
'locationAreaCode': cell.lac,
'cellId': cell.cid,
'age': 1200,
'signalStrength': -70,
'timingAdvance': 15,
}],
'considerIp': False,
})
def test_dont_recache(self, geoip_db, http_session,
session, source, stats):
cell = CellShardFactory.build()
mock_redis_client = _mock_redis_client()
mock_redis_client.mget.return_value = [self.fallback_cached_result]
with requests_mock.Mocker() as mock_request:
mock_request.register_uri(
'POST', requests_mock.ANY, json=self.fallback_result)
with mock.patch.object(source.caches[DEFAULT_SCHEMA],
'redis_client',
mock_redis_client):
query = self.model_query(
geoip_db, http_session, session, stats,
cells=[cell])
results = source.search(query)
self.check_model_results(results, [self.fallback_model])
assert mock_redis_client.mget.called
assert not mock_redis_client.mset.called
stats.check(counter=[
('locate.fallback.cache', [self.fallback_tag, 'status:hit']),
])
def test_cell(self):
cell = CellShardFactory.build(radio=Radio.lte)
query = Query(cell=[
{'radioType': cell.radio,
'mobileCountryCode': cell.mcc,
'mobileNetworkCode': cell.mnc,
'locationAreaCode': cell.lac,
'cellId': cell.cid,
'asu': 17,
'primaryScramblingCode': 5,
'signalStrength': -70,
'timingAdvance': 15,
'unknown_field': 'foo'}])
data = self._call(query.json())
assert (data == {
'cells': [{
'radio': cell.radio.name,
'mcc': cell.mcc,
'mnc': cell.mnc,
'lac': cell.lac,
'cid': cell.cid,
'asu': 17,
'psc': 5,
'signal': -70,
'tA': 15,
}],
'fallbacks': {'lacf': True},
'token': None,
})
def test_success(self, geoip_db, http_session, session, source, stats):
cell = CellShardFactory.build()
with requests_mock.Mocker() as mock_request:
mock_request.register_uri(
'POST', requests_mock.ANY, json=self.fallback_result)
query = self.model_query(
geoip_db, http_session, session, stats,
cells=[cell],
fallback={
'lacf': True,
'ipf': False,
},
)
results = source.search(query)
self.check_model_results(results, [self.fallback_model])
assert results.best().score == 5.0
request_json = mock_request.request_history[0].json()
self._check_success_fallbacks(request_json)
stats.check(counter=[
('locate.fallback.lookup', [self.fallback_tag, 'status:200']),
], timer=[
('locate.fallback.lookup', [self.fallback_tag]),
])
def test_medium_hit(self):
cells = CellShardFactory.build_batch(1)
self._make_query(self._make_result(accuracy=50000.0), cell=cells)
self.check_stats(counter=[
('locate.result',
['key:key', 'region:none', 'fallback_allowed:false',
'accuracy:medium', 'status:hit']),
])
def test_mixed_cell_wifi(self):
cells = CellShardFactory.build_batch(1)
wifis = WifiShardFactory.build_batch(2)
query = Query(
cell=self.cell_model_query(cells),
wifi=self.wifi_model_query(wifis))
self.assertEqual(query.expected_accuracy, DataAccuracy.high)
def test_cell_area_duplicated(self):
cell = CellShardFactory.build()
cell_query = self.cell_model_query([cell, cell, cell])
cell_query[1]['cid'] += 2
cell_query[2]['cid'] += 1
query = Query(cell=cell_query)
self.assertEqual(len(query.cell), 3)
self.assertEqual(len(query.cell_area), 1)
def _one_cell_query(self, radio=True):
cell = CellShardFactory.build()
query = {'lat': cell.lat, 'lon': cell.lon,
'cell': [{'mcc': cell.mcc, 'mnc': cell.mnc,
'lac': cell.lac, 'cid': cell.cid}]}
if radio:
query['cell'][0]['radio'] = cell.radio.name
return (cell, query)
