def test_multiple_cells(self, geoip_db, http_session, session, source):
now = util.utcnow()
cell = CellShardFactory(samples=100)
cell2 = CellShardFactory(
radio=cell.radio,
mcc=cell.mcc,
mnc=cell.mnc,
lac=cell.lac,
cid=cell.cid + 1,
lat=cell.lat + 1.0,
lon=cell.lon + 1.0,
samples=10,
)
session.flush()
query = self.model_query(geoip_db,
http_session,
session,
cells=[cell, cell2])
results = source.search(query)
self.check_model_results(
results,
[cell],
lat=cell.lat + 0.3333333,
lon=cell.lon + 0.3333333,
accuracy=CELL_MAX_ACCURACY,
)
assert results.best().score == station_score(
cell, now) + station_score(cell2, now)
def test_local_export(self, celery, session):
now = util.utcnow()
today = now.date()
long_ago = now - timedelta(days=367)
cell_fixture_fields = ("radio", "cid", "lat", "lon", "mnc", "mcc", "lac")
base_cell = CellShardFactory.build(radio=Radio.wcdma)
cell_key = {
"radio": Radio.wcdma,
"mcc": base_cell.mcc,
"mnc": base_cell.mnc,
"lac": base_cell.lac,
}
cells = set()
for cid in range(190, 200):
cell = dict(cid=cid, lat=base_cell.lat, lon=base_cell.lon, **cell_key)
CellShardFactory(**cell)
cell["lat"] = "%.7f" % cell["lat"]
cell["lon"] = "%.7f" % cell["lon"]
cell["radio"] = "UMTS"
cell_strings = [(field, str(value)) for (field, value) in cell.items()]
cell_tuple = tuple(sorted(cell_strings))
cells.add(cell_tuple)
# add one incomplete / unprocessed cell
CellShardFactory(cid=210, lat=None, lon=None, **cell_key)
# add one really old cell
CellShardFactory(
cid=220,
created=long_ago,
modified=long_ago,
last_seen=long_ago.date(),
**cell_key,
)
session.commit()
with util.selfdestruct_tempdir() as temp_dir:
path = os.path.join(temp_dir, "export.csv.gz")
write_stations_to_csv(session, path, today)
with util.gzip_open(path, "r") as gzip_wrapper:
with gzip_wrapper as gzip_file:
reader = csv.DictReader(gzip_file, CELL_FIELDS)
header = next(reader)
assert "area" in header.values()
exported_cells = set()
for exported_cell in reader:
exported_cell_filtered = [
(field, value)
for (field, value) in exported_cell.items()
if field in cell_fixture_fields
]
exported_cell = tuple(sorted(exported_cell_filtered))
exported_cells.add(exported_cell)
assert cells == exported_cells
def test_update(self):
now = util.utcnow()
invalid_key = dict(lac=None, cid=None)
observations = []
def obs_factory(**kw):
obs = CellObservationFactory.build(**kw)
if obs is not None:
observations.append(obs)
cell1 = CellShardFactory(samples=3, weight=3.0)
lat1, lon1 = (cell1.lat, cell1.lon)
key1 = dict(radio=cell1.radio, lac=cell1.lac, cid=cell1.cid)
obs_factory(lat=lat1, lon=lon1, created=now, **key1)
obs_factory(lat=lat1 + 0.004, lon=lon1 + 0.006, created=now, **key1)
obs_factory(lat=lat1 + 0.006, lon=lon1 + 0.009, created=now, **key1)
# The lac, cid are invalid and should be skipped
obs_factory(created=now, **invalid_key)
obs_factory(created=now, **invalid_key)
cell2 = CellShardFactory(lat=lat1 + 1.0,
lon=lon1 + 1.0,
samples=3,
weight=3.0)
lat2, lon2 = (cell2.lat, cell2.lon)
key2 = dict(radio=cell2.radio, lac=cell2.lac, cid=cell2.cid)
obs_factory(lat=lat2 + 0.001, lon=lon2 + 0.002, created=now, **key2)
obs_factory(lat=lat2 + 0.003, lon=lon2 + 0.006, created=now, **key2)
cell3 = CellShardFactory(samples=100000, weight=100000.0)
lat3, lon3 = (cell3.lat, cell3.lon)
key3 = dict(radio=cell3.radio, lac=cell3.lac, cid=cell3.cid)
for i in range(10):
obs_factory(lat=lat3 + 0.5, lon=lon3 + 0.5, **key3)
self.session.commit()
self._queue_and_update_cell(observations)
shard = CellShard.shard_model(cell1.cellid)
found = (self.session.query(shard).filter(
shard.cellid == cell1.cellid)).one()
self.assertAlmostEqual(found.lat, lat1 + 0.001667, 6)
self.assertAlmostEqual(found.lon, lon1 + 0.0025, 6)
shard = CellShard.shard_model(cell2.cellid)
found = (self.session.query(shard).filter(
shard.cellid == cell2.cellid)).one()
self.assertAlmostEqual(found.lat, lat2 + 0.0008, 6)
self.assertAlmostEqual(found.lon, lon2 + 0.0016, 6)
shard = CellShard.shard_model(cell3.cellid)
found = (self.session.query(shard).filter(
shard.cellid == cell3.cellid)).one()
expected_lat = ((lat3 * 10000) + (lat3 + 0.5) * 10) / 10010
expected_lon = ((lon3 * 10000) + (lon3 + 0.5) * 10) / 10010
self.assertAlmostEqual(found.lat, expected_lat, 7)
self.assertAlmostEqual(found.lon, expected_lon, 7)
def test_cells_over_geoip(self, app, session):
# UK GeoIP with multiple US cells
us_cell1 = CellShardFactory(radio=Radio.gsm, mcc=310, samples=100)
us_cell2 = CellShardFactory(radio=Radio.lte, mcc=311, samples=100)
session.flush()
query = self.model_query(cells=[us_cell1, us_cell2])
res = self._call(app, body=query, ip=self.test_ip)
self.check_model_response(res, us_cell1, region="US")
def test_cdma_cell(self, app, session):
"""A CDMA radio is not an error, but the information is ignored."""
cell = CellShardFactory(radio=Radio.gsm, radius=15000)
cell2 = CellShardFactory(
radio=Radio.gsm, radius=35000, lat=cell.lat + 0.0002, lon=cell.lon
)
cell2.radio = Radio.cdma
session.flush()
query = self.model_query(cells=[cell, cell2])
res = self._call(app, body=query)
self.check_model_response(res, cell)
def test_local_export(self):
cell_fixture_fields = ('radio', 'cid', 'lat', 'lon', 'mnc', 'mcc',
'lac')
base_cell = CellShardFactory.build(radio=Radio.wcdma)
cell_key = {
'radio': Radio.wcdma,
'mcc': base_cell.mcc,
'mnc': base_cell.mnc,
'lac': base_cell.lac
}
cells = set()
for cid in range(190, 200):
cell = dict(cid=cid,
lat=base_cell.lat,
lon=base_cell.lon,
**cell_key)
CellShardFactory(**cell)
cell['lat'] = '%.7f' % cell['lat']
cell['lon'] = '%.7f' % cell['lon']
cell['radio'] = 'UMTS'
cell_strings = [(field, str(value))
for (field, value) in cell.items()]
cell_tuple = tuple(sorted(cell_strings))
cells.add(cell_tuple)
# add one incomplete / unprocessed cell
CellShardFactory(cid=210, lat=None, lon=None, **cell_key)
self.session.commit()
with util.selfdestruct_tempdir() as temp_dir:
path = os.path.join(temp_dir, 'export.csv.gz')
write_stations_to_csv(self.session, path)
with util.gzip_open(path, 'r') as gzip_wrapper:
with gzip_wrapper as gzip_file:
reader = csv.DictReader(gzip_file, CELL_FIELDS)
header = six.next(reader)
self.assertTrue('area' in header.values())
exported_cells = set()
for exported_cell in reader:
exported_cell_filtered = [
(field, value)
for (field, value) in exported_cell.items()
if field in cell_fixture_fields
]
exported_cell = tuple(sorted(exported_cell_filtered))
exported_cells.add(exported_cell)
self.assertEqual(cells, exported_cells)
def test_local_export(self, celery, session):
now = util.utcnow()
today = now.date()
long_ago = now - timedelta(days=367)
cell_fixture_fields = (
'radio', 'cid', 'lat', 'lon', 'mnc', 'mcc', 'lac')
base_cell = CellShardFactory.build(radio=Radio.wcdma)
cell_key = {'radio': Radio.wcdma, 'mcc': base_cell.mcc,
'mnc': base_cell.mnc, 'lac': base_cell.lac}
cells = set()
for cid in range(190, 200):
cell = dict(cid=cid, lat=base_cell.lat,
lon=base_cell.lon, **cell_key)
CellShardFactory(**cell)
cell['lat'] = '%.7f' % cell['lat']
cell['lon'] = '%.7f' % cell['lon']
cell['radio'] = 'UMTS'
cell_strings = [
(field, str(value)) for (field, value) in cell.items()]
cell_tuple = tuple(sorted(cell_strings))
cells.add(cell_tuple)
# add one incomplete / unprocessed cell
CellShardFactory(cid=210, lat=None, lon=None, **cell_key)
# add one really old cell
CellShardFactory(cid=220, created=long_ago, modified=long_ago,
last_seen=long_ago.date(), **cell_key)
session.commit()
with util.selfdestruct_tempdir() as temp_dir:
path = os.path.join(temp_dir, 'export.csv.gz')
write_stations_to_csv(session, path, today)
with util.gzip_open(path, 'r') as gzip_wrapper:
with gzip_wrapper as gzip_file:
reader = csv.DictReader(gzip_file, CELL_FIELDS)
header = six.next(reader)
assert 'area' in header.values()
exported_cells = set()
for exported_cell in reader:
exported_cell_filtered = [
(field, value) for (field, value)
in exported_cell.items()
if field in cell_fixture_fields]
exported_cell = tuple(sorted(exported_cell_filtered))
exported_cells.add(exported_cell)
assert cells == exported_cells
def test_multiple_cells(self):
cell = CellShardFactory()
cell2 = CellShardFactory(radio=cell.radio, mcc=cell.mcc, mnc=cell.mnc,
lac=cell.lac, cid=cell.cid + 1,
lat=cell.lat + 1.0, lon=cell.lon + 1.0)
self.session.flush()
query = self.model_query(cells=[cell, cell2])
result = self.source.search(query)
self.check_model_result(
result, cell,
lat=cell.lat + 0.5, lon=cell.lon + 0.5,
accuracy=CELL_MAX_ACCURACY)
def test_cdma_cell(self):
# Specifying a CDMA radio type works,
# but the information is ignored.
cell = CellShardFactory(radio=Radio.gsm, radius=15000)
cell2 = CellShardFactory(radio=Radio.gsm,
radius=35000,
lat=cell.lat + 0.0002,
lon=cell.lon)
cell2.radio = Radio.cdma
self.session.flush()
query = self.model_query(cells=[cell, cell2])
res = self._call(body=query)
self.check_model_response(res, cell)
def test_inconsistent_cell_radio_type(self, app, session):
cell = CellShardFactory(
radio=Radio.wcdma, radius=15000, samples=10)
cell2 = CellShardFactory(
radio=Radio.gsm, radius=35000, samples=5,
lat=cell.lat + 0.0002, lon=cell.lon)
session.flush()
query = self.model_query(cells=[cell, cell2])
query['radioType'] = Radio.lte.name
query['cellTowers'][0]['radio'] = 'lte'
res = self._call(app, body=query)
self.check_model_response(res, cell)
def test_cell(self):
cell = CellShardFactory()
self.session.flush()
query = self.model_query(cells=[cell])
result = self.source.search(query)
self.check_model_result(result, cell)
def test_cell_geoip_match(self, app, session):
cell = CellShardFactory(mcc=234)
session.flush()
query = self.model_query(cells=[cell])
res = self._call(app, body=query, ip=self.test_ip)
self.check_model_response(res, cell, region="GB")
def test_blocklist(self):
now = util.utcnow()
today = now.date()
observations = CellObservationFactory.build_batch(3)
obs = observations[0]
CellShardFactory(
radio=obs.radio,
mcc=obs.mcc,
mnc=obs.mnc,
lac=obs.lac,
cid=obs.cid,
created=now,
block_first=today - timedelta(days=10),
block_last=today,
block_count=1,
)
self.session.commit()
self._queue_and_update(observations)
blocks = []
for obs in observations:
shard = CellShard.shard_model(obs.cellid)
cell = (self.session.query(shard).filter(
shard.cellid == obs.cellid)).one()
if cell.blocked():
blocks.append(cell)
self.assertEqual(len(blocks), 1)
self.check_statcounter(StatKey.cell, 2)
self.check_statcounter(StatKey.unique_cell, 2)
def test_max_min_radius_update(self):
cell = CellShardFactory(radius=150, samples=3)
cell_lat = cell.lat
cell_lon = cell.lon
cell.max_lat = cell.lat + 0.001
cell.min_lat = cell.lat - 0.001
cell.max_lon = cell.lon + 0.001
cell.min_lon = cell.lon - 0.001
k1 = dict(radio=cell.radio,
mcc=cell.mcc,
mnc=cell.mnc,
lac=cell.lac,
cid=cell.cid)
obs_factory = CellObservationFactory
obs = [
obs_factory(lat=cell.lat, lon=cell.lon - 0.002, **k1),
obs_factory(lat=cell.lat + 0.004, lon=cell.lon - 0.006, **k1),
]
self.session.commit()
self._queue_and_update(obs)
shard = CellShard.shard_model(cell.cellid)
cells = self.session.query(shard).all()
self.assertEqual(len(cells), 1)
cell = cells[0]
self.assertAlmostEqual(cell.lat, cell_lat + 0.0008)
self.assertAlmostEqual(cell.max_lat, cell_lat + 0.004)
self.assertAlmostEqual(cell.min_lat, cell_lat - 0.001)
self.assertAlmostEqual(cell.lon, cell_lon - 0.0016)
self.assertAlmostEqual(cell.max_lon, cell_lon + 0.001)
self.assertAlmostEqual(cell.min_lon, cell_lon - 0.006)
self.assertEqual(cell.radius, 468)
self.assertEqual(cell.samples, 5)
def test_export_full(self, celery, session):
now = util.utcnow()
long_ago = now - timedelta(days=367)
CellShardFactory.create_batch(10, radio=Radio.gsm)
CellShardFactory(
radio=Radio.gsm,
created=long_ago,
modified=long_ago,
last_seen=long_ago.date(),
)
session.commit()
pattern = re.compile(
r"MLS-full-cell-export-\d+-\d+-\d+T000000\.csv\.gz")
mock_conn = mock.MagicMock()
mock_bucket = mock.MagicMock(name="bucket")
mock_obj = mock.MagicMock()
mock_conn.return_value.Bucket.return_value = mock_bucket
mock_bucket.Object.return_value = mock_obj
with mock.patch.object(boto3, "resource", mock_conn):
cell_export_full(_bucket="bucket")
s3_key = mock_bucket.Object.call_args[0][0]
assert pattern.search(s3_key)
tmp_file = mock_obj.upload_file.call_args[0][0]
assert pattern.search(tmp_file)
def test_cell(self):
cell = CellShardFactory()
self.session.flush()
query = self.model_query(cells=[cell])
query['radioType'] = cell.radio.name
del query['cellTowers'][0]['radioType']
query['cellTowers'][0]['signalStrength'] = -70
query['cellTowers'][0]['timingAdvance'] = 1
res = self._call(body=query)
self.check_model_response(res, cell)
self.check_stats(counter=[
('request', [self.metric_path, 'method:post', 'status:200']),
(self.metric_type + '.request', [self.metric_path, 'key:test']),
(self.metric_type + '.result', [
'key:test', 'region:none', 'fallback_allowed:false',
'accuracy:medium', 'status:hit', 'source:internal'
]),
(self.metric_type + '.source', [
'key:test', 'region:none', 'source:internal',
'accuracy:medium', 'status:hit'
]),
],
timer=[
('request', [self.metric_path, 'method:post']),
])
def test_multiple_cells(self):
now = util.utcnow()
cell = CellShardFactory(samples=100)
cell2 = CellShardFactory(radio=cell.radio, mcc=cell.mcc, mnc=cell.mnc,
lac=cell.lac, cid=cell.cid + 1,
lat=cell.lat + 1.0, lon=cell.lon + 1.0,
samples=10)
self.session.flush()
query = self.model_query(cells=[cell, cell2])
results = self.source.search(query)
self.check_model_results(
results, [cell],
lat=cell.lat + 0.3333333, lon=cell.lon + 0.3333333,
accuracy=CELL_MAX_ACCURACY)
self.assertAlmostEqual(
results.best().score, cell.score(now) + cell2.score(now), 4)
def test_cell_geoip_match(self):
cell = CellShardFactory(mcc=234)
self.session.flush()
query = self.model_query(cells=[cell])
res = self._call(body=query, ip=self.test_ip)
self.check_model_response(res, cell, region='GB')
self.check_db_calls(rw=0, ro=2)
def test_cell_wrong_cid(self, geoip_db, http_session, session, source):
cell = CellShardFactory()
session.flush()
cell.cid += 1
query = self.model_query(geoip_db, http_session, session, cells=[cell])
results = source.search(query)
self.check_model_results(results, None)
def test_cell_ambiguous(self, app, session):
# cell with ambiguous mcc to region mapping
cell = CellShardFactory(mcc=234)
session.flush()
query = self.model_query(cells=[cell])
res = self._call(app, body=query)
self.check_model_response(res, cell, region="GB")
def test_cell_wrong_cid(self):
cell = CellShardFactory()
self.session.flush()
cell.cid += 1
query = self.model_query(cells=[cell])
result = self.source.search(query)
self.check_model_result(result, None)
def test_cell_over_geoip(self, app, session):
# UK GeoIP with single DE cell
cell = CellShardFactory(mcc=262)
session.flush()
query = self.model_query(cells=[cell])
res = self._call(app, body=query, ip=self.test_ip)
self.check_model_response(res, cell, region="DE")
def test_inconsistent_cell_radio(self):
cell = CellShardFactory(radio=Radio.wcdma, radius=15000)
cell2 = CellShardFactory(radio=Radio.gsm,
radius=35000,
lat=cell.lat + 0.0002,
lon=cell.lon)
self.session.flush()
query = self.model_query(cells=[cell, cell2])
query['radioType'] = Radio.lte.name
query['cellTowers'][0]['radio'] = 'wcdma'
query['cellTowers'][1]['radio'] = cell2.radio.name
del query['cellTowers'][0]['radioType']
del query['cellTowers'][1]['radioType']
res = self._call(body=query)
self.check_model_response(res, cell)
def test_cell(self, app, session):
# cell with unique mcc to region mapping
cell = CellShardFactory(mcc=235)
session.flush()
query = self.model_query(cells=[cell])
res = self._call(app, body=query)
self.check_model_response(res, cell, region='GB')
def test_cell_ambiguous(self):
# cell with ambiguous mcc to region mapping
cell = CellShardFactory(mcc=234)
self.session.flush()
query = self.model_query(cells=[cell])
res = self._call(body=query)
self.check_model_response(res, cell, region='GB')
self.check_db_calls(rw=0, ro=2)
def test_cell_geoip_mismatch(self, app, session):
# UK GeoIP with ambiguous US mcc
uk_cell = CellShardFactory.build(mcc=234)
us_cell = CellShardFactory(mcc=310)
session.flush()
query = self.model_query(cells=[us_cell])
res = self._call(app, body=query, ip=self.test_ip)
self.check_model_response(res, uk_cell, region="GB", fallback="ipf")
def test_cell(self, geoip_db, http_session, session, source):
now = util.utcnow()
cell = CellShardFactory(samples=10)
session.flush()
query = self.model_query(geoip_db, http_session, session, cells=[cell])
results = source.search(query)
self.check_model_results(results, [cell])
assert results.best().score == station_score(cell, now)
def test_cell(self):
now = util.utcnow()
cell = CellShardFactory(samples=10)
self.session.flush()
query = self.model_query(cells=[cell])
results = self.source.search(query)
self.check_model_results(results, [cell])
self.assertAlmostEqual(results.best().score, cell.score(now), 4)
def test_cell_over_geoip(self):
# UK GeoIP with single DE cell
cell = CellShardFactory(mcc=262)
self.session.flush()
query = self.model_query(cells=[cell])
res = self._call(body=query, ip=self.test_ip)
self.check_model_response(res, cell, region='DE')
self.check_db_calls(rw=0, ro=1)
def test_store_sample(self, app, data_queues, session):
api_key = ApiKeyFactory(store_sample_locate=0)
cell = CellShardFactory()
session.flush()
query = self.model_query(cells=[cell])
res = self._call(app, body=query, api_key=api_key.valid_key, status=200)
self.check_model_response(res, cell)
self.check_queue(data_queues, 0)
