def test_update(self):
area = CellAreaFactory(radio=Radio.gsm, num_cells=1)
area.region = None
CellAreaFactory(radio=Radio.gsm, region='DE', num_cells=1)
CellAreaFactory(radio=Radio.gsm, region='DE', num_cells=2)
CellAreaFactory(radio=Radio.gsm, region='CA', num_cells=2)
CellAreaFactory(radio=Radio.wcdma, region='DE', num_cells=3)
CellAreaFactory(radio=Radio.lte, region='CA', num_cells=4)
WifiShardFactory.create_batch(5, region='DE')
WifiShardFactory.create_batch(6, region='US')
wifi = WifiShardFactory()
wifi.region = None
self.session.add(RegionStat(region='US', wifi=2))
self.session.add(RegionStat(region='TW', wifi=1))
self.session.flush()
update_statregion.delay().get()
stats = self.session.query(RegionStat).all()
self.assertEqual(len(stats), 3)
for stat in stats:
values = (stat.gsm, stat.wcdma, stat.lte, stat.wifi)
if stat.region == 'DE':
self.assertEqual(values, (3, 3, 0, 5))
elif stat.region == 'CA':
self.assertEqual(values, (2, 0, 4, 0))
elif stat.region == 'US':
self.assertEqual(values, (0, 0, 0, 6))
def test_update(self, celery, session):
area = CellAreaFactory(radio=Radio.gsm, num_cells=1)
area.region = None
BlueShardFactory.create_batch(2, region='CA')
BlueShardFactory.create_batch(3, region='GB')
CellAreaFactory(radio=Radio.gsm, region='DE', num_cells=1)
CellAreaFactory(radio=Radio.gsm, region='DE', num_cells=2)
CellAreaFactory(radio=Radio.gsm, region='CA', num_cells=2)
CellAreaFactory(radio=Radio.wcdma, region='DE', num_cells=3)
CellAreaFactory(radio=Radio.lte, region='CA', num_cells=4)
WifiShardFactory.create_batch(5, region='DE')
WifiShardFactory.create_batch(6, region='US')
wifi = WifiShardFactory()
wifi.region = None
session.add(RegionStat(region='US', blue=1, wifi=2))
session.add(RegionStat(region='TW', wifi=1))
session.flush()
update_statregion.delay().get()
stats = session.query(RegionStat).all()
assert len(stats) == 4
for stat in stats:
values = (stat.gsm, stat.wcdma, stat.lte, stat.blue, stat.wifi)
if stat.region == 'DE':
assert values == (3, 3, 0, 0, 5)
elif stat.region == 'CA':
assert values == (2, 0, 4, 2, 0)
elif stat.region == 'GB':
assert values == (0, 0, 0, 3, 0)
elif stat.region == 'US':
assert values == (0, 0, 0, 0, 6)
def test_update(self):
area = CellAreaFactory(radio=Radio.gsm, num_cells=1)
area.region = None
CellAreaFactory(radio=Radio.gsm, region='DE', num_cells=1)
CellAreaFactory(radio=Radio.gsm, region='DE', num_cells=2)
CellAreaFactory(radio=Radio.gsm, region='CA', num_cells=2)
CellAreaFactory(radio=Radio.wcdma, region='DE', num_cells=3)
CellAreaFactory(radio=Radio.lte, region='CA', num_cells=4)
WifiShardFactory.create_batch(5, region='DE')
WifiShardFactory.create_batch(6, region='US')
wifi = WifiShardFactory()
wifi.region = None
self.session.add(RegionStat(region='US', wifi=2))
self.session.add(RegionStat(region='TW', wifi=1))
self.session.flush()
update_statregion.delay().get()
stats = self.session.query(RegionStat).all()
self.assertEqual(len(stats), 3)
for stat in stats:
values = (stat.gsm, stat.wcdma, stat.lte, stat.wifi)
if stat.region == 'DE':
self.assertEqual(values, (3, 3, 0, 5))
elif stat.region == 'CA':
self.assertEqual(values, (2, 0, 4, 0))
elif stat.region == 'US':
self.assertEqual(values, (0, 0, 0, 6))
def test_update(self, celery, session):
area = CellAreaFactory(radio=Radio.gsm, num_cells=1)
area.region = None
BlueShardFactory.create_batch(2, region='CA')
BlueShardFactory.create_batch(3, region='GB')
CellAreaFactory(radio=Radio.gsm, region='DE', num_cells=1)
CellAreaFactory(radio=Radio.gsm, region='DE', num_cells=2)
CellAreaFactory(radio=Radio.gsm, region='CA', num_cells=2)
CellAreaFactory(radio=Radio.wcdma, region='DE', num_cells=3)
CellAreaFactory(radio=Radio.lte, region='CA', num_cells=4)
WifiShardFactory.create_batch(5, region='DE')
WifiShardFactory.create_batch(6, region='US')
wifi = WifiShardFactory()
wifi.region = None
session.add(RegionStat(region='US', blue=1, wifi=2))
session.add(RegionStat(region='TW', wifi=1))
session.flush()
update_statregion.delay().get()
stats = session.query(RegionStat).all()
assert len(stats) == 4
for stat in stats:
values = (stat.gsm, stat.wcdma, stat.lte, stat.blue, stat.wifi)
if stat.region == 'DE':
assert values == (3, 3, 0, 0, 5)
elif stat.region == 'CA':
assert values == (2, 0, 4, 2, 0)
elif stat.region == 'GB':
assert values == (0, 0, 0, 3, 0)
elif stat.region == 'US':
assert values == (0, 0, 0, 0, 6)
def test_larger_cluster_over_signal(self):
wifi = WifiShardFactory()
wifis = WifiShardFactory.create_batch(
3, lat=wifi.lat, lon=wifi.lon)
wifis2 = WifiShardFactory.create_batch(
3, lat=wifi.lat + 1.0, lon=wifi.lon)
self.session.flush()
query = self.model_query(wifis=[wifi] + wifis + wifis2)
for entry in query.wifi[:-3]:
entry.signal = -80
for entry in query.wifi[-3:]:
entry.signal = -70
result = self.source.search(query)
self.check_model_result(result, wifi)
def test_larger_cluster_over_signal(self):
wifi = WifiShardFactory()
wifis = WifiShardFactory.create_batch(3, lat=wifi.lat, lon=wifi.lon)
wifis2 = WifiShardFactory.create_batch(3,
lat=wifi.lat + 1.0,
lon=wifi.lon)
self.session.flush()
query = self.model_query(wifis=[wifi] + wifis + wifis2)
for entry in query.wifi[:-3]:
entry.signal = -80
for entry in query.wifi[-3:]:
entry.signal = -70
result = self.source.search(query)
self.check_model_result(result, wifi)
def test_few_candidates(self):
wifis = WifiShardFactory.create_batch(2)
self.session.flush()
query = self.model_query(wifis=[wifis[0]])
results = self.source.search(query)
self.check_model_results(results, None)
def test_few_candidates(self):
wifis = WifiShardFactory.create_batch(2)
self.session.flush()
query = self.model_query(wifis=[wifis[0]])
result = self.source.search(query)
self.check_model_result(result, None)
def test_few_matches(self):
wifis = WifiShardFactory.create_batch(3)
wifis[0].lat = None
self.session.flush()
query = self.model_query(wifis=wifis[:2])
result = self.source.search(query)
self.check_model_result(result, None)
def test_few_matches(self):
wifis = WifiShardFactory.create_batch(3)
wifis[0].lat = None
self.session.flush()
query = self.model_query(wifis=wifis[:2])
results = self.source.search(query)
self.check_model_results(results, None)
def test_few_candidates(self, geoip_db, http_session,
session, source, stats):
wifis = WifiShardFactory.create_batch(2)
session.flush()
query = self.model_query(
geoip_db, http_session, session, stats,
wifis=[wifis[0]])
results = source.search(query)
self.check_model_results(results, None)
def test_ignore_outlier(self):
wifi = WifiShardFactory()
wifis = WifiShardFactory.create_batch(3, lat=wifi.lat, lon=wifi.lon)
wifis[0].lat = wifi.lat + 0.0001
wifis[1].lat = wifi.lat + 0.0002
wifis[2].lat = wifi.lat + 1.0
self.session.flush()
query = self.model_query(wifis=[wifi] + wifis)
result = self.source.search(query)
self.check_model_result(result, wifi, lat=wifi.lat + 0.0001)
def test_ignore_outlier(self):
wifi = WifiShardFactory()
wifis = WifiShardFactory.create_batch(3, lat=wifi.lat, lon=wifi.lon)
wifis[0].lat = wifi.lat + 0.0001
wifis[1].lat = wifi.lat + 0.0002
wifis[2].lat = wifi.lat + 1.0
self.session.flush()
query = self.model_query(wifis=[wifi] + wifis)
results = self.source.search(query)
self.check_model_results(results, [wifi], lat=wifi.lat + 0.0001)
def test_few_matches(self, geoip_db, http_session, session, source, stats):
wifis = WifiShardFactory.create_batch(3)
wifis[0].lat = None
session.flush()
query = self.model_query(geoip_db,
http_session,
session,
stats,
wifis=wifis[:2])
results = source.search(query)
self.check_model_results(results, None)
def test_ignore_outlier(self, geoip_db, http_session, session, source):
wifi = WifiShardFactory()
wifis = WifiShardFactory.create_batch(3, lat=wifi.lat, lon=wifi.lon, radius=5)
wifis[0].lat = wifi.lat + 0.00001
wifis[1].lat = wifi.lat + 0.00002
wifis[2].lat = wifi.lat + 1.0
session.flush()
query = self.model_query(geoip_db, http_session, session, wifis=[wifi] + wifis)
results = source.search(query)
self.check_model_results(results, [wifi], lat=wifi.lat + 0.00001)
assert round(results.best().score, 4) == 0.15
def test_ignore_outlier(self):
wifi = WifiShardFactory()
wifis = WifiShardFactory.create_batch(3,
lat=wifi.lat,
lon=wifi.lon,
radius=5)
wifis[0].lat = wifi.lat + 0.00001
wifis[1].lat = wifi.lat + 0.00002
wifis[2].lat = wifi.lat + 1.0
self.session.flush()
query = self.model_query(wifis=[wifi] + wifis)
results = self.source.search(query)
self.check_model_results(results, [wifi], lat=wifi.lat + 0.00001)
self.assertAlmostEqual(results.best().score, 0.15)
def test_ignore_outlier(self, geoip_db, http_session,
session, source, stats):
wifi = WifiShardFactory()
wifis = WifiShardFactory.create_batch(
3, lat=wifi.lat, lon=wifi.lon, radius=5)
wifis[0].lat = wifi.lat + 0.00001
wifis[1].lat = wifi.lat + 0.00002
wifis[2].lat = wifi.lat + 1.0
session.flush()
query = self.model_query(
geoip_db, http_session, session, stats,
wifis=[wifi] + wifis)
results = source.search(query)
self.check_model_results(results, [wifi], lat=wifi.lat + 0.00001)
assert round(results.best().score, 4) == 0.15
def test_wifi(self, sync_session):
wifis = WifiShardFactory.create_batch(5)
sync_session.flush()
self._export(sync_session, "wifi", self._mac_keys(wifis))
def test_blocklist_moving_wifis(self):
now = util.utcnow()
obs = []
obs_factory = WifiObservationFactory
moving = set()
wifis = WifiShardFactory.create_batch(7)
wifis.append(WifiShardFactory.build())
# a wifi without an entry and disagreeing observations
wifi = wifis[-1]
obs.extend([
obs_factory(lat=wifi.lat, lon=wifi.lon, mac=wifi.mac),
obs_factory(lat=wifi.lat + 2.0, lon=wifi.lon, mac=wifi.mac),
])
moving.add(wifi.mac)
# a wifi with an entry but no prior position
wifi = wifis[0]
obs.extend([
obs_factory(lat=wifi.lat + 0.001,
lon=wifi.lon + 0.001, mac=wifi.mac),
obs_factory(lat=wifi.lat + 0.002,
lon=wifi.lon + 0.005, mac=wifi.mac),
obs_factory(lat=wifi.lat + 0.003,
lon=wifi.lon + 0.009, mac=wifi.mac),
])
wifi.lat = None
wifi.lon = None
wifi.samples = None
wifi.weight = None
# a wifi with a prior known position
wifi = wifis[1]
wifi.samples = 1
wifi.weight = 1.0
wifi.lat += 1.0
wifi.lon += 1.0
obs.extend([
obs_factory(lat=wifi.lat + 0.01,
lon=wifi.lon, mac=wifi.mac),
obs_factory(lat=wifi.lat + 0.07,
lon=wifi.lon, mac=wifi.mac),
])
moving.add(wifi.mac)
# a wifi with a very different prior position
wifi = wifis[2]
wifi.samples = 1
wifi.weight = 1.0
obs.extend([
obs_factory(lat=wifi.lat + 2.0,
lon=wifi.lon, mac=wifi.mac),
obs_factory(lat=wifi.lat + 2.002,
lon=wifi.lon, mac=wifi.mac),
])
moving.add(wifi.mac)
# an already blocked wifi
wifi = wifis[3]
wifi.block_last = now.date()
wifi.block_count = 1
obs.extend([
obs_factory(lat=wifi.lat,
lon=wifi.lon, mac=wifi.mac),
obs_factory(lat=wifi.lat + 0.1,
lon=wifi.lon, mac=wifi.mac),
])
moving.add(wifi.mac)
# a permanently blocked wifi
wifi = wifis[4]
wifi_lat, wifi_lon = (wifi.lat, wifi.lon)
wifi.block_last = now.date() - 2 * TEMPORARY_BLOCKLIST_DURATION
wifi.block_count = PERMANENT_BLOCKLIST_THRESHOLD
for col in ('lat', 'lon', 'max_lat', 'min_lat', 'max_lon', 'min_lon'):
setattr(wifi, col, None)
obs.extend([
obs_factory(lat=wifi_lat, lon=wifi_lon, mac=wifi.mac),
])
moving.add(wifi.mac)
# a no longer blocked wifi
wifi = wifis[5]
wifi_lat, wifi_lon = (wifi.lat, wifi.lon)
wifi.block_last = now.date() - 2 * TEMPORARY_BLOCKLIST_DURATION
wifi.block_count = 2
for col in ('lat', 'lon', 'max_lat', 'min_lat', 'max_lon', 'min_lon'):
setattr(wifi, col, None)
obs.extend([
obs_factory(lat=wifi_lat, lon=wifi_lon, mac=wifi.mac),
])
# a no longer blocked wifi with disagreeing observations
wifi = wifis[6]
wifi_lat, wifi_lon = (wifi.lat, wifi.lon)
wifi.block_last = now.date() - 2 * TEMPORARY_BLOCKLIST_DURATION
wifi.block_count = 2
for col in ('lat', 'lon', 'max_lat', 'min_lat', 'max_lon', 'min_lon'):
setattr(wifi, col, None)
obs.extend([
obs_factory(lat=wifi_lat, lon=wifi_lon, mac=wifi.mac),
obs_factory(lat=wifi_lat + 2.0, lon=wifi_lon, mac=wifi.mac),
])
moving.add(wifi.mac)
self.session.commit()
self._queue_and_update_wifi(obs)
shards = set()
for mac in moving:
shards.add(WifiShard.shard_model(mac))
blocks = []
for shard in shards:
for row in self.session.query(shard).all():
if row.blocked():
blocks.append(row)
self.assertEqual(set([b.mac for b in blocks]), moving)
def test_blocklist_moving_wifis(self):
now = util.utcnow()
obs = []
obs_factory = WifiObservationFactory
moving = set()
wifis = WifiShardFactory.create_batch(7)
wifis.append(WifiShardFactory.build())
# a wifi without an entry and disagreeing observations
wifi = wifis[-1]
obs.extend([
obs_factory(lat=wifi.lat, lon=wifi.lon, key=wifi.mac),
obs_factory(lat=wifi.lat + 2.0, lon=wifi.lon, key=wifi.mac),
])
moving.add(wifi.mac)
# a wifi with an entry but no prior position
wifi = wifis[0]
obs.extend([
obs_factory(lat=wifi.lat + 0.001,
lon=wifi.lon + 0.001,
key=wifi.mac),
obs_factory(lat=wifi.lat + 0.002,
lon=wifi.lon + 0.005,
key=wifi.mac),
obs_factory(lat=wifi.lat + 0.003,
lon=wifi.lon + 0.009,
key=wifi.mac),
])
wifi.lat = None
wifi.lon = None
wifi.samples = 0
# a wifi with a prior known position
wifi = wifis[1]
wifi.samples = 1
wifi.lat += 1.0
wifi.lon += 1.0
obs.extend([
obs_factory(lat=wifi.lat + 0.01, lon=wifi.lon, key=wifi.mac),
obs_factory(lat=wifi.lat + 0.07, lon=wifi.lon, key=wifi.mac),
])
moving.add(wifi.mac)
# a wifi with a very different prior position
wifi = wifis[2]
wifi.samples = 1
obs.extend([
obs_factory(lat=wifi.lat + 2.0, lon=wifi.lon, key=wifi.mac),
obs_factory(lat=wifi.lat + 2.002, lon=wifi.lon, key=wifi.mac),
])
moving.add(wifi.mac)
# an already blocked wifi
wifi = wifis[3]
wifi.block_last = now.date()
wifi.block_count = 1
obs.extend([
obs_factory(lat=wifi.lat, lon=wifi.lon, key=wifi.mac),
obs_factory(lat=wifi.lat + 0.1, lon=wifi.lon, key=wifi.mac),
])
moving.add(wifi.mac)
# a permanently blocked wifi
wifi = wifis[4]
wifi_lat, wifi_lon = (wifi.lat, wifi.lon)
wifi.block_last = now.date() - 2 * TEMPORARY_BLOCKLIST_DURATION
wifi.block_count = PERMANENT_BLOCKLIST_THRESHOLD
for col in ('lat', 'lon', 'max_lat', 'min_lat', 'max_lon', 'min_lon'):
setattr(wifi, col, None)
obs.extend([
obs_factory(lat=wifi_lat, lon=wifi_lon, key=wifi.mac),
])
moving.add(wifi.mac)
# a no longer blocked wifi
wifi = wifis[5]
wifi_lat, wifi_lon = (wifi.lat, wifi.lon)
wifi.block_last = now.date() - 2 * TEMPORARY_BLOCKLIST_DURATION
wifi.block_count = 2
for col in ('lat', 'lon', 'max_lat', 'min_lat', 'max_lon', 'min_lon'):
setattr(wifi, col, None)
obs.extend([
obs_factory(lat=wifi_lat, lon=wifi_lon, key=wifi.mac),
])
# a no longer blocked wifi with disagreeing observations
wifi = wifis[6]
wifi_lat, wifi_lon = (wifi.lat, wifi.lon)
wifi.block_last = now.date() - 2 * TEMPORARY_BLOCKLIST_DURATION
wifi.block_count = 2
for col in ('lat', 'lon', 'max_lat', 'min_lat', 'max_lon', 'min_lon'):
setattr(wifi, col, None)
obs.extend([
obs_factory(lat=wifi_lat, lon=wifi_lon, key=wifi.mac),
obs_factory(lat=wifi_lat + 2.0, lon=wifi_lon, key=wifi.mac),
])
moving.add(wifi.mac)
self.session.commit()
self._queue_and_update(obs)
shards = set()
for mac in moving:
shards.add(WifiShard.shard_model(mac))
blocks = []
for shard in shards:
for row in self.session.query(shard).all():
if row.blocked():
blocks.append(row)
self.assertEqual(set([b.mac for b in blocks]), moving)
def test_blocklist_moving_wifis(self):
now = util.utcnow()
obs = []
obs_factory = WifiObservationFactory
moving = set()
wifis = WifiShardFactory.create_batch(8)
wifis.append(WifiShardFactory.build())
# a wifi without an entry and disagreeing observations
wifi = wifis[-1]
obs.extend([
obs_factory(lat=wifi.lat, lon=wifi.lon, key=wifi.mac),
obs_factory(lat=wifi.lat + 2.0, lon=wifi.lon, key=wifi.mac),
])
moving.add(wifi.mac)
# a wifi with an entry but no prior position
wifi = wifis[0]
obs.extend([
obs_factory(lat=wifi.lat + 0.001,
lon=wifi.lon + 0.001, key=wifi.mac),
obs_factory(lat=wifi.lat + 0.002,
lon=wifi.lon + 0.005, key=wifi.mac),
obs_factory(lat=wifi.lat + 0.003,
lon=wifi.lon + 0.009, key=wifi.mac),
])
wifi.lat = None
wifi.lon = None
wifi.samples = 0
# a wifi with a prior known position
wifi = wifis[1]
wifi.samples = 1
wifi.lat += 1.0
wifi.lon += 1.0
obs.extend([
obs_factory(lat=wifi.lat + 0.01,
lon=wifi.lon, key=wifi.mac),
obs_factory(lat=wifi.lat + 0.07,
lon=wifi.lon, key=wifi.mac),
])
moving.add(wifi.mac)
# a wifi with a very different prior position
wifi = wifis[2]
wifi.samples = 1
obs.extend([
obs_factory(lat=wifi.lat + 2.0,
lon=wifi.lon, key=wifi.mac),
obs_factory(lat=wifi.lat + 2.002,
lon=wifi.lon, key=wifi.mac),
])
moving.add(wifi.mac)
# another wifi with a prior known position (and negative lat)
wifi = wifis[3]
wifi.samples = 1
wifi.lat *= -1.0
obs.extend([
obs_factory(lat=wifi.lat - 0.1,
lon=wifi.lon, key=wifi.mac),
obs_factory(lat=wifi.lat - 0.16,
lon=wifi.lon, key=wifi.mac),
])
moving.add(wifi.mac)
# an already blocked wifi
wifi = wifis[4]
wifi.block_last = now.date()
wifi.block_count = 1
obs.extend([
obs_factory(lat=wifi.lat,
lon=wifi.lon, key=wifi.mac),
obs_factory(lat=wifi.lat + 0.1,
lon=wifi.lon, key=wifi.mac),
])
moving.add(wifi.mac)
# a permanently blocked wifi
wifi = wifis[5]
wifi_lat, wifi_lon = (wifi.lat, wifi.lon)
wifi.block_last = now.date() - 2 * TEMPORARY_BLOCKLIST_DURATION
wifi.block_count = PERMANENT_BLOCKLIST_THRESHOLD
for col in ('lat', 'lon', 'max_lat', 'min_lat', 'max_lon', 'min_lon'):
setattr(wifi, col, None)
obs.extend([
obs_factory(lat=wifi_lat, lon=wifi_lon, key=wifi.mac),
])
moving.add(wifi.mac)
# a no longer blocked wifi
wifi = wifis[6]
wifi_lat, wifi_lon = (wifi.lat, wifi.lon)
wifi.block_last = now.date() - 2 * TEMPORARY_BLOCKLIST_DURATION
wifi.block_count = 2
for col in ('lat', 'lon', 'max_lat', 'min_lat', 'max_lon', 'min_lon'):
setattr(wifi, col, None)
obs.extend([
obs_factory(lat=wifi_lat, lon=wifi_lon, key=wifi.mac),
])
# a no longer blocked wifi with disagreeing observations
wifi = wifis[7]
wifi_lat, wifi_lon = (wifi.lat, wifi.lon)
wifi.block_last = now.date() - 2 * TEMPORARY_BLOCKLIST_DURATION
wifi.block_count = 2
for col in ('lat', 'lon', 'max_lat', 'min_lat', 'max_lon', 'min_lon'):
setattr(wifi, col, None)
obs.extend([
obs_factory(lat=wifi_lat, lon=wifi_lon, key=wifi.mac),
obs_factory(lat=wifi_lat + 2.0, lon=wifi_lon, key=wifi.mac),
])
moving.add(wifi.mac)
self.data_queue.enqueue(obs)
self.session.commit()
update_wifi.delay().get()
shards = set()
for mac in moving:
shards.add(WifiShard.shard_model(mac))
blocks = []
for shard in shards:
for row in self.session.query(shard).all():
if row.blocked():
blocks.append(row)
self.assertEqual(set([b.mac for b in blocks]), moving)
self.check_stats(counter=[
('data.observation.drop', 1, 3,
['type:wifi', 'reason:blocklisted']),
('data.station.blocklist', 1, 5,
['type:wifi', 'action:add', 'reason:moving']),
])
