def test_update(self): area = CellAreaFactory(radio=Radio.gsm, num_cells=1) area.region = None CellAreaFactory(radio=Radio.gsm, region='DE', num_cells=1) CellAreaFactory(radio=Radio.gsm, region='DE', num_cells=2) CellAreaFactory(radio=Radio.gsm, region='CA', num_cells=2) CellAreaFactory(radio=Radio.wcdma, region='DE', num_cells=3) CellAreaFactory(radio=Radio.lte, region='CA', num_cells=4) WifiShardFactory.create_batch(5, region='DE') WifiShardFactory.create_batch(6, region='US') wifi = WifiShardFactory() wifi.region = None self.session.add(RegionStat(region='US', wifi=2)) self.session.add(RegionStat(region='TW', wifi=1)) self.session.flush() update_statregion.delay().get() stats = self.session.query(RegionStat).all() self.assertEqual(len(stats), 3) for stat in stats: values = (stat.gsm, stat.wcdma, stat.lte, stat.wifi) if stat.region == 'DE': self.assertEqual(values, (3, 3, 0, 5)) elif stat.region == 'CA': self.assertEqual(values, (2, 0, 4, 0)) elif stat.region == 'US': self.assertEqual(values, (0, 0, 0, 6))
def test_update(self, celery, session): area = CellAreaFactory(radio=Radio.gsm, num_cells=1) area.region = None BlueShardFactory.create_batch(2, region='CA') BlueShardFactory.create_batch(3, region='GB') CellAreaFactory(radio=Radio.gsm, region='DE', num_cells=1) CellAreaFactory(radio=Radio.gsm, region='DE', num_cells=2) CellAreaFactory(radio=Radio.gsm, region='CA', num_cells=2) CellAreaFactory(radio=Radio.wcdma, region='DE', num_cells=3) CellAreaFactory(radio=Radio.lte, region='CA', num_cells=4) WifiShardFactory.create_batch(5, region='DE') WifiShardFactory.create_batch(6, region='US') wifi = WifiShardFactory() wifi.region = None session.add(RegionStat(region='US', blue=1, wifi=2)) session.add(RegionStat(region='TW', wifi=1)) session.flush() update_statregion.delay().get() stats = session.query(RegionStat).all() assert len(stats) == 4 for stat in stats: values = (stat.gsm, stat.wcdma, stat.lte, stat.blue, stat.wifi) if stat.region == 'DE': assert values == (3, 3, 0, 0, 5) elif stat.region == 'CA': assert values == (2, 0, 4, 2, 0) elif stat.region == 'GB': assert values == (0, 0, 0, 3, 0) elif stat.region == 'US': assert values == (0, 0, 0, 0, 6)
def test_update(self): area = CellAreaFactory(radio=Radio.gsm, num_cells=1) area.region = None CellAreaFactory(radio=Radio.gsm, region='DE', num_cells=1) CellAreaFactory(radio=Radio.gsm, region='DE', num_cells=2) CellAreaFactory(radio=Radio.gsm, region='CA', num_cells=2) CellAreaFactory(radio=Radio.wcdma, region='DE', num_cells=3) CellAreaFactory(radio=Radio.lte, region='CA', num_cells=4) WifiShardFactory.create_batch(5, region='DE') WifiShardFactory.create_batch(6, region='US') wifi = WifiShardFactory() wifi.region = None self.session.add(RegionStat(region='US', wifi=2)) self.session.add(RegionStat(region='TW', wifi=1)) self.session.flush() update_statregion.delay().get() stats = self.session.query(RegionStat).all() self.assertEqual(len(stats), 3) for stat in stats: values = (stat.gsm, stat.wcdma, stat.lte, stat.wifi) if stat.region == 'DE': self.assertEqual(values, (3, 3, 0, 5)) elif stat.region == 'CA': self.assertEqual(values, (2, 0, 4, 0)) elif stat.region == 'US': self.assertEqual(values, (0, 0, 0, 6))
def test_update(self, celery, session): area = CellAreaFactory(radio=Radio.gsm, num_cells=1) area.region = None BlueShardFactory.create_batch(2, region='CA') BlueShardFactory.create_batch(3, region='GB') CellAreaFactory(radio=Radio.gsm, region='DE', num_cells=1) CellAreaFactory(radio=Radio.gsm, region='DE', num_cells=2) CellAreaFactory(radio=Radio.gsm, region='CA', num_cells=2) CellAreaFactory(radio=Radio.wcdma, region='DE', num_cells=3) CellAreaFactory(radio=Radio.lte, region='CA', num_cells=4) WifiShardFactory.create_batch(5, region='DE') WifiShardFactory.create_batch(6, region='US') wifi = WifiShardFactory() wifi.region = None session.add(RegionStat(region='US', blue=1, wifi=2)) session.add(RegionStat(region='TW', wifi=1)) session.flush() update_statregion.delay().get() stats = session.query(RegionStat).all() assert len(stats) == 4 for stat in stats: values = (stat.gsm, stat.wcdma, stat.lte, stat.blue, stat.wifi) if stat.region == 'DE': assert values == (3, 3, 0, 0, 5) elif stat.region == 'CA': assert values == (2, 0, 4, 2, 0) elif stat.region == 'GB': assert values == (0, 0, 0, 3, 0) elif stat.region == 'US': assert values == (0, 0, 0, 0, 6)
def test_larger_cluster_over_signal(self): wifi = WifiShardFactory() wifis = WifiShardFactory.create_batch( 3, lat=wifi.lat, lon=wifi.lon) wifis2 = WifiShardFactory.create_batch( 3, lat=wifi.lat + 1.0, lon=wifi.lon) self.session.flush() query = self.model_query(wifis=[wifi] + wifis + wifis2) for entry in query.wifi[:-3]: entry.signal = -80 for entry in query.wifi[-3:]: entry.signal = -70 result = self.source.search(query) self.check_model_result(result, wifi)
def test_larger_cluster_over_signal(self): wifi = WifiShardFactory() wifis = WifiShardFactory.create_batch(3, lat=wifi.lat, lon=wifi.lon) wifis2 = WifiShardFactory.create_batch(3, lat=wifi.lat + 1.0, lon=wifi.lon) self.session.flush() query = self.model_query(wifis=[wifi] + wifis + wifis2) for entry in query.wifi[:-3]: entry.signal = -80 for entry in query.wifi[-3:]: entry.signal = -70 result = self.source.search(query) self.check_model_result(result, wifi)
def test_few_candidates(self): wifis = WifiShardFactory.create_batch(2) self.session.flush() query = self.model_query(wifis=[wifis[0]]) results = self.source.search(query) self.check_model_results(results, None)
def test_few_candidates(self): wifis = WifiShardFactory.create_batch(2) self.session.flush() query = self.model_query(wifis=[wifis[0]]) result = self.source.search(query) self.check_model_result(result, None)
def test_few_matches(self): wifis = WifiShardFactory.create_batch(3) wifis[0].lat = None self.session.flush() query = self.model_query(wifis=wifis[:2]) result = self.source.search(query) self.check_model_result(result, None)
def test_few_matches(self): wifis = WifiShardFactory.create_batch(3) wifis[0].lat = None self.session.flush() query = self.model_query(wifis=wifis[:2]) results = self.source.search(query) self.check_model_results(results, None)
def test_few_candidates(self, geoip_db, http_session, session, source, stats): wifis = WifiShardFactory.create_batch(2) session.flush() query = self.model_query( geoip_db, http_session, session, stats, wifis=[wifis[0]]) results = source.search(query) self.check_model_results(results, None)
def test_ignore_outlier(self): wifi = WifiShardFactory() wifis = WifiShardFactory.create_batch(3, lat=wifi.lat, lon=wifi.lon) wifis[0].lat = wifi.lat + 0.0001 wifis[1].lat = wifi.lat + 0.0002 wifis[2].lat = wifi.lat + 1.0 self.session.flush() query = self.model_query(wifis=[wifi] + wifis) result = self.source.search(query) self.check_model_result(result, wifi, lat=wifi.lat + 0.0001)
def test_ignore_outlier(self): wifi = WifiShardFactory() wifis = WifiShardFactory.create_batch(3, lat=wifi.lat, lon=wifi.lon) wifis[0].lat = wifi.lat + 0.0001 wifis[1].lat = wifi.lat + 0.0002 wifis[2].lat = wifi.lat + 1.0 self.session.flush() query = self.model_query(wifis=[wifi] + wifis) results = self.source.search(query) self.check_model_results(results, [wifi], lat=wifi.lat + 0.0001)
def test_few_matches(self, geoip_db, http_session, session, source, stats): wifis = WifiShardFactory.create_batch(3) wifis[0].lat = None session.flush() query = self.model_query(geoip_db, http_session, session, stats, wifis=wifis[:2]) results = source.search(query) self.check_model_results(results, None)
def test_ignore_outlier(self, geoip_db, http_session, session, source): wifi = WifiShardFactory() wifis = WifiShardFactory.create_batch(3, lat=wifi.lat, lon=wifi.lon, radius=5) wifis[0].lat = wifi.lat + 0.00001 wifis[1].lat = wifi.lat + 0.00002 wifis[2].lat = wifi.lat + 1.0 session.flush() query = self.model_query(geoip_db, http_session, session, wifis=[wifi] + wifis) results = source.search(query) self.check_model_results(results, [wifi], lat=wifi.lat + 0.00001) assert round(results.best().score, 4) == 0.15
def test_ignore_outlier(self): wifi = WifiShardFactory() wifis = WifiShardFactory.create_batch(3, lat=wifi.lat, lon=wifi.lon, radius=5) wifis[0].lat = wifi.lat + 0.00001 wifis[1].lat = wifi.lat + 0.00002 wifis[2].lat = wifi.lat + 1.0 self.session.flush() query = self.model_query(wifis=[wifi] + wifis) results = self.source.search(query) self.check_model_results(results, [wifi], lat=wifi.lat + 0.00001) self.assertAlmostEqual(results.best().score, 0.15)
def test_ignore_outlier(self, geoip_db, http_session, session, source, stats): wifi = WifiShardFactory() wifis = WifiShardFactory.create_batch( 3, lat=wifi.lat, lon=wifi.lon, radius=5) wifis[0].lat = wifi.lat + 0.00001 wifis[1].lat = wifi.lat + 0.00002 wifis[2].lat = wifi.lat + 1.0 session.flush() query = self.model_query( geoip_db, http_session, session, stats, wifis=[wifi] + wifis) results = source.search(query) self.check_model_results(results, [wifi], lat=wifi.lat + 0.00001) assert round(results.best().score, 4) == 0.15
def test_wifi(self, sync_session): wifis = WifiShardFactory.create_batch(5) sync_session.flush() self._export(sync_session, "wifi", self._mac_keys(wifis))
def test_blocklist_moving_wifis(self): now = util.utcnow() obs = [] obs_factory = WifiObservationFactory moving = set() wifis = WifiShardFactory.create_batch(7) wifis.append(WifiShardFactory.build()) # a wifi without an entry and disagreeing observations wifi = wifis[-1] obs.extend([ obs_factory(lat=wifi.lat, lon=wifi.lon, mac=wifi.mac), obs_factory(lat=wifi.lat + 2.0, lon=wifi.lon, mac=wifi.mac), ]) moving.add(wifi.mac) # a wifi with an entry but no prior position wifi = wifis[0] obs.extend([ obs_factory(lat=wifi.lat + 0.001, lon=wifi.lon + 0.001, mac=wifi.mac), obs_factory(lat=wifi.lat + 0.002, lon=wifi.lon + 0.005, mac=wifi.mac), obs_factory(lat=wifi.lat + 0.003, lon=wifi.lon + 0.009, mac=wifi.mac), ]) wifi.lat = None wifi.lon = None wifi.samples = None wifi.weight = None # a wifi with a prior known position wifi = wifis[1] wifi.samples = 1 wifi.weight = 1.0 wifi.lat += 1.0 wifi.lon += 1.0 obs.extend([ obs_factory(lat=wifi.lat + 0.01, lon=wifi.lon, mac=wifi.mac), obs_factory(lat=wifi.lat + 0.07, lon=wifi.lon, mac=wifi.mac), ]) moving.add(wifi.mac) # a wifi with a very different prior position wifi = wifis[2] wifi.samples = 1 wifi.weight = 1.0 obs.extend([ obs_factory(lat=wifi.lat + 2.0, lon=wifi.lon, mac=wifi.mac), obs_factory(lat=wifi.lat + 2.002, lon=wifi.lon, mac=wifi.mac), ]) moving.add(wifi.mac) # an already blocked wifi wifi = wifis[3] wifi.block_last = now.date() wifi.block_count = 1 obs.extend([ obs_factory(lat=wifi.lat, lon=wifi.lon, mac=wifi.mac), obs_factory(lat=wifi.lat + 0.1, lon=wifi.lon, mac=wifi.mac), ]) moving.add(wifi.mac) # a permanently blocked wifi wifi = wifis[4] wifi_lat, wifi_lon = (wifi.lat, wifi.lon) wifi.block_last = now.date() - 2 * TEMPORARY_BLOCKLIST_DURATION wifi.block_count = PERMANENT_BLOCKLIST_THRESHOLD for col in ('lat', 'lon', 'max_lat', 'min_lat', 'max_lon', 'min_lon'): setattr(wifi, col, None) obs.extend([ obs_factory(lat=wifi_lat, lon=wifi_lon, mac=wifi.mac), ]) moving.add(wifi.mac) # a no longer blocked wifi wifi = wifis[5] wifi_lat, wifi_lon = (wifi.lat, wifi.lon) wifi.block_last = now.date() - 2 * TEMPORARY_BLOCKLIST_DURATION wifi.block_count = 2 for col in ('lat', 'lon', 'max_lat', 'min_lat', 'max_lon', 'min_lon'): setattr(wifi, col, None) obs.extend([ obs_factory(lat=wifi_lat, lon=wifi_lon, mac=wifi.mac), ]) # a no longer blocked wifi with disagreeing observations wifi = wifis[6] wifi_lat, wifi_lon = (wifi.lat, wifi.lon) wifi.block_last = now.date() - 2 * TEMPORARY_BLOCKLIST_DURATION wifi.block_count = 2 for col in ('lat', 'lon', 'max_lat', 'min_lat', 'max_lon', 'min_lon'): setattr(wifi, col, None) obs.extend([ obs_factory(lat=wifi_lat, lon=wifi_lon, mac=wifi.mac), obs_factory(lat=wifi_lat + 2.0, lon=wifi_lon, mac=wifi.mac), ]) moving.add(wifi.mac) self.session.commit() self._queue_and_update_wifi(obs) shards = set() for mac in moving: shards.add(WifiShard.shard_model(mac)) blocks = [] for shard in shards: for row in self.session.query(shard).all(): if row.blocked(): blocks.append(row) self.assertEqual(set([b.mac for b in blocks]), moving)
def test_blocklist_moving_wifis(self): now = util.utcnow() obs = [] obs_factory = WifiObservationFactory moving = set() wifis = WifiShardFactory.create_batch(7) wifis.append(WifiShardFactory.build()) # a wifi without an entry and disagreeing observations wifi = wifis[-1] obs.extend([ obs_factory(lat=wifi.lat, lon=wifi.lon, key=wifi.mac), obs_factory(lat=wifi.lat + 2.0, lon=wifi.lon, key=wifi.mac), ]) moving.add(wifi.mac) # a wifi with an entry but no prior position wifi = wifis[0] obs.extend([ obs_factory(lat=wifi.lat + 0.001, lon=wifi.lon + 0.001, key=wifi.mac), obs_factory(lat=wifi.lat + 0.002, lon=wifi.lon + 0.005, key=wifi.mac), obs_factory(lat=wifi.lat + 0.003, lon=wifi.lon + 0.009, key=wifi.mac), ]) wifi.lat = None wifi.lon = None wifi.samples = 0 # a wifi with a prior known position wifi = wifis[1] wifi.samples = 1 wifi.lat += 1.0 wifi.lon += 1.0 obs.extend([ obs_factory(lat=wifi.lat + 0.01, lon=wifi.lon, key=wifi.mac), obs_factory(lat=wifi.lat + 0.07, lon=wifi.lon, key=wifi.mac), ]) moving.add(wifi.mac) # a wifi with a very different prior position wifi = wifis[2] wifi.samples = 1 obs.extend([ obs_factory(lat=wifi.lat + 2.0, lon=wifi.lon, key=wifi.mac), obs_factory(lat=wifi.lat + 2.002, lon=wifi.lon, key=wifi.mac), ]) moving.add(wifi.mac) # an already blocked wifi wifi = wifis[3] wifi.block_last = now.date() wifi.block_count = 1 obs.extend([ obs_factory(lat=wifi.lat, lon=wifi.lon, key=wifi.mac), obs_factory(lat=wifi.lat + 0.1, lon=wifi.lon, key=wifi.mac), ]) moving.add(wifi.mac) # a permanently blocked wifi wifi = wifis[4] wifi_lat, wifi_lon = (wifi.lat, wifi.lon) wifi.block_last = now.date() - 2 * TEMPORARY_BLOCKLIST_DURATION wifi.block_count = PERMANENT_BLOCKLIST_THRESHOLD for col in ('lat', 'lon', 'max_lat', 'min_lat', 'max_lon', 'min_lon'): setattr(wifi, col, None) obs.extend([ obs_factory(lat=wifi_lat, lon=wifi_lon, key=wifi.mac), ]) moving.add(wifi.mac) # a no longer blocked wifi wifi = wifis[5] wifi_lat, wifi_lon = (wifi.lat, wifi.lon) wifi.block_last = now.date() - 2 * TEMPORARY_BLOCKLIST_DURATION wifi.block_count = 2 for col in ('lat', 'lon', 'max_lat', 'min_lat', 'max_lon', 'min_lon'): setattr(wifi, col, None) obs.extend([ obs_factory(lat=wifi_lat, lon=wifi_lon, key=wifi.mac), ]) # a no longer blocked wifi with disagreeing observations wifi = wifis[6] wifi_lat, wifi_lon = (wifi.lat, wifi.lon) wifi.block_last = now.date() - 2 * TEMPORARY_BLOCKLIST_DURATION wifi.block_count = 2 for col in ('lat', 'lon', 'max_lat', 'min_lat', 'max_lon', 'min_lon'): setattr(wifi, col, None) obs.extend([ obs_factory(lat=wifi_lat, lon=wifi_lon, key=wifi.mac), obs_factory(lat=wifi_lat + 2.0, lon=wifi_lon, key=wifi.mac), ]) moving.add(wifi.mac) self.session.commit() self._queue_and_update(obs) shards = set() for mac in moving: shards.add(WifiShard.shard_model(mac)) blocks = [] for shard in shards: for row in self.session.query(shard).all(): if row.blocked(): blocks.append(row) self.assertEqual(set([b.mac for b in blocks]), moving)
def test_blocklist_moving_wifis(self): now = util.utcnow() obs = [] obs_factory = WifiObservationFactory moving = set() wifis = WifiShardFactory.create_batch(8) wifis.append(WifiShardFactory.build()) # a wifi without an entry and disagreeing observations wifi = wifis[-1] obs.extend([ obs_factory(lat=wifi.lat, lon=wifi.lon, key=wifi.mac), obs_factory(lat=wifi.lat + 2.0, lon=wifi.lon, key=wifi.mac), ]) moving.add(wifi.mac) # a wifi with an entry but no prior position wifi = wifis[0] obs.extend([ obs_factory(lat=wifi.lat + 0.001, lon=wifi.lon + 0.001, key=wifi.mac), obs_factory(lat=wifi.lat + 0.002, lon=wifi.lon + 0.005, key=wifi.mac), obs_factory(lat=wifi.lat + 0.003, lon=wifi.lon + 0.009, key=wifi.mac), ]) wifi.lat = None wifi.lon = None wifi.samples = 0 # a wifi with a prior known position wifi = wifis[1] wifi.samples = 1 wifi.lat += 1.0 wifi.lon += 1.0 obs.extend([ obs_factory(lat=wifi.lat + 0.01, lon=wifi.lon, key=wifi.mac), obs_factory(lat=wifi.lat + 0.07, lon=wifi.lon, key=wifi.mac), ]) moving.add(wifi.mac) # a wifi with a very different prior position wifi = wifis[2] wifi.samples = 1 obs.extend([ obs_factory(lat=wifi.lat + 2.0, lon=wifi.lon, key=wifi.mac), obs_factory(lat=wifi.lat + 2.002, lon=wifi.lon, key=wifi.mac), ]) moving.add(wifi.mac) # another wifi with a prior known position (and negative lat) wifi = wifis[3] wifi.samples = 1 wifi.lat *= -1.0 obs.extend([ obs_factory(lat=wifi.lat - 0.1, lon=wifi.lon, key=wifi.mac), obs_factory(lat=wifi.lat - 0.16, lon=wifi.lon, key=wifi.mac), ]) moving.add(wifi.mac) # an already blocked wifi wifi = wifis[4] wifi.block_last = now.date() wifi.block_count = 1 obs.extend([ obs_factory(lat=wifi.lat, lon=wifi.lon, key=wifi.mac), obs_factory(lat=wifi.lat + 0.1, lon=wifi.lon, key=wifi.mac), ]) moving.add(wifi.mac) # a permanently blocked wifi wifi = wifis[5] wifi_lat, wifi_lon = (wifi.lat, wifi.lon) wifi.block_last = now.date() - 2 * TEMPORARY_BLOCKLIST_DURATION wifi.block_count = PERMANENT_BLOCKLIST_THRESHOLD for col in ('lat', 'lon', 'max_lat', 'min_lat', 'max_lon', 'min_lon'): setattr(wifi, col, None) obs.extend([ obs_factory(lat=wifi_lat, lon=wifi_lon, key=wifi.mac), ]) moving.add(wifi.mac) # a no longer blocked wifi wifi = wifis[6] wifi_lat, wifi_lon = (wifi.lat, wifi.lon) wifi.block_last = now.date() - 2 * TEMPORARY_BLOCKLIST_DURATION wifi.block_count = 2 for col in ('lat', 'lon', 'max_lat', 'min_lat', 'max_lon', 'min_lon'): setattr(wifi, col, None) obs.extend([ obs_factory(lat=wifi_lat, lon=wifi_lon, key=wifi.mac), ]) # a no longer blocked wifi with disagreeing observations wifi = wifis[7] wifi_lat, wifi_lon = (wifi.lat, wifi.lon) wifi.block_last = now.date() - 2 * TEMPORARY_BLOCKLIST_DURATION wifi.block_count = 2 for col in ('lat', 'lon', 'max_lat', 'min_lat', 'max_lon', 'min_lon'): setattr(wifi, col, None) obs.extend([ obs_factory(lat=wifi_lat, lon=wifi_lon, key=wifi.mac), obs_factory(lat=wifi_lat + 2.0, lon=wifi_lon, key=wifi.mac), ]) moving.add(wifi.mac) self.data_queue.enqueue(obs) self.session.commit() update_wifi.delay().get() shards = set() for mac in moving: shards.add(WifiShard.shard_model(mac)) blocks = [] for shard in shards: for row in self.session.query(shard).all(): if row.blocked(): blocks.append(row) self.assertEqual(set([b.mac for b in blocks]), moving) self.check_stats(counter=[ ('data.observation.drop', 1, 3, ['type:wifi', 'reason:blocklisted']), ('data.station.blocklist', 1, 5, ['type:wifi', 'action:add', 'reason:moving']), ])