def test_remove_wifi(self): observations = [] wifi_keys = [] m1 = 1.0 m2 = 2.0 for key in ['a%s1234567890' % i for i in range(5)]: wifi = Wifi(key=key) wifi_keys.append(wifi.hashkey()) observations.append(wifi) observations.append(WifiObservation(lat=m1, lon=m1, key=key)) observations.append(WifiObservation(lat=m2, lon=m2, key=key)) self.session.add_all(observations) self.session.flush() result = remove_wifi.delay(wifi_keys[:2]) self.assertEqual(result.get(), 2) wifis = self.session.query(Wifi).all() self.assertEqual(len(wifis), 3) result = remove_wifi.delay(wifi_keys) self.assertEqual(result.get(), 3) result = remove_wifi.delay(wifi_keys) self.assertEqual(result.get(), 0) wifis = self.session.query(Wifi).all() self.assertEqual(len(wifis), 0)
def test_remove_wifi(self): session = self.session observations = [] wifi_keys = [{'key': "a%s1234567890" % i} for i in range(5)] m1 = 1.0 m2 = 2.0 for key in wifi_keys: key = key['key'] observations.append(Wifi(key=key)) observations.append(WifiObservation(lat=m1, lon=m1, key=key)) observations.append(WifiObservation(lat=m2, lon=m2, key=key)) session.add_all(observations) session.flush() result = remove_wifi.delay(wifi_keys[:2]) self.assertEqual(result.get(), 2) wifis = session.query(Wifi).all() self.assertEqual(len(wifis), 3) result = remove_wifi.delay(wifi_keys) self.assertEqual(result.get(), 3) result = remove_wifi.delay(wifi_keys) self.assertEqual(result.get(), 0) wifis = session.query(Wifi).all() self.assertEqual(len(wifis), 0)
def test_max_min_range_update(self): session = self.session k1 = "ab1234567890" k2 = "cd1234567890" data = [ Wifi(key=k1, new_measures=2, total_measures=2), WifiObservation(lat=1.0, lon=1.0, key=k1), WifiObservation(lat=1.002, lon=1.004, key=k1), Wifi(key=k2, lat=2.0, lon=-2.0, max_lat=2.001, min_lat=1.999, max_lon=-1.999, min_lon=-2.001, new_measures=2, total_measures=4), WifiObservation(lat=2.002, lon=-2.004, key=k2), WifiObservation(lat=1.998, lon=-1.996, key=k2), ] session.add_all(data) session.commit() result = location_update_wifi.delay(min_new=1) self.assertEqual(result.get(), (2, 0)) wifis = dict(session.query(Wifi.key, Wifi).all()) self.assertEqual(set(wifis.keys()), set([k1, k2])) self.assertEqual(wifis[k1].lat, 1.001) self.assertEqual(wifis[k1].max_lat, 1.002) self.assertEqual(wifis[k1].min_lat, 1.0) self.assertEqual(wifis[k1].lon, 1.002) self.assertEqual(wifis[k1].max_lon, 1.004) self.assertEqual(wifis[k1].min_lon, 1.0) self.assertEqual(wifis[k2].lat, 2.0) self.assertEqual(wifis[k2].max_lat, 2.002) self.assertEqual(wifis[k2].min_lat, 1.998) self.assertEqual(wifis[k2].lon, -2.0) self.assertEqual(wifis[k2].max_lon, -1.996) self.assertEqual(wifis[k2].min_lon, -2.004) # independent calculation: the k1 bounding box is # (1.000, 1.000) to (1.002, 1.004), with centroid # at (1.001, 1.002); worst distance from centroid # to any corner is 249m self.assertEqual(wifis[k1].range, 249) # independent calculation: the k2 bounding box is # (1.998, -2.004) to (2.002, -1.996), with centroid # at (2.000, 2.000); worst distance from centroid # to any corner is 497m self.assertEqual(wifis[k2].range, 497)
def test_schedule_wifi_observations(self): session = self.db_master_session blocks = schedule_wifimeasure_archival.delay(batch=1).get() self.assertEquals(len(blocks), 0) batch_size = 10 observations = [] for i in range(batch_size * 2): observations.append(WifiObservation(created=self.really_old)) session.add_all(observations) session.flush() start_id = observations[0].id blocks = schedule_wifimeasure_archival.delay(batch=batch_size).get() self.assertEquals(len(blocks), 2) block = blocks[0] self.assertEquals(block, (start_id, start_id + batch_size)) block = blocks[1] self.assertEquals(block, (start_id + batch_size, start_id + 2 * batch_size)) blocks = schedule_wifimeasure_archival.delay(batch=batch_size).get() self.assertEquals(len(blocks), 0)
def test_unthrottle_wifi_observations(self): session = self.db_master_session block = ObservationBlock() block.measure_type = ObservationType.wifi block.start_id = 120 block.end_id = 140 block.s3_key = 'fake_key' block.archive_sha = 'fake_sha' block.archive_date = None session.add(block) k = dict(lat=1.0, lon=1.0) for i in range(100, 150): session.add( WifiObservation(id=i, key=str(i), created=self.really_old)) session.add(Wifi(total_measures=11000, key=str(i), **k)) session.commit() with patch.object(S3Backend, 'check_archive', lambda x, y, z: True): delete_wifimeasure_records.delay(batch=7).get() wifi_unthrottle_measures.delay(10000, 1000).get() wifis = session.query(Wifi).all() self.assertEquals(len(wifis), 50) for wifi in wifis: if 120 <= int(wifi.key) and int(wifi.key) < 140: self.assertEquals(wifi.total_measures, 0) else: self.assertEquals(wifi.total_measures, 1) self.check_stats(counter=['items.wifi_unthrottled'])
def test_backup_wifi_to_s3(self): session = self.db_master_session batch_size = 10 observations = [] for i in range(batch_size): observations.append(WifiObservation(created=self.really_old)) session.add_all(observations) session.flush() start_id = observations[0].id blocks = schedule_wifimeasure_archival.delay(batch=batch_size).get() self.assertEquals(len(blocks), 1) block = blocks[0] self.assertEquals(block, (start_id, start_id + batch_size)) with mock_s3(): with patch.object(S3Backend, 'backup_archive', lambda x, y, z: True): write_wifimeasure_s3_backups.delay(cleanup_zip=False).get() msgs = self.heka_client.stream.msgs info_msgs = [m for m in msgs if m.type == 'oldstyle'] self.assertEquals(1, len(info_msgs)) info = info_msgs[0] fname = info.payload.split(":")[-1] myzip = ZipFile(fname) try: contents = set(myzip.namelist()) expected_contents = set(['alembic_revision.txt', 'wifi_measure.csv']) self.assertEquals(expected_contents, contents) finally: myzip.close() blocks = session.query(ObservationBlock).all() self.assertEquals(len(blocks), 1) block = blocks[0] actual_sha = hashlib.sha1() actual_sha.update(open(fname, 'rb').read()) self.assertEquals(block.archive_sha, actual_sha.digest()) self.assertTrue(block.s3_key is not None) self.assertTrue('/wifi_' in block.s3_key) self.assertTrue(block.archive_date is None)
def test_monitor_measures(self): session = self.db_master_session result = monitor_measures.delay().get() self.check_stats( gauge=[('table.cell_measure', 1), ('table.wifi_measure', 1)], ) self.assertEqual(result, {'cell_measure': -1, 'wifi_measure': -1}) # add some observations session.add_all([CellObservation() for i in range(3)]) session.add_all([WifiObservation() for i in range(5)]) session.flush() result = monitor_measures.delay().get() self.check_stats( gauge=[('table.cell_measure', 2), ('table.wifi_measure', 2)], ) self.assertEqual(result, {'cell_measure': 3, 'wifi_measure': 5})
def test_delete_wifi_observations(self): session = self.db_master_session block = ObservationBlock() block.measure_type = ObservationType.wifi block.start_id = 120 block.end_id = 140 block.s3_key = 'fake_key' block.archive_sha = 'fake_sha' block.archive_date = None session.add(block) for i in range(100, 150): session.add(WifiObservation(id=i, created=self.really_old)) session.commit() with patch.object(S3Backend, 'check_archive', lambda x, y, z: True): delete_wifimeasure_records.delay(batch=7).get() self.assertEquals(session.query(WifiObservation).count(), 30) self.assertTrue(block.archive_date is not None)
def test_wifi_histogram(self): session = self.db_master_session today = util.utcnow().date() yesterday = (today - timedelta(1)) two_days = (today - timedelta(2)) long_ago = (today - timedelta(3)) observations = [ WifiObservation(lat=1.0, lon=2.0, created=today), WifiObservation(lat=1.0, lon=2.0, created=today), WifiObservation(lat=1.0, lon=2.0, created=yesterday), WifiObservation(lat=1.0, lon=2.0, created=two_days), WifiObservation(lat=1.0, lon=2.0, created=two_days), WifiObservation(lat=1.0, lon=2.0, created=two_days), WifiObservation(lat=1.0, lon=2.0, created=long_ago), ] session.add_all(observations) session.commit() wifi_histogram.delay(ago=3).get() stats = session.query(Stat).order_by(Stat.time).all() self.assertEqual(len(stats), 1) self.assertEqual(stats[0].key, StatKey.wifi) self.assertEqual(stats[0].time, long_ago) self.assertEqual(stats[0].value, 1) # fill in newer dates wifi_histogram.delay(ago=2).get() wifi_histogram.delay(ago=1).get() wifi_histogram.delay(ago=0).get() # test duplicate execution wifi_histogram.delay(ago=1).get() stats = session.query(Stat).order_by(Stat.time).all() self.assertEqual(len(stats), 4) self.assertEqual(stats[0].time, long_ago) self.assertEqual(stats[0].value, 1) self.assertEqual(stats[1].time, two_days) self.assertEqual(stats[1].value, 4) self.assertEqual(stats[2].time, yesterday) self.assertEqual(stats[2].value, 5) self.assertEqual(stats[3].time, today) self.assertEqual(stats[3].value, 7)
def test_location_update_wifi(self): now = util.utcnow() before = now - timedelta(days=1) session = self.session k1 = "ab1234567890" k2 = "cd1234567890" data = [ Wifi(key=k1, new_measures=3, total_measures=3), WifiObservation(lat=1.0, lon=1.0, key=k1, created=now), WifiObservation(lat=1.002, lon=1.003, key=k1, created=now), WifiObservation(lat=1.004, lon=1.006, key=k1, created=now), Wifi(key=k2, lat=2.0, lon=2.0, new_measures=2, total_measures=4), # the lat/lon is bogus and mismatches the line above on purpose # to make sure old observations are skipped WifiObservation(lat=-1.0, lon=-1.0, key=k2, created=before), WifiObservation(lat=-1.0, lon=-1.0, key=k2, created=before), WifiObservation(lat=2.002, lon=2.004, key=k2, created=now), WifiObservation(lat=2.002, lon=2.004, key=k2, created=now), ] session.add_all(data) session.commit() result = location_update_wifi.delay(min_new=1) self.assertEqual(result.get(), (2, 0)) self.check_stats( total=2, timer=['task.data.location_update_wifi'], gauge=['task.data.location_update_wifi.new_measures_1_100'], ) wifis = dict(session.query(Wifi.key, Wifi).all()) self.assertEqual(set(wifis.keys()), set([k1, k2])) self.assertEqual(wifis[k1].lat, 1.002) self.assertEqual(wifis[k1].lon, 1.003) self.assertEqual(wifis[k1].new_measures, 0) self.assertEqual(wifis[k2].lat, 2.001) self.assertEqual(wifis[k2].lon, 2.002) self.assertEqual(wifis[k2].new_measures, 0)
def test_blacklist_moving_wifis(self): now = util.utcnow() long_ago = now - timedelta(days=40) session = self.session k1 = "ab1234567890" k2 = "bc1234567890" k3 = "cd1234567890" k4 = "de1234567890" k5 = "ef1234567890" k6 = "fa1234567890" keys = set([k1, k2, k3, k4, k5, k6]) # keys k2, k3 and k4 are expected to be detected as moving data = [ # a wifi with an entry but no prior position Wifi(key=k1, new_measures=3, total_measures=0), WifiObservation(lat=1.001, lon=1.001, key=k1), WifiObservation(lat=1.002, lon=1.005, key=k1), WifiObservation(lat=1.003, lon=1.009, key=k1), # a wifi with a prior known position Wifi(lat=2.0, lon=2.0, key=k2, new_measures=2, total_measures=1), WifiObservation(lat=2.01, lon=2, key=k2), WifiObservation(lat=2.07, lon=2, key=k2), # a wifi with a very different prior position Wifi(lat=1.0, lon=1.0, key=k3, new_measures=2, total_measures=1), WifiObservation(lat=3.0, lon=3.0, key=k3), WifiObservation(lat=-3.0, lon=3.0, key=k3), # another wifi with a prior known position (and negative lat) Wifi(lat=-4.0, lon=4.0, key=k4, new_measures=2, total_measures=1), WifiObservation(lat=-4.1, lon=4, key=k4), WifiObservation(lat=-4.16, lon=4, key=k4), # an already blacklisted wifi WifiBlacklist(key=k5, time=now, count=1), WifiObservation(lat=5.0, lon=5.0, key=k5), WifiObservation(lat=5.1, lon=5.0, key=k5), # a wifi with an old different record we ignore, position # estimate has been updated since Wifi(lat=6.0, lon=6.0, key=k6, new_measures=2, total_measures=1), WifiObservation(lat=6.9, lon=6.9, key=k6, time=long_ago), WifiObservation(lat=6.0, lon=6.0, key=k6), WifiObservation(lat=6.001, lon=6.0, key=k6), ] session.add_all(data) session.commit() result = location_update_wifi.delay(min_new=1) self.assertEqual(result.get(), (5, 3)) black = session.query(WifiBlacklist).all() self.assertEqual(set([b.key for b in black]), set([k2, k3, k4, k5])) observations = session.query(WifiObservation).all() self.assertEqual(len(observations), 14) self.assertEqual(set([o.key for o in observations]), keys) # test duplicate call result = location_update_wifi.delay(min_new=1) self.assertEqual(result.get(), (0, 0)) self.check_stats( total=6, timer=[ # We made duplicate calls ('task.data.location_update_wifi', 2), # One of those would've scheduled a remove_wifi task ('task.data.remove_wifi', 1) ], gauge=[ ('task.data.location_update_wifi.new_measures_1_100', 2), ])