def test_temp_blocked(self): utcnow = util.utcnow() bad_wifi = WifiObservationFactory.build() good_wifi = WifiObservationFactory.build() WifiShardFactory( mac=bad_wifi.mac, lat=None, lon=None, created=utcnow, block_first=utcnow.date() - timedelta(days=10), block_last=utcnow.date(), block_count=1) obs = [good_wifi, bad_wifi, good_wifi] self.session.commit() self._queue_and_update(obs) shard = WifiShard.shard_model(good_wifi.mac) wifis = (self.session.query(shard) .filter(shard.mac == good_wifi.mac)).all() self.assertEqual(len(wifis), 1) self.assertTrue(wifis[0].lat is not None) self.assertTrue(wifis[0].lon is not None) self.assertEqual(wifis[0].samples, 2) shard = WifiShard.shard_model(bad_wifi.mac) wifis = (self.session.query(shard) .filter(shard.mac == bad_wifi.mac)).all() self.assertEqual(len(wifis), 1) self.assertTrue(wifis[0].block_first < utcnow.date()) self.assertTrue(wifis[0].lat is None) self.assertTrue(wifis[0].lon is None) self.check_statcounter(StatKey.wifi, 2) self.check_statcounter(StatKey.unique_wifi, 1)
def test_temp_blocked(self): utcnow = util.utcnow() bad_wifi = WifiObservationFactory.build() good_wifi = WifiObservationFactory.build() WifiShardFactory(mac=bad_wifi.mac, lat=None, lon=None, created=utcnow, block_first=utcnow.date() - timedelta(days=10), block_last=utcnow.date(), block_count=1) obs = [good_wifi, bad_wifi, good_wifi] self.session.commit() self._queue_and_update(obs) shard = WifiShard.shard_model(good_wifi.mac) wifis = (self.session.query(shard).filter( shard.mac == good_wifi.mac)).all() self.assertEqual(len(wifis), 1) self.assertTrue(wifis[0].lat is not None) self.assertTrue(wifis[0].lon is not None) self.assertEqual(wifis[0].samples, 2) shard = WifiShard.shard_model(bad_wifi.mac) wifis = (self.session.query(shard).filter( shard.mac == bad_wifi.mac)).all() self.assertEqual(len(wifis), 1) self.assertTrue(wifis[0].block_first < utcnow.date()) self.assertTrue(wifis[0].lat is None) self.assertTrue(wifis[0].lon is None) self.check_statcounter(StatKey.wifi, 2) self.check_statcounter(StatKey.unique_wifi, 1)
def test_monitor_measures(self): result = monitor_measures.delay().get() self.check_stats(gauge=[('table.cell_measure', 1), ('table.wifi_measure', 1)], ) self.assertEqual(result, {'cell_measure': -1, 'wifi_measure': -1}) # add some observations CellObservationFactory.create_batch(3) WifiObservationFactory.create_batch(5) self.session.flush() result = monitor_measures.delay().get() self.check_stats(gauge=[('table.cell_measure', 2), ('table.wifi_measure', 2)], ) self.assertEqual(result, {'cell_measure': 3, 'wifi_measure': 5})
def test_new(self): utcnow = util.utcnow() obs = WifiObservationFactory.build() self.data_queue.enqueue([obs]) self.assertEqual(self.data_queue.size(), 1) update_wifi.delay().get() shard = WifiShard.shard_model(obs.mac) wifis = self.session.query(shard).all() self.assertEqual(len(wifis), 1) wifi = wifis[0] self.assertAlmostEqual(wifi.lat, obs.lat) self.assertAlmostEqual(wifi.max_lat, obs.lat) self.assertAlmostEqual(wifi.min_lat, obs.lat) self.assertAlmostEqual(wifi.lon, obs.lon) self.assertAlmostEqual(wifi.max_lon, obs.lon) self.assertAlmostEqual(wifi.min_lon, obs.lon) self.assertEqual(wifi.country, 'GB') self.assertEqual(wifi.radius, 0) self.assertEqual(wifi.samples, 1) self.assertEqual(wifi.created.date(), utcnow.date()) self.assertEqual(wifi.modified.date(), utcnow.date()) self.assertEqual(wifi.block_first, None) self.assertEqual(wifi.block_last, None) self.assertEqual(wifi.block_count, None)
def test_temp_blocked_admitted_again(self): now = util.utcnow() last_week = now - TEMPORARY_BLOCKLIST_DURATION - timedelta(days=1) obs = WifiObservationFactory() WifiShardFactory(mac=obs.mac, lat=None, lon=None, samples=0, created=last_week, modified=last_week, block_first=last_week.date(), block_last=last_week.date(), block_count=1) self.session.commit() # add a new entry for the previously blocked wifi self._queue_and_update([obs]) # the wifi was inserted again shard = WifiShard.shard_model(obs.mac) wifis = self.session.query(shard).all() self.assertEqual(len(wifis), 1) wifi = wifis[0] self.assertEqual(wifi.block_first, last_week.date()) self.assertEqual(wifi.block_last, last_week.date()) self.assertEqual(wifi.created.date(), last_week.date()) self.assertAlmostEqual(wifi.lat, obs.lat) self.assertAlmostEqual(wifi.lon, obs.lon) self.assertEqual(wifi.region, 'GB') self.assertEqual(wifi.samples, 1) self.check_statcounter(StatKey.unique_wifi, 0)
def test_monitor_measures(self): result = monitor_measures.delay().get() self.check_stats( gauge=[('table.cell_measure', 1), ('table.wifi_measure', 1)], ) self.assertEqual(result, {'cell_measure': -1, 'wifi_measure': -1}) # add some observations CellObservationFactory.create_batch(3) WifiObservationFactory.create_batch(5) self.session.flush() result = monitor_measures.delay().get() self.check_stats( gauge=[('table.cell_measure', 2), ('table.wifi_measure', 2)], ) self.assertEqual(result, {'cell_measure': 3, 'wifi_measure': 5})
def test_json(self): obs = WifiObservationFactory.build(accuracy=None) result = WifiObservation.from_json(simplejson.loads( simplejson.dumps(obs.to_json()))) self.assertTrue(type(result), WifiObservation) self.assertTrue(result.accuracy is None) self.assertEqual(result.mac, obs.mac) self.assertEqual(result.lat, obs.lat) self.assertEqual(result.lon, obs.lon)
def test_json(self): obs = WifiObservationFactory.build(accuracy=None, source=ReportSource.query) result = WifiObservation.from_json(json.loads(json.dumps(obs.to_json()))) assert type(result) is WifiObservation assert result.accuracy is None assert result.mac == obs.mac assert result.lat == obs.lat assert result.lon == obs.lon assert result.source == ReportSource.query assert type(result.source) is ReportSource
def test_json(self): obs = WifiObservationFactory.build( accuracy=None, source=ReportSource.query) result = WifiObservation.from_json(simplejson.loads( simplejson.dumps(obs.to_json()))) assert type(result) is WifiObservation assert result.accuracy is None assert result.mac == obs.mac assert result.lat == obs.lat assert result.lon == obs.lon assert result.source == ReportSource.query assert type(result.source) is ReportSource
def test_delete_wifi_observations(self): obs = WifiObservationFactory.create_batch(50, created=self.old) self.session.flush() start_id = obs[0].id + 20 block = ObservationBlockFactory( measure_type=ObservationType.wifi, start_id=start_id, end_id=start_id + 20, archive_date=None) self.session.commit() delete_wifimeasure_records.delay(batch=7).get() self.assertEquals(self.session.query(WifiObservation).count(), 30) self.assertTrue(block.archive_date is not None)
def test_delete_wifi_observations(self): obs = WifiObservationFactory.create_batch(50, created=self.old) self.session.flush() start_id = obs[0].id + 20 block = ObservationBlockFactory(measure_type=ObservationType.wifi, start_id=start_id, end_id=start_id + 20, archive_date=None) self.session.commit() with patch.object(S3Backend, 'check_archive', lambda x, y, z: True): delete_wifimeasure_records.delay(batch=7).get() self.assertEquals(self.session.query(WifiObservation).count(), 30) self.assertTrue(block.archive_date is not None)
def test_backup_wifi_to_s3(self): batch_size = 10 obs = WifiObservationFactory.create_batch(batch_size, created=self.old) self.session.flush() start_id = obs[0].id blocks = schedule_wifimeasure_archival.delay(batch=batch_size).get() self.assertEquals(len(blocks), 1) block = blocks[0] self.assertEquals(block, (start_id, start_id + batch_size)) write_wifimeasure_s3_backups.delay(cleanup_zip=False).get() blocks = self.session.query(ObservationBlock).all() self.assertEquals(len(blocks), 1) block = blocks[0] self.assertEqual(block.archive_sha, '20bytes_mean_success') self.assertEqual(block.s3_key, 'skipped') self.assertTrue(block.archive_date is None)
def test_backup_wifi_to_s3(self): batch_size = 10 obs = WifiObservationFactory.create_batch(batch_size, created=self.old) self.session.flush() start_id = obs[0].id blocks = schedule_wifimeasure_archival.delay(batch=batch_size).get() self.assertEquals(len(blocks), 1) block = blocks[0] self.assertEquals(block, (start_id, start_id + batch_size)) with mock_s3(): with patch.object(S3Backend, 'backup_archive', lambda x, y, z: True): write_wifimeasure_s3_backups.delay(cleanup_zip=False).get() raven_msgs = self.raven_client.msgs fname = [ m['message'].split(':')[1] for m in raven_msgs if m['message'].startswith('s3.backup:') ][0] myzip = ZipFile(fname) try: contents = set(myzip.namelist()) expected_contents = set( ['alembic_revision.txt', 'wifi_measure.csv']) self.assertEquals(expected_contents, contents) finally: myzip.close() blocks = self.session.query(ObservationBlock).all() self.assertEquals(len(blocks), 1) block = blocks[0] actual_sha = hashlib.sha1() actual_sha.update(open(fname, 'rb').read()) self.assertEquals(block.archive_sha, actual_sha.digest()) self.assertTrue(block.s3_key is not None) self.assertTrue('/wifi_' in block.s3_key) self.assertTrue(block.archive_date is None)
def test_schedule_wifi_observations(self): blocks = schedule_wifimeasure_archival.delay(batch=1).get() self.assertEquals(len(blocks), 0) batch_size = 10 obs = WifiObservationFactory.create_batch(batch_size * 2, created=self.old) self.session.flush() start_id = obs[0].id blocks = schedule_wifimeasure_archival.delay(batch=batch_size).get() self.assertEquals(len(blocks), 2) block = blocks[0] self.assertEquals(block, (start_id, start_id + batch_size)) block = blocks[1] self.assertEquals(block, (start_id + batch_size, start_id + 2 * batch_size)) blocks = schedule_wifimeasure_archival.delay(batch=batch_size).get() self.assertEquals(len(blocks), 0)
def test_backup_wifi_to_s3(self): batch_size = 10 obs = WifiObservationFactory.create_batch(batch_size, created=self.old) self.session.flush() start_id = obs[0].id blocks = schedule_wifimeasure_archival.delay(batch=batch_size).get() self.assertEquals(len(blocks), 1) block = blocks[0] self.assertEquals(block, (start_id, start_id + batch_size)) with mock_s3(): with patch.object(S3Backend, 'backup_archive', lambda x, y, z: True): write_wifimeasure_s3_backups.delay(cleanup_zip=False).get() raven_msgs = self.raven_client.msgs fname = [m['message'].split(':')[1] for m in raven_msgs if m['message'].startswith('s3.backup:')][0] myzip = ZipFile(fname) try: contents = set(myzip.namelist()) expected_contents = set(['alembic_revision.txt', 'wifi_measure.csv']) self.assertEquals(expected_contents, contents) finally: myzip.close() blocks = self.session.query(ObservationBlock).all() self.assertEquals(len(blocks), 1) block = blocks[0] actual_sha = hashlib.sha1() actual_sha.update(open(fname, 'rb').read()) self.assertEquals(block.archive_sha, actual_sha.digest()) self.assertTrue(block.s3_key is not None) self.assertTrue('/wifi_' in block.s3_key) self.assertTrue(block.archive_date is None)
def test_schedule_wifi_observations(self): blocks = schedule_wifimeasure_archival.delay(batch=1).get() self.assertEquals(len(blocks), 0) batch_size = 10 obs = WifiObservationFactory.create_batch( batch_size * 2, created=self.old) self.session.flush() start_id = obs[0].id blocks = schedule_wifimeasure_archival.delay(batch=batch_size).get() self.assertEquals(len(blocks), 2) block = blocks[0] self.assertEquals(block, (start_id, start_id + batch_size)) block = blocks[1] self.assertEquals(block, (start_id + batch_size, start_id + 2 * batch_size)) blocks = schedule_wifimeasure_archival.delay(batch=batch_size).get() self.assertEquals(len(blocks), 0)
def test_new(self): utcnow = util.utcnow() obs = WifiObservationFactory.build() self._queue_and_update([obs]) shard = WifiShard.shard_model(obs.mac) wifis = self.session.query(shard).all() self.assertEqual(len(wifis), 1) wifi = wifis[0] self.assertAlmostEqual(wifi.lat, obs.lat) self.assertAlmostEqual(wifi.max_lat, obs.lat) self.assertAlmostEqual(wifi.min_lat, obs.lat) self.assertAlmostEqual(wifi.lon, obs.lon) self.assertAlmostEqual(wifi.max_lon, obs.lon) self.assertAlmostEqual(wifi.min_lon, obs.lon) self.assertEqual(wifi.radius, 0) self.assertEqual(wifi.region, 'GB') self.assertEqual(wifi.samples, 1) self.assertEqual(wifi.created.date(), utcnow.date()) self.assertEqual(wifi.modified.date(), utcnow.date()) self.assertEqual(wifi.block_first, None) self.assertEqual(wifi.block_last, None) self.assertEqual(wifi.block_count, None)
def test_wifi_histogram(self): session = self.session today = util.utcnow() yesterday = (today - timedelta(1)) two_days = (today - timedelta(2)) long_ago = (today - timedelta(3)) WifiObservationFactory.create_batch(2, created=today) WifiObservationFactory(created=yesterday) WifiObservationFactory.create_batch(3, created=two_days) WifiObservationFactory(created=long_ago) session.flush() wifi_histogram.delay(ago=3).get() stats = session.query(Stat).order_by(Stat.time).all() self.assertEqual(len(stats), 1) self.assertEqual(stats[0].key, StatKey.wifi) self.assertEqual(stats[0].time, long_ago.date()) self.assertEqual(stats[0].value, 1) # fill in newer dates wifi_histogram.delay(ago=2).get() wifi_histogram.delay(ago=1).get() wifi_histogram.delay(ago=0).get() # test duplicate execution wifi_histogram.delay(ago=1).get() stats = session.query(Stat.time, Stat.value).order_by(Stat.time).all() self.assertEqual(len(stats), 4) self.assertEqual(dict(stats), { long_ago.date(): 1, two_days.date(): 4, yesterday.date(): 5, today.date(): 7})