def test_skip_delete_new_blocks(self): now = util.utcnow() today_0000 = now.replace(hour=0, minute=0, second=0, tzinfo=pytz.UTC) yesterday_0000 = today_0000 - timedelta(days=1) yesterday_2359 = today_0000 - timedelta(seconds=1) old = now - timedelta(days=5) session = self.db_master_session for i in range(100, 150, 10): block = ObservationBlock() block.measure_type = ObservationType.cell block.start_id = i block.end_id = i + 10 block.s3_key = 'fake_key' block.archive_sha = 'fake_sha' block.archive_date = None session.add(block) observations = [] for i in range(100, 110): observations.append(CellObservation(id=i, created=old)) for i in range(110, 120): observations.append(CellObservation(id=i, created=yesterday_0000)) for i in range(120, 130): observations.append(CellObservation(id=i, created=yesterday_2359)) for i in range(130, 140): observations.append(CellObservation(id=i, created=today_0000)) for i in range(140, 150): observations.append(CellObservation(id=i, created=now)) session.add_all(observations) session.commit() def _archived_blocks(): blocks = session.query(ObservationBlock).all() return len([b for b in blocks if b.archive_date is not None]) def _delete(days=7): with patch.object(S3Backend, 'check_archive', lambda x, y, z: True): delete_cellmeasure_records.delay(days_old=days).get() session.commit() _delete(days=7) self.assertEquals(session.query(CellObservation).count(), 50) self.assertEqual(_archived_blocks(), 0) _delete(days=2) self.assertEquals(session.query(CellObservation).count(), 40) self.assertEqual(_archived_blocks(), 1) _delete(days=1) self.assertEquals(session.query(CellObservation).count(), 20) self.assertEqual(_archived_blocks(), 3) _delete(days=0) self.assertEquals(session.query(CellObservation).count(), 0) self.assertEqual(_archived_blocks(), 5)
def add_line_of_cells_and_scan_lac(self): session = self.session big = 1.0 small = big / 10 keys = dict(radio=Radio.cdma, mcc=1, mnc=1, lac=1) observations = [ CellObservation(lat=ctr + xd, lon=ctr + yd, cid=cell, **keys) for cell in range(10) for ctr in [cell * big] for (xd, yd) in [(small, small), (small, -small), (-small, small), (-small, -small)] ] session.add_all(observations) cells = [ Cell(lat=ctr, lon=ctr, cid=cell, new_measures=4, total_measures=1, **keys) for cell in range(10) for ctr in [cell * big] ] session.add_all(cells) session.commit() result = location_update_cell.delay(min_new=0, max_new=9999, batch=len(observations)) self.assertEqual(result.get(), (len(cells), 0)) scan_areas.delay()
def test_schedule_cell_observations(self): session = self.db_master_session blocks = schedule_cellmeasure_archival.delay(batch=1).get() self.assertEquals(len(blocks), 0) observations = [] for i in range(20): observations.append(CellObservation(created=self.really_old)) session.add_all(observations) session.flush() start_id = observations[0].id blocks = schedule_cellmeasure_archival.delay(batch=15).get() self.assertEquals(len(blocks), 1) block = blocks[0] self.assertEquals(block, (start_id, start_id + 15)) blocks = schedule_cellmeasure_archival.delay(batch=6).get() self.assertEquals(len(blocks), 0) blocks = schedule_cellmeasure_archival.delay(batch=5).get() self.assertEquals(len(blocks), 1) block = blocks[0] self.assertEquals(block, (start_id + 15, start_id + 20)) blocks = schedule_cellmeasure_archival.delay(batch=1).get() self.assertEquals(len(blocks), 0)
def test_unthrottle_cell_observations(self): session = self.db_master_session block = ObservationBlock() block.measure_type = ObservationType.cell block.start_id = 120 block.end_id = 140 block.s3_key = 'fake_key' block.archive_sha = 'fake_sha' block.archive_date = None session.add(block) gsm = RADIO_TYPE['gsm'] k = dict(radio=gsm, mcc=1, mnc=2, lac=4, lat=1.0, lon=1.0) for i in range(100, 150): session.add( CellObservation(id=i, cid=i, created=self.really_old, **k)) session.add(Cell(total_measures=11000, cid=i, **k)) session.commit() with patch.object(S3Backend, 'check_archive', lambda x, y, z: True): delete_cellmeasure_records.delay(batch=3).get() cell_unthrottle_measures.delay(10000, 1000).get() cells = session.query(Cell).all() self.assertEquals(len(cells), 50) for cell in cells: if 120 <= cell.cid and cell.cid < 140: self.assertEquals(cell.total_measures, 0) else: self.assertEquals(cell.total_measures, 1) self.check_stats(counter=['items.cell_unthrottled'])
def test_location_update_cell(self): now = util.utcnow() before = now - timedelta(days=1) schema = ValidCellKeySchema() session = self.db_master_session k1 = dict(radio=1, mcc=1, mnc=2, lac=3, cid=4) k2 = dict(radio=1, mcc=1, mnc=2, lac=6, cid=8) k3 = dict(radio=1, mcc=1, mnc=2, lac=schema.fields['lac'].missing, cid=schema.fields['cid'].missing) data = [ Cell(new_measures=3, total_measures=5, **k1), CellObservation(lat=1.0, lon=1.0, created=now, **k1), CellObservation(lat=1.002, lon=1.003, created=now, **k1), CellObservation(lat=1.004, lon=1.006, created=now, **k1), # The lac, cid are invalid and should be skipped CellObservation(lat=1.5, lon=1.5, created=now, **k3), CellObservation(lat=1.502, lon=1.503, created=now, **k3), Cell(lat=2.0, lon=2.0, new_measures=2, total_measures=4, **k2), # the lat/lon is bogus and mismatches the line above on purpose # to make sure old observations are skipped CellObservation(lat=-1.0, lon=-1.0, created=before, **k2), CellObservation(lat=-1.0, lon=-1.0, created=before, **k2), CellObservation(lat=2.002, lon=2.004, created=now, **k2), CellObservation(lat=2.002, lon=2.004, created=now, **k2), ] session.add_all(data) session.commit() result = location_update_cell.delay(min_new=1) self.assertEqual(result.get(), (2, 0)) self.check_stats( total=2, timer=['task.data.location_update_cell'], gauge=['task.data.location_update_cell.new_measures_1_100'], ) cells = session.query(Cell).all() self.assertEqual(len(cells), 2) self.assertEqual([c.new_measures for c in cells], [0, 0]) for cell in cells: if cell.cid == 4: self.assertEqual(cell.lat, 1.002) self.assertEqual(cell.lon, 1.003) elif cell.cid == 8: self.assertEqual(cell.lat, 2.001) self.assertEqual(cell.lon, 2.002)
def test_max_min_range_update(self): session = self.session k1 = dict(radio=Radio.cdma, mcc=1, mnc=2, lac=3, cid=4) data = [ Cell(lat=1.001, lon=-1.001, max_lat=1.002, min_lat=1.0, max_lon=-1.0, min_lon=-1.002, new_measures=2, total_measures=4, **k1), CellObservation(lat=1.001, lon=-1.003, **k1), CellObservation(lat=1.005, lon=-1.007, **k1), ] session.add_all(data) session.commit() result = location_update_cell.delay(min_new=1) self.assertEqual(result.get(), (1, 0)) cells = session.query(Cell).all() self.assertEqual(len(cells), 1) cell = cells[0] self.assertEqual(cell.lat, 1.002) self.assertEqual(cell.max_lat, 1.005) self.assertEqual(cell.min_lat, 1.0) self.assertEqual(cell.lon, -1.003) self.assertEqual(cell.max_lon, -1.0) self.assertEqual(cell.min_lon, -1.007) # independent calculation: the cell bounding box is # (1.000, -1.007) to (1.005, -1.000), with centroid # at (1.002, -1.003); worst distance from centroid # to any corner is 556m self.assertEqual(cell.range, 556)
def test_backup_cell_to_s3(self): session = self.db_master_session batch_size = 10 observations = [] for i in range(batch_size): observations.append(CellObservation(created=self.really_old)) session.add_all(observations) session.flush() start_id = observations[0].id blocks = schedule_cellmeasure_archival.delay(batch=batch_size).get() self.assertEquals(len(blocks), 1) block = blocks[0] self.assertEquals(block, (start_id, start_id + batch_size)) with mock_s3(): with patch.object(S3Backend, 'backup_archive', lambda x, y, z: True): write_cellmeasure_s3_backups.delay(cleanup_zip=False).get() msgs = self.heka_client.stream.msgs info_msgs = [m for m in msgs if m.type == 'oldstyle'] self.assertEquals(1, len(info_msgs)) info = info_msgs[0] fname = info.payload.split(":")[-1] myzip = ZipFile(fname) try: contents = set(myzip.namelist()) expected_contents = set(['alembic_revision.txt', 'cell_measure.csv']) self.assertEquals(expected_contents, contents) finally: myzip.close() blocks = session.query(ObservationBlock).all() self.assertEquals(len(blocks), 1) block = blocks[0] actual_sha = hashlib.sha1() actual_sha.update(open(fname, 'rb').read()) self.assertEquals(block.archive_sha, actual_sha.digest()) self.assertTrue(block.s3_key is not None) self.assertTrue('/cell_' in block.s3_key) self.assertTrue(block.archive_date is None)
def test_monitor_measures(self): session = self.db_master_session result = monitor_measures.delay().get() self.check_stats( gauge=[('table.cell_measure', 1), ('table.wifi_measure', 1)], ) self.assertEqual(result, {'cell_measure': -1, 'wifi_measure': -1}) # add some observations session.add_all([CellObservation() for i in range(3)]) session.add_all([WifiObservation() for i in range(5)]) session.flush() result = monitor_measures.delay().get() self.check_stats( gauge=[('table.cell_measure', 2), ('table.wifi_measure', 2)], ) self.assertEqual(result, {'cell_measure': 3, 'wifi_measure': 5})
def test_scan_lacs_asymmetric(self): session = self.db_master_session big = 0.1 small = big / 10 keys = dict(radio=1, mcc=1, mnc=1, lac=1) observations = [ CellObservation(lat=ctr + xd, lon=ctr + yd, cid=cell, **keys) for cell in range(6) for ctr in [(2**cell) * big] for (xd, yd) in [(small, small), (small, -small), (-small, small), (-small, -small)] ] session.add_all(observations) cells = [ Cell(lat=ctr, lon=ctr, cid=cell, new_measures=4, total_measures=1, **keys) for cell in range(6) for ctr in [(2**cell) * big] ] session.add_all(cells) session.commit() result = location_update_cell.delay(min_new=0, max_new=9999, batch=len(observations)) self.assertEqual(result.get(), (len(cells), 0)) scan_lacs.delay() lac = session.query(CellArea).filter(CellArea.lac == 1).first() # We produced a sequence of 0.02-degree-on-a-side # cell bounding boxes centered at # [0, 0.2, 0.4, 0.8, 1.6, 3.2] degrees. # So the lower-left corner is at (-0.01, -0.01) # and the upper-right corner is at (3.21, 3.21) # we should therefore see a LAC centroid at (1.05, 1.05) # with a range of 339.540m self.assertEqual(lac.lat, 1.05) self.assertEqual(lac.lon, 1.05) self.assertEqual(lac.range, 339540)
def test_scan_areas_race_with_location_update(self): session = self.session # First batch of cell observations for CID 1 keys = dict(radio=Radio.cdma, mcc=1, mnc=1, lac=1, cid=1) cell = Cell(new_measures=4, total_measures=1, **keys) observations = [ CellObservation(lat=1.0, lon=1.0, **keys), CellObservation(lat=1.0, lon=1.0, **keys), CellObservation(lat=1.0, lon=1.0, **keys), CellObservation(lat=1.0, lon=1.0, **keys), ] session.add(cell) session.add_all(observations) session.commit() # Periodic location_update_cell runs and updates CID 1 # to have a location, inserts LAC 1 with new_measures=1 # which will be picked up by the next scan_lac. result = location_update_cell.delay(min_new=1) self.assertEqual(result.get(), (1, 0)) # Second batch of cell observations for CID 2 keys['cid'] = 2 cell = Cell(new_measures=4, total_measures=1, **keys) observations = [ CellObservation(lat=1.0, lon=1.0, **keys), CellObservation(lat=1.0, lon=1.0, **keys), CellObservation(lat=1.0, lon=1.0, **keys), CellObservation(lat=1.0, lon=1.0, **keys), ] session.add(cell) session.add_all(observations) session.commit() # Periodic LAC scan runs, picking up LAC 1; this could # accidentally pick up CID 2, but it should not since it # has not had its location updated yet. If there's no # exception here, CID 2 is properly ignored. scan_areas.delay()
def test_delete_cell_observations(self): session = self.db_master_session block = ObservationBlock() block.measure_type = ObservationType.cell block.start_id = 120 block.end_id = 140 block.s3_key = 'fake_key' block.archive_sha = 'fake_sha' block.archive_date = None session.add(block) for i in range(100, 150): session.add(CellObservation(id=i, created=self.really_old)) session.commit() with patch.object(S3Backend, 'check_archive', lambda x, y, z: True): delete_cellmeasure_records.delay(batch=3).get() self.assertEquals(session.query(CellObservation).count(), 30) self.assertTrue(block.archive_date is not None)
def test_cell_histogram(self): session = self.db_master_session today = util.utcnow().date() yesterday = (today - timedelta(1)) two_days = (today - timedelta(2)) long_ago = (today - timedelta(3)) observations = [ CellObservation(lat=1.0, lon=2.0, created=today), CellObservation(lat=1.0, lon=2.0, created=today), CellObservation(lat=1.0, lon=2.0, created=yesterday), CellObservation(lat=1.0, lon=2.0, created=two_days), CellObservation(lat=1.0, lon=2.0, created=two_days), CellObservation(lat=1.0, lon=2.0, created=two_days), CellObservation(lat=1.0, lon=2.0, created=long_ago), ] session.add_all(observations) session.commit() cell_histogram.delay(ago=3).get() stats = session.query(Stat).order_by(Stat.time).all() self.assertEqual(len(stats), 1) self.assertEqual(stats[0].key, StatKey.cell) self.assertEqual(stats[0].time, long_ago) self.assertEqual(stats[0].value, 1) # fill up newer dates cell_histogram.delay(ago=2).get() cell_histogram.delay(ago=1).get() cell_histogram.delay(ago=0).get() # test duplicate execution cell_histogram.delay(ago=1).get() stats = session.query(Stat).order_by(Stat.time).all() self.assertEqual(len(stats), 4) self.assertEqual(stats[0].time, long_ago) self.assertEqual(stats[0].value, 1) self.assertEqual(stats[1].time, two_days) self.assertEqual(stats[1].value, 4) self.assertEqual(stats[2].time, yesterday) self.assertEqual(stats[2].value, 5) self.assertEqual(stats[3].time, today) self.assertEqual(stats[3].value, 7)
def test_blacklist_moving_cells(self): now = util.utcnow() long_ago = now - timedelta(days=40) session = self.session k1 = dict(radio=Radio.cdma, mcc=1, mnc=2, lac=3, cid=4) k2 = dict(radio=Radio.cdma, mcc=1, mnc=2, lac=6, cid=8) k3 = dict(radio=Radio.cdma, mcc=1, mnc=2, lac=9, cid=12) k4 = dict(radio=Radio.cdma, mcc=1, mnc=2, lac=12, cid=16) k5 = dict(radio=Radio.cdma, mcc=1, mnc=2, lac=15, cid=20) k6 = dict(radio=Radio.cdma, mcc=1, mnc=2, lac=18, cid=24) # keys k2, k3 and k4 are expected to be detected as moving data = [ # a cell with an entry but no prior position Cell(new_measures=3, total_measures=0, **k1), CellObservation(lat=1.001, lon=1.001, **k1), CellObservation(lat=1.002, lon=1.005, **k1), CellObservation(lat=1.003, lon=1.009, **k1), # a cell with a prior known position Cell(lat=2.0, lon=2.0, new_measures=2, total_measures=1, **k2), CellObservation(lat=2.0, lon=2.0, **k2), CellObservation(lat=4.0, lon=2.0, **k2), # a cell with a very different prior position Cell(lat=1.0, lon=1.0, new_measures=2, total_measures=1, **k3), CellObservation(lat=3.0, lon=3.0, **k3), CellObservation(lat=-3.0, lon=3.0, **k3), # another cell with a prior known position (and negative lat) Cell(lat=-4.0, lon=4.0, new_measures=2, total_measures=1, **k4), CellObservation(lat=-4.0, lon=4.0, **k4), CellObservation(lat=-6.0, lon=4.0, **k4), # an already blacklisted cell CellBlacklist(time=now, count=1, **k5), CellObservation(lat=5.0, lon=5.0, **k5), CellObservation(lat=8.0, lon=5.0, **k5), # a cell with an old different record we ignore, position # estimate has been updated since Cell(lat=6.0, lon=6.0, new_measures=2, total_measures=1, **k6), CellObservation(lat=6.9, lon=6.9, time=long_ago, **k6), CellObservation(lat=6.0, lon=6.0, **k6), CellObservation(lat=6.001, lon=6, **k6), ] session.add_all(data) session.commit() result = location_update_cell.delay(min_new=1) self.assertEqual(result.get(), (5, 3)) moving = [k2, k3, k4, k5] black = session.query(CellBlacklist).all() self.assertEqual(set([b.hashkey() for b in black]), set([CellBlacklist.to_hashkey(k) for k in moving])) keys = [k1, k2, k3, k4, k5, k6] observations = session.query(CellObservation).all() self.assertEqual(len(observations), 14) self.assertEqual(set([obs.hashkey() for obs in observations]), set([CellObservation.to_hashkey(k) for k in keys])) # test duplicate call result = location_update_cell.delay(min_new=1) self.assertEqual(result.get(), (0, 0)) self.check_stats( total=6, timer=[ # We made duplicate calls ('task.data.location_update_cell', 2), # One of those would've scheduled a remove_cell task ('task.data.remove_cell', 1) ], gauge=[ ('task.data.location_update_cell.new_measures_1_100', 2), ])