def add_line_of_cells_and_scan_lac(self): session = self.session big = 1.0 small = big / 10 keys = dict(radio=Radio.cdma, mcc=1, mnc=1, lac=1) observations = [ CellObservation(lat=ctr + xd, lon=ctr + yd, cid=cell, **keys) for cell in range(10) for ctr in [cell * big] for (xd, yd) in [(small, small), (small, -small), (-small, small), (-small, -small)] ] session.add_all(observations) cells = [ Cell(lat=ctr, lon=ctr, cid=cell, new_measures=4, total_measures=1, **keys) for cell in range(10) for ctr in [cell * big] ] session.add_all(cells) session.commit() result = location_update_cell.delay(min_new=0, max_new=9999, batch=len(observations)) self.assertEqual(result.get(), (len(cells), 0)) scan_areas.delay()
def test_max_min_range_update(self): session = self.db_master_session k1 = dict(radio=1, mcc=1, mnc=2, lac=3, cid=4) data = [ Cell(lat=1.001, lon=-1.001, max_lat=1.002, min_lat=1.0, max_lon=-1.0, min_lon=-1.002, new_measures=2, total_measures=4, **k1), CellMeasure(lat=1.001, lon=-1.003, **k1), CellMeasure(lat=1.005, lon=-1.007, **k1), ] session.add_all(data) session.commit() result = location_update_cell.delay(min_new=1) self.assertEqual(result.get(), (1, 0)) cells = session.query(Cell).filter(Cell.cid != CELLID_LAC).all() self.assertEqual(len(cells), 1) cell = cells[0] self.assertEqual(cell.lat, 1.002) self.assertEqual(cell.max_lat, 1.005) self.assertEqual(cell.min_lat, 1.0) self.assertEqual(cell.lon, -1.003) self.assertEqual(cell.max_lon, -1.0) self.assertEqual(cell.min_lon, -1.007) # independent calculation: the cell bounding box is # (1.000, -1.007) to (1.005, -1.000), with centroid # at (1.002, -1.003); worst distance from centroid # to any corner is 556m self.assertEqual(cell.range, 556)
def add_line_of_cells_and_scan_lac(self): session = self.session big = 1.0 small = big / 10 keys = dict(radio=Radio.cdma, mcc=1, mnc=1, lac=1) observations = [ CellObservation(lat=ctr + xd, lon=ctr + yd, cid=cell, **keys) for cell in range(10) for ctr in [cell * big] for (xd, yd) in [(small, small), (small, -small), (-small, small), (-small, -small)] ] session.add_all(observations) cells = [ Cell(lat=ctr, lon=ctr, cid=cell, new_measures=4, total_measures=1, **keys) for cell in range(10) for ctr in [cell * big] ] session.add_all(cells) session.commit() result = location_update_cell.delay(min_new=0, max_new=9999, batch=len(observations)) self.assertEqual(result.get(), (len(cells), 0)) scan_areas.delay()
def test_location_update_cell(self): now = util.utcnow() before = now - timedelta(hours=1) schema = ValidCellKeySchema() obs_factory = CellObservationFactory invalid_key = dict(lac=schema.fields['lac'].missing, cid=schema.fields['cid'].missing) cell1 = CellFactory(new_measures=3, total_measures=5) lat1, lon1 = (cell1.lat, cell1.lon) key1 = dict(lac=cell1.lac, cid=cell1.cid) obs_factory(lat=lat1, lon=lon1, created=now, **key1) obs_factory(lat=lat1 + 0.004, lon=lon1 + 0.006, created=now, **key1) obs_factory(lat=lat1 + 0.006, lon=lon1 + 0.009, created=now, **key1) # The lac, cid are invalid and should be skipped obs_factory.create_batch(2, created=now, **invalid_key) cell2 = CellFactory(lat=lat1 + 1.0, lon=lon1 + 1.0, new_measures=2, total_measures=4) lat2, lon2 = (cell2.lat, cell2.lon) key2 = dict(lac=cell2.lac, cid=cell2.cid) # the lat/lon is bogus and mismatches the line above on purpose # to make sure old observations are skipped obs_factory(lat=lat2 - 2.0, lon=lon2 - 2.0, created=before, **key2) obs_factory(lat=lat2 - 2.0, lon=lon2 - 2.0, created=before, **key2) obs_factory(lat=lat2 + 0.001, lon=lon2 + 0.002, created=now, **key2) obs_factory(lat=lat2 + 0.003, lon=lon2 + 0.006, created=now, **key2) cell3 = CellFactory(new_measures=10, total_measures=100000) lat3, lon3 = (cell3.lat, cell3.lon) obs_factory.create_batch( 10, lat=lat3 + 1.0, lon=lon3 + 1.0, **dict(lac=cell3.lac, cid=cell3.cid)) self.session.commit() result = location_update_cell.delay(min_new=1) self.assertEqual(result.get(), (3, 0)) self.check_stats( total=2, timer=['task.data.location_update_cell'], gauge=['task.data.location_update_cell.new_measures_1_100'], ) cells = self.session.query(Cell).all() self.assertEqual(len(cells), 3) self.assertEqual(set([c.new_measures for c in cells]), set([0])) for cell in cells: if cell.hashkey() == cell1.hashkey(): self.assertEqual(cell.lat, lat1 + 0.002) self.assertEqual(cell.lon, lon1 + 0.003) if cell.hashkey() == cell2.hashkey(): self.assertEqual(cell.lat, lat2 + 0.001) self.assertEqual(cell.lon, lon2 + 0.002) if cell.hashkey() == cell3.hashkey(): expected_lat = ((lat3 * 1000) + (lat3 + 1.0) * 10) / 1010 expected_lon = ((lon3 * 1000) + (lon3 + 1.0) * 10) / 1010 self.assertAlmostEqual(cell.lat, expected_lat, 7) self.assertAlmostEqual(cell.lon, expected_lon, 7)
def test_update_cell(self): now = util.utcnow() invalid_key = dict(lac=None, cid=None) observations = [] def obs_factory(**kw): obs = CellObservationFactory.create(**kw) observations.append(obs) cell1 = CellFactory(new_measures=3, total_measures=3) lat1, lon1 = (cell1.lat, cell1.lon) key1 = dict(lac=cell1.lac, cid=cell1.cid) obs_factory(lat=lat1, lon=lon1, created=now, **key1) obs_factory(lat=lat1 + 0.004, lon=lon1 + 0.006, created=now, **key1) obs_factory(lat=lat1 + 0.006, lon=lon1 + 0.009, created=now, **key1) # The lac, cid are invalid and should be skipped obs_factory(created=now, **invalid_key) obs_factory(created=now, **invalid_key) cell2 = CellFactory(lat=lat1 + 1.0, lon=lon1 + 1.0, new_measures=2, total_measures=3) lat2, lon2 = (cell2.lat, cell2.lon) key2 = dict(lac=cell2.lac, cid=cell2.cid) obs_factory(lat=lat2 + 0.001, lon=lon2 + 0.002, created=now, **key2) obs_factory(lat=lat2 + 0.003, lon=lon2 + 0.006, created=now, **key2) cell3 = CellFactory(new_measures=10, total_measures=100000) lat3, lon3 = (cell3.lat, cell3.lon) for i in range(10): obs_factory(lat=lat3 + 1.0, lon=lon3 + 1.0, **dict(lac=cell3.lac, cid=cell3.cid)) self.session.add_all(observations) self.session.commit() result = location_update_cell.delay() self.assertEqual(result.get(), (3, 0)) self.check_stats( timer=['task.data.location_update_cell'], ) cells = self.session.query(Cell).all() self.assertEqual(len(cells), 3) self.assertEqual(set([c.new_measures for c in cells]), set([0])) for cell in cells: if cell.hashkey() == cell1.hashkey(): self.assertAlmostEqual(cell.lat, lat1 + 0.001667, 6) self.assertAlmostEqual(cell.lon, lon1 + 0.0025, 6) if cell.hashkey() == cell2.hashkey(): self.assertAlmostEqual(cell.lat, lat2 + 0.0008, 6) self.assertAlmostEqual(cell.lon, lon2 + 0.0016, 6) if cell.hashkey() == cell3.hashkey(): expected_lat = ((lat3 * 1000) + (lat3 + 1.0) * 10) / 1010 expected_lon = ((lon3 * 1000) + (lon3 + 1.0) * 10) / 1010 self.assertAlmostEqual(cell.lat, expected_lat, 7) self.assertAlmostEqual(cell.lon, expected_lon, 7)
def test_location_update_cell(self): now = util.utcnow() before = now - timedelta(days=1) schema = ValidCellKeySchema() session = self.db_master_session k1 = dict(radio=1, mcc=1, mnc=2, lac=3, cid=4) k2 = dict(radio=1, mcc=1, mnc=2, lac=6, cid=8) k3 = dict(radio=1, mcc=1, mnc=2, lac=schema.fields['lac'].missing, cid=schema.fields['cid'].missing) data = [ Cell(new_measures=3, total_measures=5, **k1), CellObservation(lat=1.0, lon=1.0, created=now, **k1), CellObservation(lat=1.002, lon=1.003, created=now, **k1), CellObservation(lat=1.004, lon=1.006, created=now, **k1), # The lac, cid are invalid and should be skipped CellObservation(lat=1.5, lon=1.5, created=now, **k3), CellObservation(lat=1.502, lon=1.503, created=now, **k3), Cell(lat=2.0, lon=2.0, new_measures=2, total_measures=4, **k2), # the lat/lon is bogus and mismatches the line above on purpose # to make sure old observations are skipped CellObservation(lat=-1.0, lon=-1.0, created=before, **k2), CellObservation(lat=-1.0, lon=-1.0, created=before, **k2), CellObservation(lat=2.002, lon=2.004, created=now, **k2), CellObservation(lat=2.002, lon=2.004, created=now, **k2), ] session.add_all(data) session.commit() result = location_update_cell.delay(min_new=1) self.assertEqual(result.get(), (2, 0)) self.check_stats( total=2, timer=['task.data.location_update_cell'], gauge=['task.data.location_update_cell.new_measures_1_100'], ) cells = session.query(Cell).all() self.assertEqual(len(cells), 2) self.assertEqual([c.new_measures for c in cells], [0, 0]) for cell in cells: if cell.cid == 4: self.assertEqual(cell.lat, 1.002) self.assertEqual(cell.lon, 1.003) elif cell.cid == 8: self.assertEqual(cell.lat, 2.001) self.assertEqual(cell.lon, 2.002)
def test_location_update_cell(self): now = util.utcnow() before = now - timedelta(days=1) session = self.db_master_session k1 = dict(radio=1, mcc=1, mnc=2, lac=3, cid=4) k2 = dict(radio=1, mcc=1, mnc=2, lac=6, cid=8) k3 = dict(radio=1, mcc=1, mnc=2, lac=-1, cid=-1) data = [ Cell(new_measures=3, total_measures=5, **k1), CellMeasure(lat=1.0, lon=1.0, **k1), CellMeasure(lat=1.002, lon=1.003, **k1), CellMeasure(lat=1.004, lon=1.006, **k1), # The lac, cid are invalid and should be skipped CellMeasure(lat=1.5, lon=1.5, **k3), CellMeasure(lat=1.502, lon=1.503, **k3), Cell(lat=2.0, lon=2.0, new_measures=2, total_measures=4, **k2), # the lat/lon is bogus and mismatches the line above on purpose # to make sure old measures are skipped CellMeasure(lat=-1.0, lon=-1.0, created=before, **k2), CellMeasure(lat=-1.0, lon=-1.0, created=before, **k2), CellMeasure(lat=2.002, lon=2.004, **k2), CellMeasure(lat=2.002, lon=2.004, **k2), ] session.add_all(data) session.commit() result = location_update_cell.delay(min_new=1) self.assertEqual(result.get(), (2, 0)) self.check_stats( total=2, timer=['task.data.location_update_cell'], gauge=['task.data.location_update_cell.new_measures_1_100'], ) cells = session.query(Cell).filter(Cell.cid != CELLID_LAC).all() self.assertEqual(len(cells), 2) self.assertEqual([c.new_measures for c in cells], [0, 0]) for cell in cells: if cell.cid == 4: self.assertEqual(cell.lat, 1.002) self.assertEqual(cell.lon, 1.003) elif cell.cid == 8: self.assertEqual(cell.lat, 2.001) self.assertEqual(cell.lon, 2.002)
def test_scan_lacs_asymmetric(self): session = self.db_master_session big = 0.1 small = big / 10 keys = dict(radio=1, mcc=1, mnc=1, lac=1) measures = [ CellMeasure(lat=ctr + xd, lon=ctr + yd, cid=cell, **keys) for cell in range(6) for ctr in [(2 ** cell) * big] for (xd, yd) in [(small, small), (small, -small), (-small, small), (-small, -small)] ] session.add_all(measures) cells = [ Cell(lat=ctr, lon=ctr, cid=cell, new_measures=4, total_measures=1, **keys) for cell in range(6) for ctr in [(2 ** cell) * big] ] session.add_all(cells) session.commit() result = location_update_cell.delay(min_new=0, max_new=9999, batch=len(measures)) self.assertEqual(result.get(), (len(cells), 0)) scan_lacs.delay() lac = session.query(Cell).filter( Cell.lac == 1, Cell.cid == CELLID_LAC).first() # We produced a sequence of 0.02-degree-on-a-side # cell bounding boxes centered at # [0, 0.2, 0.4, 0.8, 1.6, 3.2] degrees. # So the lower-left corner is at (-0.01, -0.01) # and the upper-right corner is at (3.21, 3.21) # we should therefore see a LAC centroid at (1.05, 1.05) # with a range of 339.540m self.assertEqual(lac.lat, 1.05) self.assertEqual(lac.lon, 1.05) self.assertEqual(lac.range, 339540)
def test_scan_lacs_asymmetric(self): session = self.db_master_session big = 0.1 small = big / 10 keys = dict(radio=1, mcc=1, mnc=1, lac=1) measures = [ CellMeasure(lat=ctr + xd, lon=ctr + yd, cid=cell, **keys) for cell in range(6) for ctr in [(2**cell) * big] for (xd, yd) in [(small, small), (small, -small), (-small, small), (-small, -small)] ] session.add_all(measures) cells = [ Cell(lat=ctr, lon=ctr, cid=cell, new_measures=4, total_measures=1, **keys) for cell in range(6) for ctr in [(2**cell) * big] ] session.add_all(cells) session.commit() result = location_update_cell.delay(min_new=0, max_new=9999, batch=len(measures)) self.assertEqual(result.get(), (len(cells), 0)) scan_lacs.delay() lac = session.query(Cell).filter(Cell.lac == 1, Cell.cid == CELLID_LAC).first() # We produced a sequence of 0.02-degree-on-a-side # cell bounding boxes centered at # [0, 0.2, 0.4, 0.8, 1.6, 3.2] degrees. # So the lower-left corner is at (-0.01, -0.01) # and the upper-right corner is at (3.21, 3.21) # we should therefore see a LAC centroid at (1.05, 1.05) # with a range of 339.540m self.assertEqual(lac.lat, 1.05) self.assertEqual(lac.lon, 1.05) self.assertEqual(lac.range, 339540)
def test_scan_areas_race_with_location_update(self): session = self.session # First batch of cell observations for CID 1 keys = dict(radio=Radio.cdma, mcc=1, mnc=1, lac=1, cid=1) cell = Cell(new_measures=4, total_measures=1, **keys) observations = [ CellObservation(lat=1.0, lon=1.0, **keys), CellObservation(lat=1.0, lon=1.0, **keys), CellObservation(lat=1.0, lon=1.0, **keys), CellObservation(lat=1.0, lon=1.0, **keys), ] session.add(cell) session.add_all(observations) session.commit() # Periodic location_update_cell runs and updates CID 1 # to have a location, inserts LAC 1 with new_measures=1 # which will be picked up by the next scan_lac. result = location_update_cell.delay(min_new=1) self.assertEqual(result.get(), (1, 0)) # Second batch of cell observations for CID 2 keys['cid'] = 2 cell = Cell(new_measures=4, total_measures=1, **keys) observations = [ CellObservation(lat=1.0, lon=1.0, **keys), CellObservation(lat=1.0, lon=1.0, **keys), CellObservation(lat=1.0, lon=1.0, **keys), CellObservation(lat=1.0, lon=1.0, **keys), ] session.add(cell) session.add_all(observations) session.commit() # Periodic LAC scan runs, picking up LAC 1; this could # accidentally pick up CID 2, but it should not since it # has not had its location updated yet. If there's no # exception here, CID 2 is properly ignored. scan_areas.delay()
def test_scan_areas_race_with_location_update(self): session = self.session # First batch of cell observations for CID 1 keys = dict(radio=Radio.cdma, mcc=1, mnc=1, lac=1, cid=1) cell = Cell(new_measures=4, total_measures=1, **keys) observations = [ CellObservation(lat=1.0, lon=1.0, **keys), CellObservation(lat=1.0, lon=1.0, **keys), CellObservation(lat=1.0, lon=1.0, **keys), CellObservation(lat=1.0, lon=1.0, **keys), ] session.add(cell) session.add_all(observations) session.commit() # Periodic location_update_cell runs and updates CID 1 # to have a location, inserts LAC 1 with new_measures=1 # which will be picked up by the next scan_lac. result = location_update_cell.delay(min_new=1) self.assertEqual(result.get(), (1, 0)) # Second batch of cell observations for CID 2 keys['cid'] = 2 cell = Cell(new_measures=4, total_measures=1, **keys) observations = [ CellObservation(lat=1.0, lon=1.0, **keys), CellObservation(lat=1.0, lon=1.0, **keys), CellObservation(lat=1.0, lon=1.0, **keys), CellObservation(lat=1.0, lon=1.0, **keys), ] session.add(cell) session.add_all(observations) session.commit() # Periodic LAC scan runs, picking up LAC 1; this could # accidentally pick up CID 2, but it should not since it # has not had its location updated yet. If there's no # exception here, CID 2 is properly ignored. scan_areas.delay()
def test_max_min_range_update(self): session = self.db_master_session k1 = dict(radio=1, mcc=1, mnc=2, lac=3, cid=4) data = [ Cell(lat=1.001, lon=-1.001, max_lat=1.002, min_lat=1.0, max_lon=-1.0, min_lon=-1.002, new_measures=2, total_measures=4, **k1), CellMeasure(lat=1.001, lon=-1.003, **k1), CellMeasure(lat=1.005, lon=-1.007, **k1), ] session.add_all(data) session.commit() result = location_update_cell.delay(min_new=1) self.assertEqual(result.get(), (1, 0)) cells = session.query(Cell).filter(Cell.cid != CELLID_LAC).all() self.assertEqual(len(cells), 1) cell = cells[0] self.assertEqual(cell.lat, 1.002) self.assertEqual(cell.max_lat, 1.005) self.assertEqual(cell.min_lat, 1.0) self.assertEqual(cell.lon, -1.003) self.assertEqual(cell.max_lon, -1.0) self.assertEqual(cell.min_lon, -1.007) # independent calculation: the cell bounding box is # (1.000, -1.007) to (1.005, -1.000), with centroid # at (1.002, -1.003); worst distance from centroid # to any corner is 556m self.assertEqual(cell.range, 556)
def test_blacklist_temporary_and_permanent(self): session = self.db_master_session # This test simulates a cell that moves once a month, for 2 years. # The first 2 * PERMANENT_BLACKLIST_THRESHOLD (12) moves should be # temporary, forgotten after a week; after that it should be # permanently blacklisted. now = util.utcnow() # Station moves between these 4 points, all in the USA: points = [ # NYC (40.0, -74.0), # SF (37.0, -122.0), # Seattle (47.0, -122.0), # Miami (25.0, -80.0), ] N = 4 * PERMANENT_BLACKLIST_THRESHOLD for month in range(0, N): days_ago = (N - (month + 1)) * 30 time = now - timedelta(days=days_ago) time_enc = encode_datetime(time) measure = dict(id=month, radio=RADIO_TYPE['gsm'], mcc=USA_MCC, mnc=ATT_MNC, lac=456, cid=123, time=time_enc, lat=points[month % 4][0], lon=points[month % 4][1]) # insert_result is num-accepted-measures, override # utcnow to set creation date insert_result = insert_measures_cell.delay( [measure], utcnow=time_enc) # update_result is (num-stations, num-moving-stations) update_result = location_update_cell.delay(min_new=1) # Assuming PERMANENT_BLACKLIST_THRESHOLD == 6: # # 0th insert will create the station # 1st insert will create first blacklist entry, delete station # 2nd insert will recreate the station at new position # 3rd insert will update blacklist, re-delete station # 4th insert will recreate the station at new position # 5th insert will update blacklist, re-delete station # 6th insert will recreate the station at new position # ... # 11th insert will make blacklisting permanent, re-delete station # 12th insert will not recreate station # 13th insert will not recreate station # ... # 23rd insert will not recreate station bl = session.query(CellBlacklist).all() if month == 0: self.assertEqual(len(bl), 0) else: self.assertEqual(len(bl), 1) # force the blacklist back in time to whenever the # measure was supposedly inserted. bl = bl[0] bl.time = time session.add(bl) session.commit() if month < N / 2: # We still haven't exceeded the threshold, so the # measurement was admitted. self.assertEqual(insert_result.get(), 1) self.assertEqual(session.query(CellMeasure).count(), month + 1) if month % 2 == 0: # The station was (re)created. self.assertEqual(update_result.get(), (1, 0)) # One cell + one cell-LAC record should exist. self.assertEqual(session.query(Cell).count(), 2) else: # The station existed and was seen moving, # thereby activating the blacklist and deleting the cell. self.assertEqual(update_result.get(), (1, 1)) self.assertEqual(bl.count, ((month + 1) / 2)) self.assertEqual(session.query(CellBlacklist).count(), 1) self.assertEqual(session.query(Cell).count(), 0) # Try adding one more measurement 1 day later # to be sure it is dropped by the now-active blacklist. next_day = encode_datetime(time + timedelta(days=1)) measure['time'] = next_day self.assertEqual( 0, insert_measures_cell.delay([measure], utcnow=next_day).get()) else: # Blacklist has exceeded threshold, gone to "permanent" mode, # so no measures accepted, no stations seen. self.assertEqual(insert_result.get(), 0) self.assertEqual(update_result.get(), 0)
def test_blacklist_moving_cells(self): now = util.utcnow() long_ago = now - timedelta(days=40) session = self.db_master_session k1 = dict(radio=1, mcc=1, mnc=2, lac=3, cid=4) k2 = dict(radio=1, mcc=1, mnc=2, lac=6, cid=8) k3 = dict(radio=1, mcc=1, mnc=2, lac=9, cid=12) k4 = dict(radio=1, mcc=1, mnc=2, lac=12, cid=16) k5 = dict(radio=1, mcc=1, mnc=2, lac=15, cid=20) k6 = dict(radio=1, mcc=1, mnc=2, lac=18, cid=24) keys = set([CellKey(**k) for k in [k1, k2, k3, k4, k5, k6]]) # keys k2, k3 and k4 are expected to be detected as moving data = [ # a cell with an entry but no prior position Cell(new_measures=3, total_measures=0, **k1), CellMeasure(lat=1.001, lon=1.001, **k1), CellMeasure(lat=1.002, lon=1.005, **k1), CellMeasure(lat=1.003, lon=1.009, **k1), # a cell with a prior known position Cell(lat=2.0, lon=2.0, new_measures=2, total_measures=1, **k2), CellMeasure(lat=2.0, lon=2.0, **k2), CellMeasure(lat=4.0, lon=2.0, **k2), # a cell with a very different prior position Cell(lat=1.0, lon=1.0, new_measures=2, total_measures=1, **k3), CellMeasure(lat=3.0, lon=3.0, **k3), CellMeasure(lat=-3.0, lon=3.0, **k3), # another cell with a prior known position (and negative lat) Cell(lat=-4.0, lon=4.0, new_measures=2, total_measures=1, **k4), CellMeasure(lat=-4.0, lon=4.0, **k4), CellMeasure(lat=-6.0, lon=4.0, **k4), # an already blacklisted cell CellBlacklist(**k5), CellMeasure(lat=5.0, lon=5.0, **k5), CellMeasure(lat=8.0, lon=5.0, **k5), # a cell with an old different record we ignore, position # estimate has been updated since Cell(lat=6.0, lon=6.0, new_measures=2, total_measures=1, **k6), CellMeasure(lat=6.9, lon=6.9, time=long_ago, **k6), CellMeasure(lat=6.0, lon=6.0, **k6), CellMeasure(lat=6.001, lon=6, **k6), ] session.add_all(data) session.commit() result = location_update_cell.delay(min_new=1) self.assertEqual(result.get(), (5, 3)) black = session.query(CellBlacklist).all() self.assertEqual(set([to_cellkey(b) for b in black]), set([CellKey(**k) for k in [k2, k3, k4, k5]])) measures = session.query(CellMeasure).all() self.assertEqual(len(measures), 14) self.assertEqual(set([to_cellkey(m) for m in measures]), keys) # test duplicate call result = location_update_cell.delay(min_new=1) self.assertEqual(result.get(), 0) self.check_stats( total=6, timer=[ # We made duplicate calls ('task.data.location_update_cell', 2), # One of those would've scheduled a remove_cell task ('task.data.remove_cell', 1) ], gauge=[ ('task.data.location_update_cell.new_measures_1_100', 2), ])
def test_location_update_cell(self): now = util.utcnow() before = now - timedelta(hours=1) schema = ValidCellKeySchema() obs_factory = CellObservationFactory invalid_key = dict(lac=schema.fields['lac'].missing, cid=schema.fields['cid'].missing) cell1 = CellFactory(new_measures=3, total_measures=5) lat1, lon1 = (cell1.lat, cell1.lon) key1 = dict(lac=cell1.lac, cid=cell1.cid) obs_factory(lat=lat1, lon=lon1, created=now, **key1) obs_factory(lat=lat1 + 0.004, lon=lon1 + 0.006, created=now, **key1) obs_factory(lat=lat1 + 0.006, lon=lon1 + 0.009, created=now, **key1) # The lac, cid are invalid and should be skipped obs_factory.create_batch(2, created=now, **invalid_key) cell2 = CellFactory(lat=lat1 + 1.0, lon=lon1 + 1.0, new_measures=2, total_measures=4) lat2, lon2 = (cell2.lat, cell2.lon) key2 = dict(lac=cell2.lac, cid=cell2.cid) # the lat/lon is bogus and mismatches the line above on purpose # to make sure old observations are skipped obs_factory(lat=lat2 - 2.0, lon=lon2 - 2.0, created=before, **key2) obs_factory(lat=lat2 - 2.0, lon=lon2 - 2.0, created=before, **key2) obs_factory(lat=lat2 + 0.001, lon=lon2 + 0.002, created=now, **key2) obs_factory(lat=lat2 + 0.003, lon=lon2 + 0.006, created=now, **key2) cell3 = CellFactory(new_measures=10, total_measures=100000) lat3, lon3 = (cell3.lat, cell3.lon) obs_factory.create_batch(10, lat=lat3 + 1.0, lon=lon3 + 1.0, **dict(lac=cell3.lac, cid=cell3.cid)) self.session.commit() result = location_update_cell.delay(min_new=1) self.assertEqual(result.get(), (3, 0)) self.check_stats( total=2, timer=['task.data.location_update_cell'], gauge=['task.data.location_update_cell.new_measures_1_100'], ) cells = self.session.query(Cell).all() self.assertEqual(len(cells), 3) self.assertEqual(set([c.new_measures for c in cells]), set([0])) for cell in cells: if cell.hashkey() == cell1.hashkey(): self.assertEqual(cell.lat, lat1 + 0.002) self.assertEqual(cell.lon, lon1 + 0.003) if cell.hashkey() == cell2.hashkey(): self.assertEqual(cell.lat, lat2 + 0.001) self.assertEqual(cell.lon, lon2 + 0.002) if cell.hashkey() == cell3.hashkey(): expected_lat = ((lat3 * 1000) + (lat3 + 1.0) * 10) / 1010 expected_lon = ((lon3 * 1000) + (lon3 + 1.0) * 10) / 1010 self.assertAlmostEqual(cell.lat, expected_lat, 7) self.assertAlmostEqual(cell.lon, expected_lon, 7)
def test_blacklist_temporary_and_permanent(self): session = self.db_master_session # This test simulates a cell that moves once a month, for 2 years. # The first 2 * PERMANENT_BLACKLIST_THRESHOLD (12) moves should be # temporary, forgotten after a week; after that it should be # permanently blacklisted. now = util.utcnow() # Station moves between these 4 points, all in the USA: points = [ # NYC (40.0, -74.0), # SF (37.0, -122.0), # Seattle (47.0, -122.0), # Miami (25.0, -80.0), ] N = 4 * PERMANENT_BLACKLIST_THRESHOLD for month in range(0, N): days_ago = (N - (month + 1)) * 30 time = now - timedelta(days=days_ago) time_enc = encode_datetime(time) measure = dict(radio=RADIO_TYPE['gsm'], mcc=USA_MCC, mnc=ATT_MNC, lac=456, cid=123, time=time_enc, lat=points[month % 4][0], lon=points[month % 4][1]) # insert_result is num-accepted-measures, override # utcnow to set creation date insert_result = insert_measures_cell.delay([measure], utcnow=time_enc) # update_result is (num-stations, num-moving-stations) update_result = location_update_cell.delay(min_new=1) # Assuming PERMANENT_BLACKLIST_THRESHOLD == 6: # # 0th insert will create the station # 1st insert will create first blacklist entry, delete station # 2nd insert will recreate the station at new position # 3rd insert will update blacklist, re-delete station # 4th insert will recreate the station at new position # 5th insert will update blacklist, re-delete station # 6th insert will recreate the station at new position # ... # 11th insert will make blacklisting permanent, re-delete station # 12th insert will not recreate station # 13th insert will not recreate station # ... # 23rd insert will not recreate station bl = session.query(CellBlacklist).all() if month == 0: self.assertEqual(len(bl), 0) else: self.assertEqual(len(bl), 1) # force the blacklist back in time to whenever the # measure was supposedly inserted. bl = bl[0] bl.time = time session.add(bl) session.commit() if month < N / 2: # We still haven't exceeded the threshold, so the # measurement was admitted. self.assertEqual(insert_result.get(), 1) self.assertEqual(session.query(CellMeasure).count(), month + 1) if month % 2 == 0: # The station was (re)created. self.assertEqual(update_result.get(), (1, 0)) # Rescan lacs to update entries self.assertEqual(scan_lacs.delay().get(), 1) # One cell + one cell-LAC record should exist. self.assertEqual(session.query(Cell).count(), 2) else: # The station existed and was seen moving, # thereby activating the blacklist and deleting the cell. self.assertEqual(update_result.get(), (1, 1)) # Rescan lacs to delete orphaned lac entry self.assertEqual(scan_lacs.delay().get(), 1) self.assertEqual(bl.count, ((month + 1) / 2)) self.assertEqual(session.query(CellBlacklist).count(), 1) self.assertEqual(session.query(Cell).count(), 0) # Try adding one more measurement 1 day later # to be sure it is dropped by the now-active blacklist. next_day = encode_datetime(time + timedelta(days=1)) measure['time'] = next_day self.assertEqual( 0, insert_measures_cell.delay([measure], utcnow=next_day).get()) else: # Blacklist has exceeded threshold, gone to "permanent" mode, # so no measures accepted, no stations seen. self.assertEqual(insert_result.get(), 0) self.assertEqual(update_result.get(), 0)
def test_blacklist_moving_cells(self): now = util.utcnow() long_ago = now - timedelta(days=40) session = self.db_master_session k1 = dict(radio=1, mcc=1, mnc=2, lac=3, cid=4) k2 = dict(radio=1, mcc=1, mnc=2, lac=6, cid=8) k3 = dict(radio=1, mcc=1, mnc=2, lac=9, cid=12) k4 = dict(radio=1, mcc=1, mnc=2, lac=12, cid=16) k5 = dict(radio=1, mcc=1, mnc=2, lac=15, cid=20) k6 = dict(radio=1, mcc=1, mnc=2, lac=18, cid=24) keys = set([CellKey(**k) for k in [k1, k2, k3, k4, k5, k6]]) # keys k2, k3 and k4 are expected to be detected as moving data = [ # a cell with an entry but no prior position Cell(new_measures=3, total_measures=0, **k1), CellMeasure(lat=1.001, lon=1.001, **k1), CellMeasure(lat=1.002, lon=1.005, **k1), CellMeasure(lat=1.003, lon=1.009, **k1), # a cell with a prior known position Cell(lat=2.0, lon=2.0, new_measures=2, total_measures=1, **k2), CellMeasure(lat=2.0, lon=2.0, **k2), CellMeasure(lat=4.0, lon=2.0, **k2), # a cell with a very different prior position Cell(lat=1.0, lon=1.0, new_measures=2, total_measures=1, **k3), CellMeasure(lat=3.0, lon=3.0, **k3), CellMeasure(lat=-3.0, lon=3.0, **k3), # another cell with a prior known position (and negative lat) Cell(lat=-4.0, lon=4.0, new_measures=2, total_measures=1, **k4), CellMeasure(lat=-4.0, lon=4.0, **k4), CellMeasure(lat=-6.0, lon=4.0, **k4), # an already blacklisted cell CellBlacklist(**k5), CellMeasure(lat=5.0, lon=5.0, **k5), CellMeasure(lat=8.0, lon=5.0, **k5), # a cell with an old different record we ignore, position # estimate has been updated since Cell(lat=6.0, lon=6.0, new_measures=2, total_measures=1, **k6), CellMeasure(lat=6.9, lon=6.9, time=long_ago, **k6), CellMeasure(lat=6.0, lon=6.0, **k6), CellMeasure(lat=6.001, lon=6, **k6), ] session.add_all(data) session.commit() result = location_update_cell.delay(min_new=1) self.assertEqual(result.get(), (5, 3)) black = session.query(CellBlacklist).all() self.assertEqual(set([to_cellkey(b) for b in black]), set([CellKey(**k) for k in [k2, k3, k4, k5]])) measures = session.query(CellMeasure).all() self.assertEqual(len(measures), 14) self.assertEqual(set([to_cellkey(m) for m in measures]), keys) # test duplicate call result = location_update_cell.delay(min_new=1) self.assertEqual(result.get(), 0) self.check_stats( total=6, timer=[ # We made duplicate calls ('task.data.location_update_cell', 2), # One of those would've scheduled a remove_cell task ('task.data.remove_cell', 1) ], gauge=[ ('task.data.location_update_cell.new_measures_1_100', 2), ])