def add_line_of_cells_and_scan_lac(self): session = self.db_master_session big = 1.0 small = big / 10 keys = dict(radio=1, mcc=1, mnc=1, lac=1) measures = [ CellMeasure(lat=ctr + xd, lon=ctr + yd, cid=cell, **keys) for cell in range(10) for ctr in [cell * big] for (xd, yd) in [(small, small), (small, -small), (-small, small), (-small, -small)] ] session.add_all(measures) cells = [ Cell(lat=ctr, lon=ctr, cid=cell, new_measures=4, total_measures=1, **keys) for cell in range(10) for ctr in [cell * big] ] session.add_all(cells) session.commit() result = location_update_cell.delay(min_new=0, max_new=9999, batch=len(measures)) self.assertEqual(result.get(), (len(cells), 0)) scan_lacs.delay()
def test_removal_updates_lac(self): session = self.db_master_session keys = dict(radio=1, mcc=1, mnc=1, lac=1) # setup: build LAC as above self.add_line_of_cells_and_scan_lac() # confirm we got one lac = session.query(Cell).filter( Cell.lac == 1, Cell.cid == CELLID_LAC).first() self.assertEqual(lac.lat, 4.5) self.assertEqual(lac.lon, 4.5) self.assertEqual(lac.range, 723001) # Remove cells one by one checking that the LAC # changes shape along the way. steps = [ ((5.0, 5.0), 644242), ((5.5, 5.5), 565475), ((6.0, 6.0), 486721), ((6.5, 6.5), 408000), ((7.0, 7.0), 329334), ((7.5, 7.5), 250743), ((8.0, 8.0), 172249), ((8.5, 8.5), 93871), ((9.0, 9.0), 15630), ] for i in range(9): session.expire(lac) k = CellKey(cid=i, **keys) result = remove_cell.delay([k]) self.assertEqual(1, result.get()) result = scan_lacs.delay() self.assertEqual(1, result.get()) lac = session.query(Cell).filter( Cell.lac == 1, Cell.cid == CELLID_LAC).first() self.assertEqual(lac.lat, steps[i][0][0]) self.assertEqual(lac.lon, steps[i][0][1]) self.assertEqual(lac.range, steps[i][1]) # Remove final cell, check LAC is gone k = CellKey(cid=9, **keys) result = remove_cell.delay([k]) self.assertEqual(1, result.get()) result = scan_lacs.delay() self.assertEqual(0, result.get()) lac = session.query(Cell).filter( Cell.lac == 1, Cell.cid == CELLID_LAC).first() self.assertEqual(lac, None)
def test_removal_updates_lac(self): session = self.db_master_session keys = dict(radio=1, mcc=1, mnc=1, lac=1) # setup: build LAC as above self.add_line_of_cells_and_scan_lac() # confirm we got one lac = session.query(Cell).filter(Cell.lac == 1, Cell.cid == CELLID_LAC).first() self.assertEqual(lac.lat, 4.5) self.assertEqual(lac.lon, 4.5) self.assertEqual(lac.range, 723001) # Remove cells one by one checking that the LAC # changes shape along the way. steps = [ ((5.0, 5.0), 644242), ((5.5, 5.5), 565475), ((6.0, 6.0), 486721), ((6.5, 6.5), 408000), ((7.0, 7.0), 329334), ((7.5, 7.5), 250743), ((8.0, 8.0), 172249), ((8.5, 8.5), 93871), ((9.0, 9.0), 15630), ] for i in range(9): session.expire(lac) k = CellKey(cid=i, **keys) result = remove_cell.delay([k]) self.assertEqual(1, result.get()) result = scan_lacs.delay() self.assertEqual(1, result.get()) lac = session.query(Cell).filter(Cell.lac == 1, Cell.cid == CELLID_LAC).first() self.assertEqual(lac.lat, steps[i][0][0]) self.assertEqual(lac.lon, steps[i][0][1]) self.assertEqual(lac.range, steps[i][1]) # Remove final cell, check LAC is gone k = CellKey(cid=9, **keys) result = remove_cell.delay([k]) self.assertEqual(1, result.get()) result = scan_lacs.delay() self.assertEqual(1, result.get()) lac = session.query(Cell).filter(Cell.lac == 1, Cell.cid == CELLID_LAC).first() self.assertEqual(lac, None)
def test_scan_lacs_asymmetric(self): session = self.db_master_session big = 0.1 small = big / 10 keys = dict(radio=1, mcc=1, mnc=1, lac=1) measures = [ CellMeasure(lat=ctr + xd, lon=ctr + yd, cid=cell, **keys) for cell in range(6) for ctr in [(2 ** cell) * big] for (xd, yd) in [(small, small), (small, -small), (-small, small), (-small, -small)] ] session.add_all(measures) cells = [ Cell(lat=ctr, lon=ctr, cid=cell, new_measures=4, total_measures=1, **keys) for cell in range(6) for ctr in [(2 ** cell) * big] ] session.add_all(cells) session.commit() result = location_update_cell.delay(min_new=0, max_new=9999, batch=len(measures)) self.assertEqual(result.get(), (len(cells), 0)) scan_lacs.delay() lac = session.query(Cell).filter( Cell.lac == 1, Cell.cid == CELLID_LAC).first() # We produced a sequence of 0.02-degree-on-a-side # cell bounding boxes centered at # [0, 0.2, 0.4, 0.8, 1.6, 3.2] degrees. # So the lower-left corner is at (-0.01, -0.01) # and the upper-right corner is at (3.21, 3.21) # we should therefore see a LAC centroid at (1.05, 1.05) # with a range of 339.540m self.assertEqual(lac.lat, 1.05) self.assertEqual(lac.lon, 1.05) self.assertEqual(lac.range, 339540)
def test_scan_lacs_asymmetric(self): session = self.db_master_session big = 0.1 small = big / 10 keys = dict(radio=1, mcc=1, mnc=1, lac=1) measures = [ CellMeasure(lat=ctr + xd, lon=ctr + yd, cid=cell, **keys) for cell in range(6) for ctr in [(2**cell) * big] for (xd, yd) in [(small, small), (small, -small), (-small, small), (-small, -small)] ] session.add_all(measures) cells = [ Cell(lat=ctr, lon=ctr, cid=cell, new_measures=4, total_measures=1, **keys) for cell in range(6) for ctr in [(2**cell) * big] ] session.add_all(cells) session.commit() result = location_update_cell.delay(min_new=0, max_new=9999, batch=len(measures)) self.assertEqual(result.get(), (len(cells), 0)) scan_lacs.delay() lac = session.query(Cell).filter(Cell.lac == 1, Cell.cid == CELLID_LAC).first() # We produced a sequence of 0.02-degree-on-a-side # cell bounding boxes centered at # [0, 0.2, 0.4, 0.8, 1.6, 3.2] degrees. # So the lower-left corner is at (-0.01, -0.01) # and the upper-right corner is at (3.21, 3.21) # we should therefore see a LAC centroid at (1.05, 1.05) # with a range of 339.540m self.assertEqual(lac.lat, 1.05) self.assertEqual(lac.lon, 1.05) self.assertEqual(lac.range, 339540)
def test_scan_lacs_race_with_location_update(self): session = self.db_master_session # First batch of cell measurements for CID 1 keys = dict(radio=1, mcc=1, mnc=1, lac=1, cid=1) cell = Cell(new_measures=4, total_measures=1, **keys) measures = [ CellMeasure(lat=1.0, lon=1.0, **keys), CellMeasure(lat=1.0, lon=1.0, **keys), CellMeasure(lat=1.0, lon=1.0, **keys), CellMeasure(lat=1.0, lon=1.0, **keys), ] session.add(cell) session.add_all(measures) session.commit() # Periodic location_update_cell runs and updates CID 1 # to have a location, inserts LAC 1 with new_measures=1 # which will be picked up by the next scan_lac. result = location_update_cell.delay(min_new=1) self.assertEqual(result.get(), (1, 0)) # Second batch of cell measurements for CID 2 keys['cid'] = 2 cell = Cell(new_measures=4, total_measures=1, **keys) measures = [ CellMeasure(lat=1.0, lon=1.0, **keys), CellMeasure(lat=1.0, lon=1.0, **keys), CellMeasure(lat=1.0, lon=1.0, **keys), CellMeasure(lat=1.0, lon=1.0, **keys), ] session.add(cell) session.add_all(measures) session.commit() # Periodic LAC scan runs, picking up LAC 1; this could # accidentally pick up CID 2, but it should not since it # has not had its location updated yet. If there's no # exception here, CID 2 is properly ignored. scan_lacs.delay()
def test_scan_lacs_remove(self): session = self.db_master_session redis_client = self.redis_client # create an orphaned lac entry key = dict(radio=1, mcc=1, mnc=1, lac=1, cid=CELLID_LAC) session.add(Cell(**key)) session.flush() enqueue_lacs(session, redis_client, [CellKey(**key)]) # after scanning the orphaned record gets removed self.assertEqual(scan_lacs.delay().get(), 1) lacs = session.query(Cell).filter(Cell.cid == CELLID_LAC).all() self.assertEqual(lacs, [])
def test_scan_lacs_remove(self): session = self.db_master_session redis_client = self.redis_client # create an orphaned lac entry key = dict(radio=1, mcc=1, mnc=1, lac=1) session.add(CellArea(**key)) session.flush() enqueue_lacs(session, redis_client, [CellArea.to_hashkey(key)], UPDATE_KEY['cell_lac']) # after scanning the orphaned record gets removed self.assertEqual(scan_lacs.delay().get(), 1) lacs = session.query(CellArea).all() self.assertEqual(lacs, [])
def test_scan_lacs_empty(self): # test tasks with an empty queue self.assertEqual(scan_lacs.delay().get(), 0) self.check_expected_heka_messages(sentry=[('msg', RAVEN_ERROR, 0)])
def test_blacklist_temporary_and_permanent(self): session = self.db_master_session # This test simulates a cell that moves once a month, for 2 years. # The first 2 * PERMANENT_BLACKLIST_THRESHOLD (12) moves should be # temporary, forgotten after a week; after that it should be # permanently blacklisted. now = util.utcnow() # Station moves between these 4 points, all in the USA: points = [ # NYC (40.0, -74.0), # SF (37.0, -122.0), # Seattle (47.0, -122.0), # Miami (25.0, -80.0), ] N = 4 * PERMANENT_BLACKLIST_THRESHOLD for month in range(0, N): days_ago = (N - (month + 1)) * 30 time = now - timedelta(days=days_ago) time_enc = encode_datetime(time) measure = dict(radio=RADIO_TYPE['gsm'], mcc=USA_MCC, mnc=ATT_MNC, lac=456, cid=123, time=time_enc, lat=points[month % 4][0], lon=points[month % 4][1]) # insert_result is num-accepted-measures, override # utcnow to set creation date insert_result = insert_measures_cell.delay([measure], utcnow=time_enc) # update_result is (num-stations, num-moving-stations) update_result = location_update_cell.delay(min_new=1) # Assuming PERMANENT_BLACKLIST_THRESHOLD == 6: # # 0th insert will create the station # 1st insert will create first blacklist entry, delete station # 2nd insert will recreate the station at new position # 3rd insert will update blacklist, re-delete station # 4th insert will recreate the station at new position # 5th insert will update blacklist, re-delete station # 6th insert will recreate the station at new position # ... # 11th insert will make blacklisting permanent, re-delete station # 12th insert will not recreate station # 13th insert will not recreate station # ... # 23rd insert will not recreate station bl = session.query(CellBlacklist).all() if month == 0: self.assertEqual(len(bl), 0) else: self.assertEqual(len(bl), 1) # force the blacklist back in time to whenever the # measure was supposedly inserted. bl = bl[0] bl.time = time session.add(bl) session.commit() if month < N / 2: # We still haven't exceeded the threshold, so the # measurement was admitted. self.assertEqual(insert_result.get(), 1) self.assertEqual(session.query(CellMeasure).count(), month + 1) if month % 2 == 0: # The station was (re)created. self.assertEqual(update_result.get(), (1, 0)) # Rescan lacs to update entries self.assertEqual(scan_lacs.delay().get(), 1) # One cell + one cell-LAC record should exist. self.assertEqual(session.query(Cell).count(), 2) else: # The station existed and was seen moving, # thereby activating the blacklist and deleting the cell. self.assertEqual(update_result.get(), (1, 1)) # Rescan lacs to delete orphaned lac entry self.assertEqual(scan_lacs.delay().get(), 1) self.assertEqual(bl.count, ((month + 1) / 2)) self.assertEqual(session.query(CellBlacklist).count(), 1) self.assertEqual(session.query(Cell).count(), 0) # Try adding one more measurement 1 day later # to be sure it is dropped by the now-active blacklist. next_day = encode_datetime(time + timedelta(days=1)) measure['time'] = next_day self.assertEqual( 0, insert_measures_cell.delay([measure], utcnow=next_day).get()) else: # Blacklist has exceeded threshold, gone to "permanent" mode, # so no measures accepted, no stations seen. self.assertEqual(insert_result.get(), 0) self.assertEqual(update_result.get(), 0)
def test_scan_lacs_empty(self): # test tasks with an empty queue self.assertEqual(scan_lacs.delay().get(), 0) self.check_expected_heka_messages( sentry=[('msg', RAVEN_ERROR, 0)] )
def test_blacklist_temporary_and_permanent(self): session = self.db_master_session # This test simulates a cell that moves once a month, for 2 years. # The first 2 * PERMANENT_BLACKLIST_THRESHOLD (12) moves should be # temporary, forgotten after a week; after that it should be # permanently blacklisted. now = util.utcnow() # Station moves between these 4 points, all in the USA: points = [ # NYC (40.0, -74.0), # SF (37.0, -122.0), # Seattle (47.0, -122.0), # Miami (25.0, -80.0), ] N = 4 * PERMANENT_BLACKLIST_THRESHOLD for month in range(0, N): days_ago = (N - (month + 1)) * 30 time = now - timedelta(days=days_ago) time_enc = encode_datetime(time) measure = dict(radio=RADIO_TYPE['gsm'], mcc=USA_MCC, mnc=ATT_MNC, lac=456, cid=123, time=time_enc, lat=points[month % 4][0], lon=points[month % 4][1]) # insert_result is num-accepted-measures, override # utcnow to set creation date insert_result = insert_measures_cell.delay( [measure], utcnow=time_enc) # update_result is (num-stations, num-moving-stations) update_result = location_update_cell.delay(min_new=1) # Assuming PERMANENT_BLACKLIST_THRESHOLD == 6: # # 0th insert will create the station # 1st insert will create first blacklist entry, delete station # 2nd insert will recreate the station at new position # 3rd insert will update blacklist, re-delete station # 4th insert will recreate the station at new position # 5th insert will update blacklist, re-delete station # 6th insert will recreate the station at new position # ... # 11th insert will make blacklisting permanent, re-delete station # 12th insert will not recreate station # 13th insert will not recreate station # ... # 23rd insert will not recreate station bl = session.query(CellBlacklist).all() if month == 0: self.assertEqual(len(bl), 0) else: self.assertEqual(len(bl), 1) # force the blacklist back in time to whenever the # measure was supposedly inserted. bl = bl[0] bl.time = time session.add(bl) session.commit() if month < N / 2: # We still haven't exceeded the threshold, so the # measurement was admitted. self.assertEqual(insert_result.get(), 1) self.assertEqual(session.query(CellMeasure).count(), month + 1) if month % 2 == 0: # The station was (re)created. self.assertEqual(update_result.get(), (1, 0)) # Rescan lacs to update entries self.assertEqual(scan_lacs.delay().get(), 1) # One cell + one cell-LAC record should exist. self.assertEqual(session.query(Cell).count(), 2) else: # The station existed and was seen moving, # thereby activating the blacklist and deleting the cell. self.assertEqual(update_result.get(), (1, 1)) # Rescan lacs to delete orphaned lac entry self.assertEqual(scan_lacs.delay().get(), 1) self.assertEqual(bl.count, ((month + 1) / 2)) self.assertEqual(session.query(CellBlacklist).count(), 1) self.assertEqual(session.query(Cell).count(), 0) # Try adding one more measurement 1 day later # to be sure it is dropped by the now-active blacklist. next_day = encode_datetime(time + timedelta(days=1)) measure['time'] = next_day self.assertEqual( 0, insert_measures_cell.delay([measure], utcnow=next_day).get()) else: # Blacklist has exceeded threshold, gone to "permanent" mode, # so no measures accepted, no stations seen. self.assertEqual(insert_result.get(), 0) self.assertEqual(update_result.get(), 0)