def test_backfill_cell_location_update(self): from ichnaea.tasks import backfill_cell_location_update session = self.db_master_session k1 = dict(radio=1, mcc=1, mnc=2, lac=3, cid=4) data = [ Cell(lat=10010000, lon=10010000, new_measures=0, total_measures=1, **k1), CellMeasure(lat=10000000, lon=10000000, **k1), CellMeasure(lat=10050000, lon=10080000, **k1), ] session.add_all(data) session.commit() query = session.query(CellMeasure.id) cm_ids = [x[0] for x in query.all()] # TODO: refactor this to be constants in the method new_measures = [((1, 1, 2, 3, 4), cm_ids)] result = backfill_cell_location_update.delay(new_measures) self.assertEqual(result.get(), (1, 0)) cells = session.query(Cell).filter(Cell.cid != CELLID_LAC).all() self.assertEqual(len(cells), 1) cell = cells[0] self.assertEqual(cell.lat, 10020000) self.assertEqual(cell.lon, 10030000) self.assertEqual(cell.new_measures, 0) self.assertEqual(cell.total_measures, 3)
def test_cell_max_min_range_update(self): from ichnaea.tasks import cell_location_update session = self.db_master_session k1 = dict(radio=1, mcc=1, mnc=2, lac=3, cid=4) data = [ Cell(lat=10010000, lon=-10010000, max_lat=10020000, min_lat=10000000, max_lon=-10000000, min_lon=-10020000, new_measures=2, total_measures=4, **k1), CellMeasure(lat=10010000, lon=-10030000, **k1), CellMeasure(lat=10050000, lon=-10070000, **k1), ] session.add_all(data) session.commit() result = cell_location_update.delay(min_new=1) self.assertEqual(result.get(), (1, 0)) cells = session.query(Cell).all() self.assertEqual(len(cells), 1) cell = cells[0] self.assertEqual(cell.lat, 10020000) self.assertEqual(cell.max_lat, 10050000) self.assertEqual(cell.min_lat, 10000000) self.assertEqual(cell.lon, -10030000) self.assertEqual(cell.max_lon, -10000000) self.assertEqual(cell.min_lon, -10070000) # independent calculation: the cell bounding box is # (1.000, -1.007) to (1.005, -1.000), and distance # between those is 956.43m, int(round(dist/2.0)) is 478m self.assertEqual(cell.range, 478)
def test_unthrottle_cell_measures(self): session = self.db_master_session block = MeasureBlock() block.measure_type = MEASURE_TYPE_CODE['cell'] block.start_id = 120 block.end_id = 140 block.s3_key = 'fake_key' block.archive_sha = 'fake_sha' block.archive_date = None session.add(block) gsm = RADIO_TYPE['gsm'] k = dict(radio=gsm, mcc=1, mnc=2, lac=4, lat=1.0, lon=1.0) for i in range(100, 150): session.add(CellMeasure(id=i, cid=i, created=self.really_old, **k)) session.add(Cell(total_measures=11000, cid=i, **k)) session.commit() with patch.object(S3Backend, 'check_archive', lambda x, y, z: True): delete_cellmeasure_records.delay(batch=3).get() cell_unthrottle_measures.delay(10000, 1000).get() cells = session.query(Cell).all() self.assertEquals(len(cells), 50) for cell in cells: if 120 <= cell.cid and cell.cid < 140: self.assertEquals(cell.total_measures, 0) else: self.assertEquals(cell.total_measures, 1) self.check_stats(counter=['items.cell_unthrottled'])
def test_schedule_cell_measures(self): session = self.db_master_session blocks = schedule_cellmeasure_archival.delay(batch=1).get() self.assertEquals(len(blocks), 0) measures = [] for i in range(20): measures.append(CellMeasure(created=self.really_old)) session.add_all(measures) session.flush() start_id = measures[0].id blocks = schedule_cellmeasure_archival.delay(batch=15).get() self.assertEquals(len(blocks), 1) block = blocks[0] self.assertEquals(block, (start_id, start_id + 15)) blocks = schedule_cellmeasure_archival.delay(batch=6).get() self.assertEquals(len(blocks), 0) blocks = schedule_cellmeasure_archival.delay(batch=5).get() self.assertEquals(len(blocks), 1) block = blocks[0] self.assertEquals(block, (start_id + 15, start_id + 20)) blocks = schedule_cellmeasure_archival.delay(batch=1).get() self.assertEquals(len(blocks), 0)
def add_line_of_cells_and_scan_lac(self): session = self.db_master_session big = 1.0 small = big / 10 keys = dict(radio=1, mcc=1, mnc=1, lac=1) measures = [ CellMeasure(lat=ctr + xd, lon=ctr + yd, cid=cell, **keys) for cell in range(10) for ctr in [cell * big] for (xd, yd) in [(small, small), (small, -small), (-small, small), (-small, -small)] ] session.add_all(measures) cells = [ Cell(lat=ctr, lon=ctr, cid=cell, new_measures=4, total_measures=1, **keys) for cell in range(10) for ctr in [cell * big] ] session.add_all(cells) session.commit() result = location_update_cell.delay(min_new=0, max_new=9999, batch=len(measures)) self.assertEqual(result.get(), (len(cells), 0)) scan_lacs.delay()
def create_cell_measure(utcnow, entry): # skip records with missing or invalid mcc or mnc if 'mcc' not in entry or entry['mcc'] < 1 or entry['mcc'] > 999: return if 'mnc' not in entry or entry['mnc'] < 0 or entry['mnc'] > 32767: return # Skip CDMA towers missing lac or cid (no psc on CDMA exists to # backfill using inference) if entry.get('radio', -1) == 1 and \ (entry.get('lac', -1) < 0 or entry.get('cid', -1) < 0): return # some phones send maxint32 to signal "unknown" # ignore anything above the maximum valid values if 'lac' not in entry or entry['lac'] < 0 or entry['lac'] > 65535: entry['lac'] = -1 if 'cid' not in entry or entry['cid'] < 0 or entry['cid'] > 268435455: entry['cid'] = -1 if 'psc' not in entry or entry['psc'] < 0 or entry['psc'] > 512: entry['psc'] = -1 # Treat the lac=0, cid=65535 combination as unspecified values if entry['lac'] == 0 and entry['cid'] == 65535: entry['lac'] = -1 entry['cid'] = -1 # Must have LAC+CID or PSC if (entry['lac'] == -1 or entry['cid'] == -1) and entry['psc'] == -1: return # make sure fields stay within reasonable bounds if 'asu' not in entry or entry['asu'] < 0 or entry['asu'] > 100: entry['asu'] = -1 if 'signal' not in entry or entry['signal'] < -200 or entry['signal'] > -1: entry['signal'] = 0 if 'ta' not in entry or entry['ta'] < 0 or entry['ta'] > 100: entry['ta'] = 0 return CellMeasure( measure_id=entry.get('measure_id'), created=utcnow, lat=entry['lat'], lon=entry['lon'], time=decode_datetime(entry.get('time', '')), accuracy=entry.get('accuracy', 0), altitude=entry.get('altitude', 0), altitude_accuracy=entry.get('altitude_accuracy', 0), radio=entry.get('radio', -1), mcc=entry.get('mcc', -1), mnc=entry.get('mnc', -1), lac=entry.get('lac', -1), cid=entry.get('cid', -1), psc=entry.get('psc', -1), asu=entry.get('asu', -1), signal=entry.get('signal', 0), ta=entry.get('ta', 0), )
def test_location_update_cell(self): now = util.utcnow() before = now - timedelta(days=1) schema = ValidCellBaseSchema() session = self.db_master_session k1 = dict(radio=1, mcc=1, mnc=2, lac=3, cid=4) k2 = dict(radio=1, mcc=1, mnc=2, lac=6, cid=8) k3 = dict(radio=1, mcc=1, mnc=2, lac=schema.fields['lac'].missing, cid=schema.fields['cid'].missing) data = [ Cell(new_measures=3, total_measures=5, **k1), CellMeasure(lat=1.0, lon=1.0, **k1), CellMeasure(lat=1.002, lon=1.003, **k1), CellMeasure(lat=1.004, lon=1.006, **k1), # The lac, cid are invalid and should be skipped CellMeasure(lat=1.5, lon=1.5, **k3), CellMeasure(lat=1.502, lon=1.503, **k3), Cell(lat=2.0, lon=2.0, new_measures=2, total_measures=4, **k2), # the lat/lon is bogus and mismatches the line above on purpose # to make sure old measures are skipped CellMeasure(lat=-1.0, lon=-1.0, created=before, **k2), CellMeasure(lat=-1.0, lon=-1.0, created=before, **k2), CellMeasure(lat=2.002, lon=2.004, **k2), CellMeasure(lat=2.002, lon=2.004, **k2), ] session.add_all(data) session.commit() result = location_update_cell.delay(min_new=1) self.assertEqual(result.get(), (2, 0)) self.check_stats( total=2, timer=['task.data.location_update_cell'], gauge=['task.data.location_update_cell.new_measures_1_100'], ) cells = session.query(Cell).all() self.assertEqual(len(cells), 2) self.assertEqual([c.new_measures for c in cells], [0, 0]) for cell in cells: if cell.cid == 4: self.assertEqual(cell.lat, 1.002) self.assertEqual(cell.lon, 1.003) elif cell.cid == 8: self.assertEqual(cell.lat, 2.001) self.assertEqual(cell.lon, 2.002)
def process_cell(entries, measure): result = [] cells = [] for entry in entries: cell = CellMeasure( measure_id=measure.id, created=measure.created, lat=measure.lat, lon=measure.lon, time=measure.time, accuracy=measure.accuracy, altitude=measure.altitude, altitude_accuracy=measure.altitude_accuracy, mcc=entry['mcc'], mnc=entry['mnc'], lac=entry['lac'], cid=entry['cid'], psc=entry['psc'], asu=entry['asu'], signal=entry['signal'], ta=entry['ta'], ) # use more specific cell type or fall back to less precise measure if entry['radio']: cell.radio = RADIO_TYPE.get(entry['radio'], -1) else: cell.radio = measure.radio cells.append(cell) result.append(entry) return (cells, result)
def create_cell_measure(utcnow, entry): # creates a dict.copy() avoiding test leaks reusing the same # entries for multiple task calls entry = normalized_cell_measure_dict(entry) if entry is None: return None # add creation date entry['created'] = utcnow # decode from JSON compatible format entry['report_id'] = uuid.UUID(hex=entry['report_id']).bytes entry['time'] = decode_datetime(entry['time']) return CellMeasure(**entry)
def test_cell_max_min_range_update(self): from ichnaea.tasks import cell_location_update session = self.db_master_session k1 = dict(radio=1, mcc=1, mnc=2, lac=3, cid=4) data = [ Cell(lat=10010000, lon=-10010000, max_lat=10020000, min_lat=10000000, max_lon=-10000000, min_lon=-10020000, new_measures=2, total_measures=4, **k1), CellMeasure(lat=10010000, lon=-10030000, **k1), CellMeasure(lat=10050000, lon=-10070000, **k1), ] session.add_all(data) session.commit() result = cell_location_update.delay(min_new=1) self.assertEqual(result.get(), (1, 0)) cells = session.query(Cell).filter(Cell.cid != CELLID_LAC).all() self.assertEqual(len(cells), 1) cell = cells[0] self.assertEqual(cell.lat, 10020000) self.assertEqual(cell.max_lat, 10050000) self.assertEqual(cell.min_lat, 10000000) self.assertEqual(cell.lon, -10030000) self.assertEqual(cell.max_lon, -10000000) self.assertEqual(cell.min_lon, -10070000) # independent calculation: the cell bounding box is # (1.000, -1.007) to (1.005, -1.000), with centroid # at (1.002, -1.003); worst distance from centroid # to any corner is 556m self.assertEqual(cell.range, 556)
def test_cell_location_update(self): from ichnaea.tasks import cell_location_update now = datetime.utcnow() before = now - timedelta(days=1) session = self.db_master_session k1 = dict(radio=1, mcc=1, mnc=2, lac=3, cid=4) k2 = dict(radio=1, mcc=1, mnc=2, lac=6, cid=8) k3 = dict(radio=1, mcc=1, mnc=2, lac=-1, cid=-1) data = [ Cell(new_measures=3, total_measures=5, **k1), CellMeasure(lat=10000000, lon=10000000, **k1), CellMeasure(lat=10020000, lon=10030000, **k1), CellMeasure(lat=10040000, lon=10060000, **k1), # The lac, cid are invalid and should be skipped CellMeasure(lat=15000000, lon=15000000, **k3), CellMeasure(lat=15020000, lon=15030000, **k3), Cell(lat=20000000, lon=20000000, new_measures=2, total_measures=4, **k2), # the lat/lon is bogus and mismatches the line above on purpose # to make sure old measures are skipped CellMeasure(lat=-10000000, lon=-10000000, created=before, **k2), CellMeasure(lat=-10000000, lon=-10000000, created=before, **k2), CellMeasure(lat=20020000, lon=20040000, **k2), CellMeasure(lat=20020000, lon=20040000, **k2), ] session.add_all(data) session.commit() result = cell_location_update.delay(min_new=1) self.assertEqual(result.get(), (2, 0)) cells = session.query(Cell).filter(Cell.cid != CELLID_LAC).all() self.assertEqual(len(cells), 2) self.assertEqual([c.new_measures for c in cells], [0, 0]) for cell in cells: if cell.cid == 4: self.assertEqual(cell.lat, 10020000) self.assertEqual(cell.lon, 10030000) elif cell.cid == 8: self.assertEqual(cell.lat, 20010000) self.assertEqual(cell.lon, 20020000)
def test_monitor_measures(self): session = self.db_master_session result = monitor_measures.delay().get() self.check_stats(gauge=[('table.cell_measure', 1), ('table.wifi_measure', 1)], ) self.assertEqual(result, {'cell_measure': -1, 'wifi_measure': -1}) # add some measures session.add_all([CellMeasure() for i in range(3)]) session.add_all([WifiMeasure() for i in range(5)]) session.flush() result = monitor_measures.delay().get() self.check_stats(gauge=[('table.cell_measure', 2), ('table.wifi_measure', 2)], ) self.assertEqual(result, {'cell_measure': 3, 'wifi_measure': 5})
def test_backup_cell_to_s3(self): session = self.db_master_session batch_size = 10 measures = [] for i in range(batch_size): measures.append(CellMeasure(created=self.really_old)) session.add_all(measures) session.flush() start_id = measures[0].id blocks = schedule_cellmeasure_archival.delay(batch=batch_size).get() self.assertEquals(len(blocks), 1) block = blocks[0] self.assertEquals(block, (start_id, start_id + batch_size)) with mock_s3(): with patch.object(S3Backend, 'backup_archive', lambda x, y, z: True): write_cellmeasure_s3_backups.delay(cleanup_zip=False).get() msgs = self.heka_client.stream.msgs info_msgs = [m for m in msgs if m.type == 'oldstyle'] self.assertEquals(1, len(info_msgs)) info = info_msgs[0] fname = info.payload.split(":")[-1] myzip = ZipFile(fname) try: contents = set(myzip.namelist()) expected_contents = set( ['alembic_revision.txt', 'cell_measure.csv']) self.assertEquals(expected_contents, contents) finally: myzip.close() blocks = session.query(MeasureBlock).all() self.assertEquals(len(blocks), 1) block = blocks[0] actual_sha = hashlib.sha1() actual_sha.update(open(fname, 'rb').read()) self.assertEquals(block.archive_sha, actual_sha.digest()) self.assertTrue(block.s3_key is not None) self.assertTrue('/cell_' in block.s3_key) self.assertTrue(block.archive_date is None)
def test_cell_lac_asymmetric(self): from ichnaea.tasks import cell_location_update, scan_lacs session = self.db_master_session big = 1000000 small = big / 10 keys = dict(radio=1, mcc=1, mnc=1, lac=1) measures = [ CellMeasure(lat=ctr + xd, lon=ctr + yd, cid=cell, **keys) for cell in range(0, 6) for ctr in [(2**cell) * big] for (xd, yd) in [(small, small), (small, -small), (-small, small), (-small, -small)] ] session.add_all(measures) cells = [ Cell(lat=ctr, lon=ctr, cid=cell, new_measures=4, total_measures=1, **keys) for cell in range(0, 6) for ctr in [(2**cell) * big] ] session.add_all(cells) session.commit() result = cell_location_update.delay(min_new=0, max_new=9999, batch=len(measures)) self.assertEqual(result.get(), (len(cells), 0)) scan_lacs.delay() lac = session.query(Cell).filter(Cell.lac == 1, Cell.cid == CELLID_LAC).first() # We produced a sequence of 0.02-degree-on-a-side # cell bounding boxes centered at # [0, 0.2, 0.4, 0.8, 1.6, 3.2] degrees. # So the lower-left corner is at (-0.01, -0.01) # and the upper-right corner is at (3.21, 3.21) # we should therefore see a LAC centroid at (1.05, 1.05) # with a range of 339.540m self.assertEqual(lac.lat, 10500000) self.assertEqual(lac.lon, 10500000) self.assertEqual(lac.range, 339540)
def test_unique_cell_histogram(self): from ichnaea.content.tasks import unique_cell_histogram session = self.db_master_session today = datetime.utcnow().date() one_day = (today - timedelta(1)) two_days = (today - timedelta(2)) long_ago = (today - timedelta(40)) measures = [ CellMeasure(created=long_ago, radio=0, mcc=1, mnc=2, lac=3, cid=4), CellMeasure(created=two_days, radio=2, mcc=1, mnc=2, lac=3, cid=4), CellMeasure(created=two_days, radio=0, mcc=1, mnc=2, lac=3, cid=4), CellMeasure(created=two_days, radio=0, mcc=2, mnc=2, lac=3, cid=4), CellMeasure(created=one_day, radio=0, mcc=2, mnc=2, lac=3, cid=5), CellMeasure(created=today, radio=0, mcc=1, mnc=3, lac=3, cid=4), CellMeasure(created=today, radio=0, mcc=1, mnc=2, lac=4, cid=4), ] session.add_all(measures) session.commit() result = unique_cell_histogram.delay(ago=40) self.assertEqual(result.get(), 1) stats = session.query(Stat).order_by(Stat.time).all() self.assertEqual(len(stats), 1) self.assertEqual(stats[0].key, STAT_TYPE['unique_cell']) self.assertEqual(stats[0].time, long_ago) self.assertEqual(stats[0].value, 1) # fill up newer dates unique_cell_histogram.delay(ago=2).get() unique_cell_histogram.delay(ago=1).get() unique_cell_histogram.delay(ago=0).get() stats = session.query(Stat).order_by(Stat.time).all() self.assertEqual(len(stats), 4) self.assertEqual(stats[0].time, long_ago) self.assertEqual(stats[0].value, 1) self.assertEqual(stats[1].time, two_days) self.assertEqual(stats[1].value, 3) self.assertEqual(stats[2].time, one_day) self.assertEqual(stats[2].value, 4) self.assertEqual(stats[3].time, today) self.assertEqual(stats[3].value, 6) # test duplicate execution result = unique_cell_histogram.delay() self.assertEqual(result.get(), 0)
def test_cell_histogram(self): from ichnaea.content.tasks import cell_histogram session = self.db_master_session today = datetime.utcnow().date() yesterday = (today - timedelta(1)) two_days = (today - timedelta(2)) long_ago = (today - timedelta(40)) measures = [ CellMeasure(lat=10000000, lon=20000000, created=today), CellMeasure(lat=10000000, lon=20000000, created=today), CellMeasure(lat=10000000, lon=20000000, created=yesterday), CellMeasure(lat=10000000, lon=20000000, created=two_days), CellMeasure(lat=10000000, lon=20000000, created=two_days), CellMeasure(lat=10000000, lon=20000000, created=two_days), CellMeasure(lat=10000000, lon=20000000, created=long_ago), ] session.add_all(measures) session.commit() cell_histogram.delay(ago=40).get() stats = session.query(Stat).order_by(Stat.time).all() self.assertEqual(len(stats), 1) self.assertEqual(stats[0].key, STAT_TYPE['cell']) self.assertEqual(stats[0].time, long_ago) self.assertEqual(stats[0].value, 1) # fill up newer dates cell_histogram.delay(ago=2).get() cell_histogram.delay(ago=1).get() cell_histogram.delay(ago=0).get() stats = session.query(Stat).order_by(Stat.time).all() self.assertEqual(len(stats), 4) self.assertEqual(stats[0].time, long_ago) self.assertEqual(stats[0].value, 1) self.assertEqual(stats[1].time, two_days) self.assertEqual(stats[1].value, 4) self.assertEqual(stats[2].time, yesterday) self.assertEqual(stats[2].value, 5) self.assertEqual(stats[3].time, today) self.assertEqual(stats[3].value, 7) # test duplicate execution result = cell_histogram.delay(ago=1) self.assertEqual(result.get(), 0)
def create_cell_measure(measure_data, entry): return CellMeasure( measure_id=measure_data['id'], created=decode_datetime(measure_data.get('created', '')), lat=measure_data['lat'], lon=measure_data['lon'], time=decode_datetime(measure_data.get('time', '')), accuracy=measure_data.get('accuracy', 0), altitude=measure_data.get('altitude', 0), altitude_accuracy=measure_data.get('altitude_accuracy', 0), mcc=entry['mcc'], mnc=entry['mnc'], lac=entry.get('lac', 0), cid=entry.get('cid', 0), psc=entry.get('psc', 0), asu=entry.get('asu', 0), signal=entry.get('signal', 0), ta=entry.get('ta', 0), )
def test_scan_lacs_race_with_location_update(self): session = self.db_master_session # First batch of cell measurements for CID 1 keys = dict(radio=1, mcc=1, mnc=1, lac=1, cid=1) cell = Cell(new_measures=4, total_measures=1, **keys) measures = [ CellMeasure(lat=1.0, lon=1.0, **keys), CellMeasure(lat=1.0, lon=1.0, **keys), CellMeasure(lat=1.0, lon=1.0, **keys), CellMeasure(lat=1.0, lon=1.0, **keys), ] session.add(cell) session.add_all(measures) session.commit() # Periodic location_update_cell runs and updates CID 1 # to have a location, inserts LAC 1 with new_measures=1 # which will be picked up by the next scan_lac. result = location_update_cell.delay(min_new=1) self.assertEqual(result.get(), (1, 0)) # Second batch of cell measurements for CID 2 keys['cid'] = 2 cell = Cell(new_measures=4, total_measures=1, **keys) measures = [ CellMeasure(lat=1.0, lon=1.0, **keys), CellMeasure(lat=1.0, lon=1.0, **keys), CellMeasure(lat=1.0, lon=1.0, **keys), CellMeasure(lat=1.0, lon=1.0, **keys), ] session.add(cell) session.add_all(measures) session.commit() # Periodic LAC scan runs, picking up LAC 1; this could # accidentally pick up CID 2, but it should not since it # has not had its location updated yet. If there's no # exception here, CID 2 is properly ignored. scan_lacs.delay()
def test_delete_cell_measures(self): session = self.db_master_session block = MeasureBlock() block.measure_type = MEASURE_TYPE_CODE['cell'] block.start_id = 120 block.end_id = 140 block.s3_key = 'fake_key' block.archive_sha = 'fake_sha' block.archive_date = None session.add(block) for i in range(100, 150): session.add(CellMeasure(id=i, created=self.really_old)) session.commit() with patch.object(S3Backend, 'check_archive', lambda x, y, z: True): delete_cellmeasure_records.delay(batch=3).get() self.assertEquals(session.query(CellMeasure).count(), 30) self.assertTrue(block.archive_date is not None)
def test_skip_delete_new_blocks(self): now = util.utcnow() session = self.db_master_session block = MeasureBlock() block.measure_type = MEASURE_TYPE_CODE['cell'] block.start_id = 120 block.end_id = 140 block.s3_key = 'fake_key' block.archive_sha = 'fake_sha' block.archive_date = None session.add(block) measures = [] for i in range(100, 150): measures.append(CellMeasure(id=i, created=now)) session.add_all(measures) session.commit() with patch.object(S3Backend, 'check_archive', lambda x, y, z: True): delete_cellmeasure_records.delay(batch=3).get() self.assertEquals(session.query(CellMeasure).count(), 50) # The archive_date should *not* be set as we haven't deleted # the actual records yet self.assertTrue(block.archive_date is None) # Update all the create dates to be far in the past for m in measures: m.created = self.really_old session.add_all(measures) session.commit() with patch.object(S3Backend, 'check_archive', lambda x, y, z: True): delete_cellmeasure_records.delay(batch=3).get() self.assertEquals(session.query(CellMeasure).count(), 30) # The archive_date should now be set as the measure records # have been deleted. self.assertTrue(block.archive_date is not None)
def _make_one(self, **kw): from ichnaea.models import CellMeasure return CellMeasure(**kw)
def test_do_backfill(self): session = self.db_master_session # These are our reference towers that will be used to match # similar towers data = [ # These are measurements for tower A Cell(lat=378348600, lon=-1222828703, radio=2, lac=56955, cid=5286246, mcc=310, mnc=410, psc=38, new_measures=0, total_measures=1), # These are measurements for tower B Cell(lat=30, lon=-20, radio=3, lac=20, cid=31, mcc=310, mnc=410, psc=38, new_measures=0, total_measures=1), ] session.add_all(data) # This is tower C and should map back to tower A towerC_lat = 378348600 towerC_lon = -1222828703 session.add_all([CellMeasure(lat=towerC_lat, lon=towerC_lon, radio=2, lac=-1, cid=-1, mcc=310, mnc=410, psc=38, accuracy=20)]) # This is tower D and should map back to tower B session.add_all([CellMeasure(lat=30, lon=-20, radio=3, lac=-1, cid=-1, mcc=310, mnc=410, psc=38, accuracy=20)]) # This is tower E and should not map back to anything as the # radio doesn't match up session.add_all([CellMeasure(lat=30, lon=-20, radio=0, lac=-1, cid=-1, mcc=310, mnc=410, psc=38, accuracy=20)]) # This is tower F and should not map back to anything as it's # too far away. session.add_all([CellMeasure(lat=998409925, lon=998409925, radio=3, lac=-1, cid=-1, mcc=310, mnc=410, psc=38, accuracy=20)]) session.commit() do_backfill.delay() # check that tower C was mapped correctly rset = session.execute(text( "select * from cell_measure where " "radio = 2 and lac = 56955 and cid = 5286246")) rset = list(rset) self.assertEquals(len(rset), 1) lat_longs = [(row['lat'], row['lon']) for row in rset] assert (towerC_lat, towerC_lon) in lat_longs # check that tower D was mapped correctly rset = session.execute(text( "select * from cell_measure where " "radio = 3 and lac = 20 and cid = 31")) rset = list(rset) self.assertEquals(len(rset), 1) lat_longs = [(row['lat'], row['lon']) for row in rset] assert (30, -20) in lat_longs # we shouldn't map tower E when the known towers have # different radios than our incomplete tower records rset = session.execute(text( "select count(*) from cell_measure where " "radio = 0 and lac = - 1 and cid = -1")) rset = list(rset) self.assertEquals(len(rset), 1) # Tower F shouldn't map to any known tower as it's too far away rset = session.execute(text( "select count(*) from cell_measure where radio = 3 and " "lat = 98409925 and lon = 98409925 and lac = -1 and cid = -1")) rset = list(rset) self.assertEquals(len(rset), 1)
def test_blacklist_moving_cells(self): now = util.utcnow() long_ago = now - timedelta(days=40) session = self.db_master_session k1 = dict(radio=1, mcc=1, mnc=2, lac=3, cid=4) k2 = dict(radio=1, mcc=1, mnc=2, lac=6, cid=8) k3 = dict(radio=1, mcc=1, mnc=2, lac=9, cid=12) k4 = dict(radio=1, mcc=1, mnc=2, lac=12, cid=16) k5 = dict(radio=1, mcc=1, mnc=2, lac=15, cid=20) k6 = dict(radio=1, mcc=1, mnc=2, lac=18, cid=24) keys = set([CellKey(**k) for k in [k1, k2, k3, k4, k5, k6]]) # keys k2, k3 and k4 are expected to be detected as moving data = [ # a cell with an entry but no prior position Cell(new_measures=3, total_measures=0, **k1), CellMeasure(lat=1.001, lon=1.001, **k1), CellMeasure(lat=1.002, lon=1.005, **k1), CellMeasure(lat=1.003, lon=1.009, **k1), # a cell with a prior known position Cell(lat=2.0, lon=2.0, new_measures=2, total_measures=1, **k2), CellMeasure(lat=2.0, lon=2.0, **k2), CellMeasure(lat=4.0, lon=2.0, **k2), # a cell with a very different prior position Cell(lat=1.0, lon=1.0, new_measures=2, total_measures=1, **k3), CellMeasure(lat=3.0, lon=3.0, **k3), CellMeasure(lat=-3.0, lon=3.0, **k3), # another cell with a prior known position (and negative lat) Cell(lat=-4.0, lon=4.0, new_measures=2, total_measures=1, **k4), CellMeasure(lat=-4.0, lon=4.0, **k4), CellMeasure(lat=-6.0, lon=4.0, **k4), # an already blacklisted cell CellBlacklist(**k5), CellMeasure(lat=5.0, lon=5.0, **k5), CellMeasure(lat=8.0, lon=5.0, **k5), # a cell with an old different record we ignore, position # estimate has been updated since Cell(lat=6.0, lon=6.0, new_measures=2, total_measures=1, **k6), CellMeasure(lat=6.9, lon=6.9, time=long_ago, **k6), CellMeasure(lat=6.0, lon=6.0, **k6), CellMeasure(lat=6.001, lon=6, **k6), ] session.add_all(data) session.commit() result = location_update_cell.delay(min_new=1) self.assertEqual(result.get(), (5, 3)) black = session.query(CellBlacklist).all() self.assertEqual(set([to_cellkey(b) for b in black]), set([CellKey(**k) for k in [k2, k3, k4, k5]])) measures = session.query(CellMeasure).all() self.assertEqual(len(measures), 14) self.assertEqual(set([to_cellkey(m) for m in measures]), keys) # test duplicate call result = location_update_cell.delay(min_new=1) self.assertEqual(result.get(), 0) self.check_stats( total=6, timer=[ # We made duplicate calls ('task.data.location_update_cell', 2), # One of those would've scheduled a remove_cell task ('task.data.remove_cell', 1) ], gauge=[ ('task.data.location_update_cell.new_measures_1_100', 2), ])