def test_backfill_cell_location_update(self):
from ichnaea.tasks import backfill_cell_location_update
session = self.db_master_session
k1 = dict(radio=1, mcc=1, mnc=2, lac=3, cid=4)
data = [
Cell(lat=10010000,
lon=10010000,
new_measures=0,
total_measures=1,
**k1),
CellMeasure(lat=10000000, lon=10000000, **k1),
CellMeasure(lat=10050000, lon=10080000, **k1),
]
session.add_all(data)
session.commit()
query = session.query(CellMeasure.id)
cm_ids = [x[0] for x in query.all()]
# TODO: refactor this to be constants in the method
new_measures = [((1, 1, 2, 3, 4), cm_ids)]
result = backfill_cell_location_update.delay(new_measures)
self.assertEqual(result.get(), (1, 0))
cells = session.query(Cell).filter(Cell.cid != CELLID_LAC).all()
self.assertEqual(len(cells), 1)
cell = cells[0]
self.assertEqual(cell.lat, 10020000)
self.assertEqual(cell.lon, 10030000)
self.assertEqual(cell.new_measures, 0)
self.assertEqual(cell.total_measures, 3)
def test_cell_max_min_range_update(self):
from ichnaea.tasks import cell_location_update
session = self.db_master_session
k1 = dict(radio=1, mcc=1, mnc=2, lac=3, cid=4)
data = [
Cell(lat=10010000, lon=-10010000,
max_lat=10020000, min_lat=10000000,
max_lon=-10000000, min_lon=-10020000,
new_measures=2, total_measures=4, **k1),
CellMeasure(lat=10010000, lon=-10030000, **k1),
CellMeasure(lat=10050000, lon=-10070000, **k1),
]
session.add_all(data)
session.commit()
result = cell_location_update.delay(min_new=1)
self.assertEqual(result.get(), (1, 0))
cells = session.query(Cell).all()
self.assertEqual(len(cells), 1)
cell = cells[0]
self.assertEqual(cell.lat, 10020000)
self.assertEqual(cell.max_lat, 10050000)
self.assertEqual(cell.min_lat, 10000000)
self.assertEqual(cell.lon, -10030000)
self.assertEqual(cell.max_lon, -10000000)
self.assertEqual(cell.min_lon, -10070000)
# independent calculation: the cell bounding box is
# (1.000, -1.007) to (1.005, -1.000), and distance
# between those is 956.43m, int(round(dist/2.0)) is 478m
self.assertEqual(cell.range, 478)
def test_unthrottle_cell_measures(self):
session = self.db_master_session
block = MeasureBlock()
block.measure_type = MEASURE_TYPE_CODE['cell']
block.start_id = 120
block.end_id = 140
block.s3_key = 'fake_key'
block.archive_sha = 'fake_sha'
block.archive_date = None
session.add(block)
gsm = RADIO_TYPE['gsm']
k = dict(radio=gsm, mcc=1, mnc=2, lac=4, lat=1.0, lon=1.0)
for i in range(100, 150):
session.add(CellMeasure(id=i, cid=i, created=self.really_old, **k))
session.add(Cell(total_measures=11000, cid=i, **k))
session.commit()
with patch.object(S3Backend, 'check_archive', lambda x, y, z: True):
delete_cellmeasure_records.delay(batch=3).get()
cell_unthrottle_measures.delay(10000, 1000).get()
cells = session.query(Cell).all()
self.assertEquals(len(cells), 50)
for cell in cells:
if 120 <= cell.cid and cell.cid < 140:
self.assertEquals(cell.total_measures, 0)
else:
self.assertEquals(cell.total_measures, 1)
self.check_stats(counter=['items.cell_unthrottled'])
def test_schedule_cell_measures(self):
session = self.db_master_session
blocks = schedule_cellmeasure_archival.delay(batch=1).get()
self.assertEquals(len(blocks), 0)
measures = []
for i in range(20):
measures.append(CellMeasure(created=self.really_old))
session.add_all(measures)
session.flush()
start_id = measures[0].id
blocks = schedule_cellmeasure_archival.delay(batch=15).get()
self.assertEquals(len(blocks), 1)
block = blocks[0]
self.assertEquals(block, (start_id, start_id + 15))
blocks = schedule_cellmeasure_archival.delay(batch=6).get()
self.assertEquals(len(blocks), 0)
blocks = schedule_cellmeasure_archival.delay(batch=5).get()
self.assertEquals(len(blocks), 1)
block = blocks[0]
self.assertEquals(block, (start_id + 15, start_id + 20))
blocks = schedule_cellmeasure_archival.delay(batch=1).get()
self.assertEquals(len(blocks), 0)
def add_line_of_cells_and_scan_lac(self):
session = self.db_master_session
big = 1.0
small = big / 10
keys = dict(radio=1, mcc=1, mnc=1, lac=1)
measures = [
CellMeasure(lat=ctr + xd, lon=ctr + yd, cid=cell, **keys)
for cell in range(10) for ctr in [cell * big]
for (xd, yd) in [(small, small), (small,
-small), (-small,
small), (-small,
-small)]
]
session.add_all(measures)
cells = [
Cell(lat=ctr,
lon=ctr,
cid=cell,
new_measures=4,
total_measures=1,
**keys) for cell in range(10) for ctr in [cell * big]
]
session.add_all(cells)
session.commit()
result = location_update_cell.delay(min_new=0,
max_new=9999,
batch=len(measures))
self.assertEqual(result.get(), (len(cells), 0))
scan_lacs.delay()
def create_cell_measure(utcnow, entry):
# skip records with missing or invalid mcc or mnc
if 'mcc' not in entry or entry['mcc'] < 1 or entry['mcc'] > 999:
return
if 'mnc' not in entry or entry['mnc'] < 0 or entry['mnc'] > 32767:
return
# Skip CDMA towers missing lac or cid (no psc on CDMA exists to
# backfill using inference)
if entry.get('radio', -1) == 1 and \
(entry.get('lac', -1) < 0 or entry.get('cid', -1) < 0):
return
# some phones send maxint32 to signal "unknown"
# ignore anything above the maximum valid values
if 'lac' not in entry or entry['lac'] < 0 or entry['lac'] > 65535:
entry['lac'] = -1
if 'cid' not in entry or entry['cid'] < 0 or entry['cid'] > 268435455:
entry['cid'] = -1
if 'psc' not in entry or entry['psc'] < 0 or entry['psc'] > 512:
entry['psc'] = -1
# Treat the lac=0, cid=65535 combination as unspecified values
if entry['lac'] == 0 and entry['cid'] == 65535:
entry['lac'] = -1
entry['cid'] = -1
# Must have LAC+CID or PSC
if (entry['lac'] == -1 or entry['cid'] == -1) and entry['psc'] == -1:
return
# make sure fields stay within reasonable bounds
if 'asu' not in entry or entry['asu'] < 0 or entry['asu'] > 100:
entry['asu'] = -1
if 'signal' not in entry or entry['signal'] < -200 or entry['signal'] > -1:
entry['signal'] = 0
if 'ta' not in entry or entry['ta'] < 0 or entry['ta'] > 100:
entry['ta'] = 0
return CellMeasure(
measure_id=entry.get('measure_id'),
created=utcnow,
lat=entry['lat'],
lon=entry['lon'],
time=decode_datetime(entry.get('time', '')),
accuracy=entry.get('accuracy', 0),
altitude=entry.get('altitude', 0),
altitude_accuracy=entry.get('altitude_accuracy', 0),
radio=entry.get('radio', -1),
mcc=entry.get('mcc', -1),
mnc=entry.get('mnc', -1),
lac=entry.get('lac', -1),
cid=entry.get('cid', -1),
psc=entry.get('psc', -1),
asu=entry.get('asu', -1),
signal=entry.get('signal', 0),
ta=entry.get('ta', 0),
)
def test_location_update_cell(self):
now = util.utcnow()
before = now - timedelta(days=1)
schema = ValidCellBaseSchema()
session = self.db_master_session
k1 = dict(radio=1, mcc=1, mnc=2, lac=3, cid=4)
k2 = dict(radio=1, mcc=1, mnc=2, lac=6, cid=8)
k3 = dict(radio=1,
mcc=1,
mnc=2,
lac=schema.fields['lac'].missing,
cid=schema.fields['cid'].missing)
data = [
Cell(new_measures=3, total_measures=5, **k1),
CellMeasure(lat=1.0, lon=1.0, **k1),
CellMeasure(lat=1.002, lon=1.003, **k1),
CellMeasure(lat=1.004, lon=1.006, **k1),
# The lac, cid are invalid and should be skipped
CellMeasure(lat=1.5, lon=1.5, **k3),
CellMeasure(lat=1.502, lon=1.503, **k3),
Cell(lat=2.0, lon=2.0, new_measures=2, total_measures=4, **k2),
# the lat/lon is bogus and mismatches the line above on purpose
# to make sure old measures are skipped
CellMeasure(lat=-1.0, lon=-1.0, created=before, **k2),
CellMeasure(lat=-1.0, lon=-1.0, created=before, **k2),
CellMeasure(lat=2.002, lon=2.004, **k2),
CellMeasure(lat=2.002, lon=2.004, **k2),
]
session.add_all(data)
session.commit()
result = location_update_cell.delay(min_new=1)
self.assertEqual(result.get(), (2, 0))
self.check_stats(
total=2,
timer=['task.data.location_update_cell'],
gauge=['task.data.location_update_cell.new_measures_1_100'],
)
cells = session.query(Cell).all()
self.assertEqual(len(cells), 2)
self.assertEqual([c.new_measures for c in cells], [0, 0])
for cell in cells:
if cell.cid == 4:
self.assertEqual(cell.lat, 1.002)
self.assertEqual(cell.lon, 1.003)
elif cell.cid == 8:
self.assertEqual(cell.lat, 2.001)
self.assertEqual(cell.lon, 2.002)
def process_cell(entries, measure):
result = []
cells = []
for entry in entries:
cell = CellMeasure(
measure_id=measure.id, created=measure.created,
lat=measure.lat, lon=measure.lon, time=measure.time,
accuracy=measure.accuracy, altitude=measure.altitude,
altitude_accuracy=measure.altitude_accuracy,
mcc=entry['mcc'], mnc=entry['mnc'], lac=entry['lac'],
cid=entry['cid'], psc=entry['psc'], asu=entry['asu'],
signal=entry['signal'], ta=entry['ta'],
)
# use more specific cell type or fall back to less precise measure
if entry['radio']:
cell.radio = RADIO_TYPE.get(entry['radio'], -1)
else:
cell.radio = measure.radio
cells.append(cell)
result.append(entry)
return (cells, result)
def create_cell_measure(utcnow, entry):
# creates a dict.copy() avoiding test leaks reusing the same
# entries for multiple task calls
entry = normalized_cell_measure_dict(entry)
if entry is None:
return None
# add creation date
entry['created'] = utcnow
# decode from JSON compatible format
entry['report_id'] = uuid.UUID(hex=entry['report_id']).bytes
entry['time'] = decode_datetime(entry['time'])
return CellMeasure(**entry)
def test_cell_max_min_range_update(self):
from ichnaea.tasks import cell_location_update
session = self.db_master_session
k1 = dict(radio=1, mcc=1, mnc=2, lac=3, cid=4)
data = [
Cell(lat=10010000,
lon=-10010000,
max_lat=10020000,
min_lat=10000000,
max_lon=-10000000,
min_lon=-10020000,
new_measures=2,
total_measures=4,
**k1),
CellMeasure(lat=10010000, lon=-10030000, **k1),
CellMeasure(lat=10050000, lon=-10070000, **k1),
]
session.add_all(data)
session.commit()
result = cell_location_update.delay(min_new=1)
self.assertEqual(result.get(), (1, 0))
cells = session.query(Cell).filter(Cell.cid != CELLID_LAC).all()
self.assertEqual(len(cells), 1)
cell = cells[0]
self.assertEqual(cell.lat, 10020000)
self.assertEqual(cell.max_lat, 10050000)
self.assertEqual(cell.min_lat, 10000000)
self.assertEqual(cell.lon, -10030000)
self.assertEqual(cell.max_lon, -10000000)
self.assertEqual(cell.min_lon, -10070000)
# independent calculation: the cell bounding box is
# (1.000, -1.007) to (1.005, -1.000), with centroid
# at (1.002, -1.003); worst distance from centroid
# to any corner is 556m
self.assertEqual(cell.range, 556)
def test_cell_location_update(self):
from ichnaea.tasks import cell_location_update
now = datetime.utcnow()
before = now - timedelta(days=1)
session = self.db_master_session
k1 = dict(radio=1, mcc=1, mnc=2, lac=3, cid=4)
k2 = dict(radio=1, mcc=1, mnc=2, lac=6, cid=8)
k3 = dict(radio=1, mcc=1, mnc=2, lac=-1, cid=-1)
data = [
Cell(new_measures=3, total_measures=5, **k1),
CellMeasure(lat=10000000, lon=10000000, **k1),
CellMeasure(lat=10020000, lon=10030000, **k1),
CellMeasure(lat=10040000, lon=10060000, **k1),
# The lac, cid are invalid and should be skipped
CellMeasure(lat=15000000, lon=15000000, **k3),
CellMeasure(lat=15020000, lon=15030000, **k3),
Cell(lat=20000000,
lon=20000000,
new_measures=2,
total_measures=4,
**k2),
# the lat/lon is bogus and mismatches the line above on purpose
# to make sure old measures are skipped
CellMeasure(lat=-10000000, lon=-10000000, created=before, **k2),
CellMeasure(lat=-10000000, lon=-10000000, created=before, **k2),
CellMeasure(lat=20020000, lon=20040000, **k2),
CellMeasure(lat=20020000, lon=20040000, **k2),
]
session.add_all(data)
session.commit()
result = cell_location_update.delay(min_new=1)
self.assertEqual(result.get(), (2, 0))
cells = session.query(Cell).filter(Cell.cid != CELLID_LAC).all()
self.assertEqual(len(cells), 2)
self.assertEqual([c.new_measures for c in cells], [0, 0])
for cell in cells:
if cell.cid == 4:
self.assertEqual(cell.lat, 10020000)
self.assertEqual(cell.lon, 10030000)
elif cell.cid == 8:
self.assertEqual(cell.lat, 20010000)
self.assertEqual(cell.lon, 20020000)
def test_monitor_measures(self):
session = self.db_master_session
result = monitor_measures.delay().get()
self.check_stats(gauge=[('table.cell_measure', 1),
('table.wifi_measure', 1)], )
self.assertEqual(result, {'cell_measure': -1, 'wifi_measure': -1})
# add some measures
session.add_all([CellMeasure() for i in range(3)])
session.add_all([WifiMeasure() for i in range(5)])
session.flush()
result = monitor_measures.delay().get()
self.check_stats(gauge=[('table.cell_measure', 2),
('table.wifi_measure', 2)], )
self.assertEqual(result, {'cell_measure': 3, 'wifi_measure': 5})
def test_backup_cell_to_s3(self):
session = self.db_master_session
batch_size = 10
measures = []
for i in range(batch_size):
measures.append(CellMeasure(created=self.really_old))
session.add_all(measures)
session.flush()
start_id = measures[0].id
blocks = schedule_cellmeasure_archival.delay(batch=batch_size).get()
self.assertEquals(len(blocks), 1)
block = blocks[0]
self.assertEquals(block, (start_id, start_id + batch_size))
with mock_s3():
with patch.object(S3Backend, 'backup_archive',
lambda x, y, z: True):
write_cellmeasure_s3_backups.delay(cleanup_zip=False).get()
msgs = self.heka_client.stream.msgs
info_msgs = [m for m in msgs if m.type == 'oldstyle']
self.assertEquals(1, len(info_msgs))
info = info_msgs[0]
fname = info.payload.split(":")[-1]
myzip = ZipFile(fname)
try:
contents = set(myzip.namelist())
expected_contents = set(
['alembic_revision.txt', 'cell_measure.csv'])
self.assertEquals(expected_contents, contents)
finally:
myzip.close()
blocks = session.query(MeasureBlock).all()
self.assertEquals(len(blocks), 1)
block = blocks[0]
actual_sha = hashlib.sha1()
actual_sha.update(open(fname, 'rb').read())
self.assertEquals(block.archive_sha, actual_sha.digest())
self.assertTrue(block.s3_key is not None)
self.assertTrue('/cell_' in block.s3_key)
self.assertTrue(block.archive_date is None)
def test_cell_lac_asymmetric(self):
from ichnaea.tasks import cell_location_update, scan_lacs
session = self.db_master_session
big = 1000000
small = big / 10
keys = dict(radio=1, mcc=1, mnc=1, lac=1)
measures = [
CellMeasure(lat=ctr + xd, lon=ctr + yd, cid=cell, **keys)
for cell in range(0, 6) for ctr in [(2**cell) * big]
for (xd, yd) in [(small, small), (small,
-small), (-small,
small), (-small,
-small)]
]
session.add_all(measures)
cells = [
Cell(lat=ctr,
lon=ctr,
cid=cell,
new_measures=4,
total_measures=1,
**keys) for cell in range(0, 6) for ctr in [(2**cell) * big]
]
session.add_all(cells)
session.commit()
result = cell_location_update.delay(min_new=0,
max_new=9999,
batch=len(measures))
self.assertEqual(result.get(), (len(cells), 0))
scan_lacs.delay()
lac = session.query(Cell).filter(Cell.lac == 1,
Cell.cid == CELLID_LAC).first()
# We produced a sequence of 0.02-degree-on-a-side
# cell bounding boxes centered at
# [0, 0.2, 0.4, 0.8, 1.6, 3.2] degrees.
# So the lower-left corner is at (-0.01, -0.01)
# and the upper-right corner is at (3.21, 3.21)
# we should therefore see a LAC centroid at (1.05, 1.05)
# with a range of 339.540m
self.assertEqual(lac.lat, 10500000)
self.assertEqual(lac.lon, 10500000)
self.assertEqual(lac.range, 339540)
def test_unique_cell_histogram(self):
from ichnaea.content.tasks import unique_cell_histogram
session = self.db_master_session
today = datetime.utcnow().date()
one_day = (today - timedelta(1))
two_days = (today - timedelta(2))
long_ago = (today - timedelta(40))
measures = [
CellMeasure(created=long_ago, radio=0, mcc=1, mnc=2, lac=3, cid=4),
CellMeasure(created=two_days, radio=2, mcc=1, mnc=2, lac=3, cid=4),
CellMeasure(created=two_days, radio=0, mcc=1, mnc=2, lac=3, cid=4),
CellMeasure(created=two_days, radio=0, mcc=2, mnc=2, lac=3, cid=4),
CellMeasure(created=one_day, radio=0, mcc=2, mnc=2, lac=3, cid=5),
CellMeasure(created=today, radio=0, mcc=1, mnc=3, lac=3, cid=4),
CellMeasure(created=today, radio=0, mcc=1, mnc=2, lac=4, cid=4),
]
session.add_all(measures)
session.commit()
result = unique_cell_histogram.delay(ago=40)
self.assertEqual(result.get(), 1)
stats = session.query(Stat).order_by(Stat.time).all()
self.assertEqual(len(stats), 1)
self.assertEqual(stats[0].key, STAT_TYPE['unique_cell'])
self.assertEqual(stats[0].time, long_ago)
self.assertEqual(stats[0].value, 1)
# fill up newer dates
unique_cell_histogram.delay(ago=2).get()
unique_cell_histogram.delay(ago=1).get()
unique_cell_histogram.delay(ago=0).get()
stats = session.query(Stat).order_by(Stat.time).all()
self.assertEqual(len(stats), 4)
self.assertEqual(stats[0].time, long_ago)
self.assertEqual(stats[0].value, 1)
self.assertEqual(stats[1].time, two_days)
self.assertEqual(stats[1].value, 3)
self.assertEqual(stats[2].time, one_day)
self.assertEqual(stats[2].value, 4)
self.assertEqual(stats[3].time, today)
self.assertEqual(stats[3].value, 6)
# test duplicate execution
result = unique_cell_histogram.delay()
self.assertEqual(result.get(), 0)
def test_cell_histogram(self):
from ichnaea.content.tasks import cell_histogram
session = self.db_master_session
today = datetime.utcnow().date()
yesterday = (today - timedelta(1))
two_days = (today - timedelta(2))
long_ago = (today - timedelta(40))
measures = [
CellMeasure(lat=10000000, lon=20000000, created=today),
CellMeasure(lat=10000000, lon=20000000, created=today),
CellMeasure(lat=10000000, lon=20000000, created=yesterday),
CellMeasure(lat=10000000, lon=20000000, created=two_days),
CellMeasure(lat=10000000, lon=20000000, created=two_days),
CellMeasure(lat=10000000, lon=20000000, created=two_days),
CellMeasure(lat=10000000, lon=20000000, created=long_ago),
]
session.add_all(measures)
session.commit()
cell_histogram.delay(ago=40).get()
stats = session.query(Stat).order_by(Stat.time).all()
self.assertEqual(len(stats), 1)
self.assertEqual(stats[0].key, STAT_TYPE['cell'])
self.assertEqual(stats[0].time, long_ago)
self.assertEqual(stats[0].value, 1)
# fill up newer dates
cell_histogram.delay(ago=2).get()
cell_histogram.delay(ago=1).get()
cell_histogram.delay(ago=0).get()
stats = session.query(Stat).order_by(Stat.time).all()
self.assertEqual(len(stats), 4)
self.assertEqual(stats[0].time, long_ago)
self.assertEqual(stats[0].value, 1)
self.assertEqual(stats[1].time, two_days)
self.assertEqual(stats[1].value, 4)
self.assertEqual(stats[2].time, yesterday)
self.assertEqual(stats[2].value, 5)
self.assertEqual(stats[3].time, today)
self.assertEqual(stats[3].value, 7)
# test duplicate execution
result = cell_histogram.delay(ago=1)
self.assertEqual(result.get(), 0)
def create_cell_measure(measure_data, entry):
return CellMeasure(
measure_id=measure_data['id'],
created=decode_datetime(measure_data.get('created', '')),
lat=measure_data['lat'],
lon=measure_data['lon'],
time=decode_datetime(measure_data.get('time', '')),
accuracy=measure_data.get('accuracy', 0),
altitude=measure_data.get('altitude', 0),
altitude_accuracy=measure_data.get('altitude_accuracy', 0),
mcc=entry['mcc'],
mnc=entry['mnc'],
lac=entry.get('lac', 0),
cid=entry.get('cid', 0),
psc=entry.get('psc', 0),
asu=entry.get('asu', 0),
signal=entry.get('signal', 0),
ta=entry.get('ta', 0),
)
def test_scan_lacs_race_with_location_update(self):
session = self.db_master_session
# First batch of cell measurements for CID 1
keys = dict(radio=1, mcc=1, mnc=1, lac=1, cid=1)
cell = Cell(new_measures=4, total_measures=1, **keys)
measures = [
CellMeasure(lat=1.0, lon=1.0, **keys),
CellMeasure(lat=1.0, lon=1.0, **keys),
CellMeasure(lat=1.0, lon=1.0, **keys),
CellMeasure(lat=1.0, lon=1.0, **keys),
]
session.add(cell)
session.add_all(measures)
session.commit()
# Periodic location_update_cell runs and updates CID 1
# to have a location, inserts LAC 1 with new_measures=1
# which will be picked up by the next scan_lac.
result = location_update_cell.delay(min_new=1)
self.assertEqual(result.get(), (1, 0))
# Second batch of cell measurements for CID 2
keys['cid'] = 2
cell = Cell(new_measures=4, total_measures=1, **keys)
measures = [
CellMeasure(lat=1.0, lon=1.0, **keys),
CellMeasure(lat=1.0, lon=1.0, **keys),
CellMeasure(lat=1.0, lon=1.0, **keys),
CellMeasure(lat=1.0, lon=1.0, **keys),
]
session.add(cell)
session.add_all(measures)
session.commit()
# Periodic LAC scan runs, picking up LAC 1; this could
# accidentally pick up CID 2, but it should not since it
# has not had its location updated yet. If there's no
# exception here, CID 2 is properly ignored.
scan_lacs.delay()
def test_delete_cell_measures(self):
session = self.db_master_session
block = MeasureBlock()
block.measure_type = MEASURE_TYPE_CODE['cell']
block.start_id = 120
block.end_id = 140
block.s3_key = 'fake_key'
block.archive_sha = 'fake_sha'
block.archive_date = None
session.add(block)
for i in range(100, 150):
session.add(CellMeasure(id=i, created=self.really_old))
session.commit()
with patch.object(S3Backend, 'check_archive', lambda x, y, z: True):
delete_cellmeasure_records.delay(batch=3).get()
self.assertEquals(session.query(CellMeasure).count(), 30)
self.assertTrue(block.archive_date is not None)
def test_skip_delete_new_blocks(self):
now = util.utcnow()
session = self.db_master_session
block = MeasureBlock()
block.measure_type = MEASURE_TYPE_CODE['cell']
block.start_id = 120
block.end_id = 140
block.s3_key = 'fake_key'
block.archive_sha = 'fake_sha'
block.archive_date = None
session.add(block)
measures = []
for i in range(100, 150):
measures.append(CellMeasure(id=i, created=now))
session.add_all(measures)
session.commit()
with patch.object(S3Backend, 'check_archive', lambda x, y, z: True):
delete_cellmeasure_records.delay(batch=3).get()
self.assertEquals(session.query(CellMeasure).count(), 50)
# The archive_date should *not* be set as we haven't deleted
# the actual records yet
self.assertTrue(block.archive_date is None)
# Update all the create dates to be far in the past
for m in measures:
m.created = self.really_old
session.add_all(measures)
session.commit()
with patch.object(S3Backend, 'check_archive', lambda x, y, z: True):
delete_cellmeasure_records.delay(batch=3).get()
self.assertEquals(session.query(CellMeasure).count(), 30)
# The archive_date should now be set as the measure records
# have been deleted.
self.assertTrue(block.archive_date is not None)
def _make_one(self, **kw):
from ichnaea.models import CellMeasure
return CellMeasure(**kw)
def test_do_backfill(self):
session = self.db_master_session
# These are our reference towers that will be used to match
# similar towers
data = [
# These are measurements for tower A
Cell(lat=378348600, lon=-1222828703, radio=2,
lac=56955, cid=5286246, mcc=310, mnc=410, psc=38,
new_measures=0, total_measures=1),
# These are measurements for tower B
Cell(lat=30, lon=-20, radio=3,
lac=20, cid=31, mcc=310, mnc=410, psc=38,
new_measures=0, total_measures=1),
]
session.add_all(data)
# This is tower C and should map back to tower A
towerC_lat = 378348600
towerC_lon = -1222828703
session.add_all([CellMeasure(lat=towerC_lat, lon=towerC_lon, radio=2,
lac=-1, cid=-1, mcc=310, mnc=410, psc=38,
accuracy=20)])
# This is tower D and should map back to tower B
session.add_all([CellMeasure(lat=30, lon=-20, radio=3,
lac=-1, cid=-1, mcc=310, mnc=410, psc=38,
accuracy=20)])
# This is tower E and should not map back to anything as the
# radio doesn't match up
session.add_all([CellMeasure(lat=30, lon=-20, radio=0,
lac=-1, cid=-1, mcc=310, mnc=410, psc=38,
accuracy=20)])
# This is tower F and should not map back to anything as it's
# too far away.
session.add_all([CellMeasure(lat=998409925, lon=998409925, radio=3,
lac=-1, cid=-1, mcc=310, mnc=410, psc=38,
accuracy=20)])
session.commit()
do_backfill.delay()
# check that tower C was mapped correctly
rset = session.execute(text(
"select * from cell_measure where "
"radio = 2 and lac = 56955 and cid = 5286246"))
rset = list(rset)
self.assertEquals(len(rset), 1)
lat_longs = [(row['lat'], row['lon']) for row in rset]
assert (towerC_lat, towerC_lon) in lat_longs
# check that tower D was mapped correctly
rset = session.execute(text(
"select * from cell_measure where "
"radio = 3 and lac = 20 and cid = 31"))
rset = list(rset)
self.assertEquals(len(rset), 1)
lat_longs = [(row['lat'], row['lon']) for row in rset]
assert (30, -20) in lat_longs
# we shouldn't map tower E when the known towers have
# different radios than our incomplete tower records
rset = session.execute(text(
"select count(*) from cell_measure where "
"radio = 0 and lac = - 1 and cid = -1"))
rset = list(rset)
self.assertEquals(len(rset), 1)
# Tower F shouldn't map to any known tower as it's too far away
rset = session.execute(text(
"select count(*) from cell_measure where radio = 3 and "
"lat = 98409925 and lon = 98409925 and lac = -1 and cid = -1"))
rset = list(rset)
self.assertEquals(len(rset), 1)
def test_blacklist_moving_cells(self):
now = util.utcnow()
long_ago = now - timedelta(days=40)
session = self.db_master_session
k1 = dict(radio=1, mcc=1, mnc=2, lac=3, cid=4)
k2 = dict(radio=1, mcc=1, mnc=2, lac=6, cid=8)
k3 = dict(radio=1, mcc=1, mnc=2, lac=9, cid=12)
k4 = dict(radio=1, mcc=1, mnc=2, lac=12, cid=16)
k5 = dict(radio=1, mcc=1, mnc=2, lac=15, cid=20)
k6 = dict(radio=1, mcc=1, mnc=2, lac=18, cid=24)
keys = set([CellKey(**k) for k in [k1, k2, k3, k4, k5, k6]])
# keys k2, k3 and k4 are expected to be detected as moving
data = [
# a cell with an entry but no prior position
Cell(new_measures=3, total_measures=0, **k1),
CellMeasure(lat=1.001, lon=1.001, **k1),
CellMeasure(lat=1.002, lon=1.005, **k1),
CellMeasure(lat=1.003, lon=1.009, **k1),
# a cell with a prior known position
Cell(lat=2.0, lon=2.0, new_measures=2, total_measures=1, **k2),
CellMeasure(lat=2.0, lon=2.0, **k2),
CellMeasure(lat=4.0, lon=2.0, **k2),
# a cell with a very different prior position
Cell(lat=1.0, lon=1.0, new_measures=2, total_measures=1, **k3),
CellMeasure(lat=3.0, lon=3.0, **k3),
CellMeasure(lat=-3.0, lon=3.0, **k3),
# another cell with a prior known position (and negative lat)
Cell(lat=-4.0, lon=4.0, new_measures=2, total_measures=1, **k4),
CellMeasure(lat=-4.0, lon=4.0, **k4),
CellMeasure(lat=-6.0, lon=4.0, **k4),
# an already blacklisted cell
CellBlacklist(**k5),
CellMeasure(lat=5.0, lon=5.0, **k5),
CellMeasure(lat=8.0, lon=5.0, **k5),
# a cell with an old different record we ignore, position
# estimate has been updated since
Cell(lat=6.0, lon=6.0, new_measures=2, total_measures=1, **k6),
CellMeasure(lat=6.9, lon=6.9, time=long_ago, **k6),
CellMeasure(lat=6.0, lon=6.0, **k6),
CellMeasure(lat=6.001, lon=6, **k6),
]
session.add_all(data)
session.commit()
result = location_update_cell.delay(min_new=1)
self.assertEqual(result.get(), (5, 3))
black = session.query(CellBlacklist).all()
self.assertEqual(set([to_cellkey(b) for b in black]),
set([CellKey(**k) for k in [k2, k3, k4, k5]]))
measures = session.query(CellMeasure).all()
self.assertEqual(len(measures), 14)
self.assertEqual(set([to_cellkey(m) for m in measures]), keys)
# test duplicate call
result = location_update_cell.delay(min_new=1)
self.assertEqual(result.get(), 0)
self.check_stats(
total=6,
timer=[
# We made duplicate calls
('task.data.location_update_cell', 2),
# One of those would've scheduled a remove_cell task
('task.data.remove_cell', 1)
],
gauge=[
('task.data.location_update_cell.new_measures_1_100', 2),
])
