def test_cell_out_of_range_values(self):
from ichnaea.service.submit.tasks import insert_cell_measures
session = self.db_master_session
time = datetime.utcnow().replace(microsecond=0) - timedelta(days=1)
measure = dict(
id=0, created=encode_datetime(time), lat=10000000, lon=20000000,
time=encode_datetime(time), accuracy=0, altitude=0,
altitude_accuracy=0, radio=RADIO_TYPE['gsm'], mcc=1,
mnc=2, lac=3, cid=4)
entries = [
{"asu": 8, "signal": -70, "ta": 32},
{"asu": -10, "signal": -300, "ta": -10},
{"asu": 256, "signal": 16, "ta": 128},
]
for e in entries:
e.update(measure)
result = insert_cell_measures.delay(entries)
self.assertEqual(result.get(), 3)
measures = session.query(CellMeasure).all()
self.assertEqual(len(measures), 3)
self.assertEqual(set([m.asu for m in measures]), set([-1, 8]))
self.assertEqual(set([m.signal for m in measures]), set([0, -70]))
self.assertEqual(set([m.ta for m in measures]), set([0, 32]))
def test_ignore_unhelpful_incomplete_cdma_cells(self):
# CDMA cell records must have MNC, MCC, LAC and CID filled in
from ichnaea.service.submit.tasks import insert_cell_measures
session = self.db_master_session
time = datetime.utcnow().replace(microsecond=0) - timedelta(days=1)
measure = dict(
id=0, created=encode_datetime(time), lat=10000000, lon=20000000,
time=encode_datetime(time), accuracy=0, altitude=0,
altitude_accuracy=0, radio=RADIO_TYPE['cdma'],
)
entries = [
# This records is valid
{"mcc": 1, "mnc": 2, "lac": 3, "cid": 4},
# This record should fail as it's missing CID
{"mcc": 1, "mnc": 2, "lac": 3},
# This fails for missing lac
{"mcc": 1, "mnc": 2, "cid": 4},
# Adding a psc doesn't change things
{"mcc": 1, "mnc": 2, "psc": 5},
]
for e in entries:
e.update(measure)
result = insert_cell_measures.delay(entries, userid=1)
self.assertEqual(result.get(), 1)
measures = session.query(CellMeasure).all()
self.assertEqual(len(measures), 1)
cells = session.query(Cell).all()
self.assertEqual(len(cells), 1)
def reprocess_cell_measure(self, measure_ids, userid=None):
measures = []
try:
with self.db_session() as session:
measures = session.query(Measure).filter(
Measure.id.in_(measure_ids)).filter(
Measure.cell != sql_null).all()
for measure in measures:
measure_data = dict(
id=measure.id, created=encode_datetime(measure.created),
lat=measure.lat, lon=measure.lon,
time=encode_datetime(measure.time),
accuracy=measure.accuracy, altitude=measure.altitude,
altitude_accuracy=measure.altitude_accuracy,
radio=measure.radio,
)
# adds data to this session
process_cell_measure(
session, measure_data, loads(measure.cell), userid=userid)
session.commit()
return len(measures)
except IntegrityError as exc: # pragma: no cover
logger.exception('error')
return 0
except Exception as exc: # pragma: no cover
raise self.retry(exc=exc)
def test_insert_invalid_lac(self):
from ichnaea.service.submit.tasks import insert_cell_measures
session = self.db_master_session
time = datetime.utcnow().replace(microsecond=0) - timedelta(days=1)
session.add(
Cell(radio=RADIO_TYPE['gsm'],
mcc=1,
mnc=2,
lac=3,
cid=4,
new_measures=2,
total_measures=5))
session.add(Score(userid=1, key=SCORE_TYPE['new_cell'], value=7))
session.flush()
measure = dict(id=0,
created=encode_datetime(time),
lat=10000000,
lon=20000000,
time=encode_datetime(time),
accuracy=0,
altitude=0,
altitude_accuracy=0,
radio=RADIO_TYPE['gsm'])
entries = [
{
"mcc": 1,
"mnc": 2,
"lac": 3147483647,
"cid": 2147483647,
"psc": 5,
"asu": 8
},
{
"mcc": 1,
"mnc": 2,
"lac": -1,
"cid": -1,
"psc": 5,
"asu": 8
},
]
for e in entries:
e.update(measure)
result = insert_cell_measures.delay(entries, userid=1)
self.assertEqual(result.get(), 2)
measures = session.query(CellMeasure).all()
self.assertEqual(len(measures), 2)
self.assertEqual(set([m.lac for m in measures]), set([-1]))
self.assertEqual(set([m.cid for m in measures]), set([-1]))
# Nothing should change in the initially created Cell record
cells = session.query(Cell).all()
self.assertEqual(len(cells), 1)
self.assertEqual(set([c.new_measures for c in cells]), set([2]))
self.assertEqual(set([c.total_measures for c in cells]), set([5]))
def process_wifi(entries, measure):
measure_data = dict(
id=measure.id, created=encode_datetime(measure.created),
lat=measure.lat, lon=measure.lon, time=encode_datetime(measure.time),
accuracy=measure.accuracy, altitude=measure.altitude,
altitude_accuracy=measure.altitude_accuracy,
)
insert_wifi_measure.delay(measure_data, entries)
def test_ignore_unhelpful_incomplete_cdma_cells(self):
# CDMA cell records must have MNC, MCC, LAC and CID filled in
from ichnaea.service.submit.tasks import insert_cell_measures
session = self.db_master_session
time = datetime.utcnow().replace(microsecond=0) - timedelta(days=1)
measure = dict(
id=0,
created=encode_datetime(time),
lat=10000000,
lon=20000000,
time=encode_datetime(time),
accuracy=0,
altitude=0,
altitude_accuracy=0,
radio=RADIO_TYPE['cdma'],
)
entries = [
# This records is valid
{
"mcc": 1,
"mnc": 2,
"lac": 3,
"cid": 4
},
# This record should fail as it's missing CID
{
"mcc": 1,
"mnc": 2,
"lac": 3
},
# This fails for missing lac
{
"mcc": 1,
"mnc": 2,
"cid": 4
},
# Adding a psc doesn't change things
{
"mcc": 1,
"mnc": 2,
"psc": 5
},
]
for e in entries:
e.update(measure)
result = insert_cell_measures.delay(entries, userid=1)
self.assertEqual(result.get(), 1)
measures = session.query(CellMeasure).all()
self.assertEqual(len(measures), 1)
cells = session.query(Cell).all()
self.assertEqual(len(cells), 1)
def test_wifi(self):
from ichnaea.service.submit.tasks import insert_wifi_measures
session = self.db_master_session
time = datetime.utcnow().replace(microsecond=0) - timedelta(days=1)
session.add(Wifi(key="ab1234567890"))
session.add(Score(userid=1, key=SCORE_TYPE["new_wifi"], value=7))
session.flush()
measure = dict(
id=0,
created=encode_datetime(time),
lat=10000000,
lon=20000000,
time=encode_datetime(time),
accuracy=0,
altitude=0,
altitude_accuracy=0,
radio=-1,
)
entries = [
{"key": "ab1234567890", "channel": 11, "signal": -80},
{"key": "ab1234567890", "channel": 3, "signal": -90},
{"key": "ab1234567890", "channel": 3, "signal": -80},
{"key": "cd3456789012", "channel": 3, "signal": -90},
]
for e in entries:
e.update(measure)
result = insert_wifi_measures.delay(entries, userid=1)
self.assertEqual(result.get(), 4)
measures = session.query(WifiMeasure).all()
self.assertEqual(len(measures), 4)
self.assertEqual(set([m.key for m in measures]), set(["ab1234567890", "cd3456789012"]))
self.assertEqual(set([m.channel for m in measures]), set([3, 11]))
self.assertEqual(set([m.signal for m in measures]), set([-80, -90]))
wifis = session.query(Wifi).all()
self.assertEqual(len(wifis), 2)
self.assertEqual(set([w.key for w in wifis]), set(["ab1234567890", "cd3456789012"]))
self.assertEqual(set([w.new_measures for w in wifis]), set([1, 3]))
self.assertEqual(set([w.total_measures for w in wifis]), set([1, 3]))
scores = session.query(Score).all()
self.assertEqual(len(scores), 1)
self.assertEqual(scores[0].key, SCORE_TYPE["new_wifi"])
self.assertEqual(scores[0].value, 8)
# test duplicate execution
result = insert_wifi_measures.delay(entries, userid=1)
self.assertEqual(result.get(), 4)
# TODO this task isn't idempotent yet
measures = session.query(WifiMeasure).all()
self.assertEqual(len(measures), 8)
def test_cell(self):
from ichnaea.service.submit.tasks import insert_cell_measure
session = self.db_master_session
time = datetime.utcnow().replace(microsecond=0) - timedelta(days=1)
session.add(Cell(radio=0, mcc=1, mnc=2, lac=3, cid=4,
new_measures=2, total_measures=5))
session.add(Score(userid=1, key=SCORE_TYPE['new_cell'], value=7))
session.flush()
measure = dict(
id=0, created=encode_datetime(time), lat=10000000, lon=20000000,
time=encode_datetime(time), accuracy=0, altitude=0,
altitude_accuracy=0, radio=0,
)
entries = [
{"mcc": 1, "mnc": 2, "signal": -100},
{"mcc": 1, "mnc": 2, "lac": 3, "cid": 4, "psc": 5, "asu": 8},
{"mcc": 1, "mnc": 2, "lac": 3, "cid": 4, "asu": 8},
{"mcc": 1, "mnc": 2, "lac": 3, "cid": 4, "asu": 15},
{"mcc": 1, "mnc": 2, "lac": 3, "cid": 7},
]
result = insert_cell_measure.delay(measure, entries, userid=1)
self.assertEqual(result.get(), 5)
measures = session.query(CellMeasure).all()
self.assertEqual(len(measures), 5)
self.assertEqual(set([m.mcc for m in measures]), set([1]))
self.assertEqual(set([m.mnc for m in measures]), set([2]))
self.assertEqual(set([m.asu for m in measures]), set([0, 8, 15]))
self.assertEqual(set([m.psc for m in measures]), set([0, 5]))
self.assertEqual(set([m.signal for m in measures]), set([0, -100]))
cells = session.query(Cell).all()
self.assertEqual(len(cells), 2)
self.assertEqual(set([c.mcc for c in cells]), set([1]))
self.assertEqual(set([c.mnc for c in cells]), set([2]))
self.assertEqual(set([c.lac for c in cells]), set([3]))
self.assertEqual(set([c.cid for c in cells]), set([4, 7]))
self.assertEqual(set([c.new_measures for c in cells]), set([1, 5]))
self.assertEqual(set([c.total_measures for c in cells]), set([1, 8]))
scores = session.query(Score).all()
self.assertEqual(len(scores), 1)
self.assertEqual(scores[0].key, SCORE_TYPE['new_cell'])
self.assertEqual(scores[0].value, 8)
# test duplicate execution
result = insert_cell_measure.delay(measure, entries, userid=1)
self.assertEqual(result.get(), 5)
# TODO this task isn't idempotent yet
measures = session.query(CellMeasure).all()
self.assertEqual(len(measures), 10)
def test_wifi(self):
from ichnaea.service.submit.tasks import insert_wifi_measure
session = self.db_master_session
utcnow = datetime.utcnow()
session.add(Wifi(key="ab12"))
session.add(Score(userid=1, key=SCORE_TYPE['new_wifi'], value=7))
session.flush()
measure = dict(
id=0, created=encode_datetime(utcnow), lat=10000000, lon=20000000,
time=encode_datetime(utcnow), accuracy=0, altitude=0,
altitude_accuracy=0, radio=-1,
)
entries = [
{"key": "ab12", "channel": 11, "signal": -80},
{"key": "ab12", "channel": 3, "signal": -90},
{"key": "ab12", "channel": 3, "signal": -80},
{"key": "cd34", "channel": 3, "signal": -90},
]
result = insert_wifi_measure.delay(measure, entries, userid=1)
self.assertEqual(result.get(), 4)
measures = session.query(WifiMeasure).all()
self.assertEqual(len(measures), 4)
self.assertEqual(set([m.key for m in measures]), set(["ab12", "cd34"]))
self.assertEqual(set([m.channel for m in measures]), set([3, 11]))
self.assertEqual(set([m.signal for m in measures]), set([-80, -90]))
wifis = session.query(Wifi).all()
self.assertEqual(len(wifis), 2)
self.assertEqual(set([w.key for w in wifis]), set(["ab12", "cd34"]))
self.assertEqual(set([w.new_measures for w in wifis]), set([1, 3]))
self.assertEqual(set([w.total_measures for w in wifis]), set([1, 3]))
scores = session.query(Score).all()
self.assertEqual(len(scores), 1)
self.assertEqual(scores[0].key, SCORE_TYPE['new_wifi'])
self.assertEqual(scores[0].value, 9)
# test duplicate execution
result = insert_wifi_measure.delay(measure, entries, userid=1)
self.assertEqual(result.get(), 4)
# TODO this task isn't idempotent yet
measures = session.query(WifiMeasure).all()
self.assertEqual(len(measures), 8)
# test error case
entries[0]['id'] = measures[0].id
result = insert_wifi_measure.delay(measure, entries)
self.assertEqual(result.get(), 0)
def test_time(self):
app = self.app
# test two weeks ago and "now"
time = (datetime.utcnow() - timedelta(14)).replace(microsecond=0)
tstr = encode_datetime(time)
app.post_json('/v1/submit', {
"items": [
{
"lat": 1.0,
"lon": 2.0,
"wifi": [{
"key": "a"
}],
"time": tstr
},
{
"lat": 2.0,
"lon": 3.0,
"wifi": [{
"key": "b"
}]
},
]
},
status=204)
session = self.db_master_session
result = session.query(Measure).all()
self.assertEqual(len(result), 2)
for item in result:
if '"key": "a"' in item.wifi:
self.assertEqual(item.time, time)
else:
self.assertEqual(item.time.date(), datetime.utcnow().date())
def test_time(self):
app = self.app
# test two weeks ago and "now"
time = (datetime.utcnow() - timedelta(14)).replace(microsecond=0)
tstr = encode_datetime(time)
app.post_json(
'/v1/submit', {"items": [
{"lat": 1.0,
"lon": 2.0,
"wifi": [{"key": "00aaaaaaaaaa"}],
"time": tstr},
{"lat": 2.0,
"lon": 3.0,
"wifi": [{"key": "00bbbbbbbbbb"}]},
]},
status=204)
session = self.db_master_session
result = session.query(WifiMeasure).all()
self.assertEqual(len(result), 2)
wifis = dict([(w.key, (w.created, w.time)) for w in result])
today = datetime.utcnow().date()
month_rounded_tday = time.replace(day=1, hour=0, minute=0, second=0)
month_rounded_today = today.replace(day=1)
self.assertEqual(wifis['00aaaaaaaaaa'][0].date(), today)
self.assertEqual(wifis['00aaaaaaaaaa'][1], month_rounded_tday)
self.assertEqual(wifis['00bbbbbbbbbb'][0].date(), today)
self.assertEqual(wifis['00bbbbbbbbbb'][1].date(), month_rounded_today)
def test_time(self):
app = self.app
# test two weeks ago and "now"
time = (datetime.utcnow() - timedelta(14)).replace(microsecond=0)
tday = time.replace(hour=0, minute=0, second=0)
tstr = encode_datetime(time)
app.post_json(
'/v1/submit', {"items": [
{"lat": 1.0, "lon": 2.0, "wifi": [{"key": "a"}], "time": tstr},
{"lat": 2.0, "lon": 3.0, "wifi": [{"key": "b"}]},
]},
status=204)
session = self.db_master_session
result = session.query(Measure).all()
self.assertEqual(len(result), 2)
today = datetime.utcnow().date()
for item in result:
self.assertEqual(item.created.date(), today)
self.assertEqual(item.created.hour, 0)
self.assertEqual(item.created.minute, 0)
self.assertEqual(item.created.second, 0)
if '"key": "a"' in item.wifi:
self.assertEqual(item.time, tday)
else:
self.assertEqual(item.time.date(), today)
def test_cell_out_of_range_values(self):
from ichnaea.service.submit.tasks import insert_cell_measures
session = self.db_master_session
time = datetime.utcnow().replace(microsecond=0) - timedelta(days=1)
measure = dict(id=0,
created=encode_datetime(time),
lat=10000000,
lon=20000000,
time=encode_datetime(time),
accuracy=0,
altitude=0,
altitude_accuracy=0,
radio=RADIO_TYPE['gsm'],
mcc=1,
mnc=2,
lac=3,
cid=4)
entries = [
{
"asu": 8,
"signal": -70,
"ta": 32
},
{
"asu": -10,
"signal": -300,
"ta": -10
},
{
"asu": 256,
"signal": 16,
"ta": 128
},
]
for e in entries:
e.update(measure)
result = insert_cell_measures.delay(entries)
self.assertEqual(result.get(), 3)
measures = session.query(CellMeasure).all()
self.assertEqual(len(measures), 3)
self.assertEqual(set([m.asu for m in measures]), set([-1, 8]))
self.assertEqual(set([m.signal for m in measures]), set([0, -70]))
self.assertEqual(set([m.ta for m in measures]), set([0, 32]))
def test_insert_invalid_lac(self):
from ichnaea.service.submit.tasks import insert_cell_measures
session = self.db_master_session
time = datetime.utcnow().replace(microsecond=0) - timedelta(days=1)
session.add(Cell(radio=RADIO_TYPE["gsm"], mcc=1, mnc=2, lac=3, cid=4, new_measures=2, total_measures=5))
session.add(Score(userid=1, key=SCORE_TYPE["new_cell"], value=7))
session.flush()
measure = dict(
id=0,
created=encode_datetime(time),
lat=10000000,
lon=20000000,
time=encode_datetime(time),
accuracy=0,
altitude=0,
altitude_accuracy=0,
radio=RADIO_TYPE["gsm"],
)
entries = [
{"mcc": 1, "mnc": 2, "lac": 3147483647, "cid": 2147483647, "psc": 5, "asu": 8},
{"mcc": 1, "mnc": 2, "lac": -1, "cid": -1, "psc": 5, "asu": 8},
]
for e in entries:
e.update(measure)
result = insert_cell_measures.delay(entries, userid=1)
self.assertEqual(result.get(), 2)
measures = session.query(CellMeasure).all()
self.assertEqual(len(measures), 2)
self.assertEqual(set([m.lac for m in measures]), set([-1]))
self.assertEqual(set([m.cid for m in measures]), set([-1]))
# Nothing should change in the initially created Cell record
cells = session.query(Cell).all()
self.assertEqual(len(cells), 1)
self.assertEqual(set([c.new_measures for c in cells]), set([2]))
self.assertEqual(set([c.total_measures for c in cells]), set([5]))
def process_measure(data, utcnow, session, userid=None):
measure = Measure()
measure.created = utcnow
measure.time = data["time"]
measure.lat = to_precise_int(data["lat"])
measure.lon = to_precise_int(data["lon"])
measure.accuracy = data["accuracy"]
measure.altitude = data["altitude"]
measure.altitude_accuracy = data["altitude_accuracy"]
measure.radio = RADIO_TYPE.get(data["radio"], -1)
# get measure.id set
session.add(measure)
session.flush()
measure_data = dict(
id=measure.id,
created=encode_datetime(measure.created),
lat=measure.lat,
lon=measure.lon,
time=encode_datetime(measure.time),
accuracy=measure.accuracy,
altitude=measure.altitude,
altitude_accuracy=measure.altitude_accuracy,
radio=measure.radio,
)
if data.get("cell"):
insert_cell_measure.delay(measure_data, data["cell"], userid=userid)
measure.cell = dumps(data["cell"])
if data.get("wifi"):
# filter out old-style sha1 hashes
too_long_keys = False
for w in data["wifi"]:
w["key"] = key = normalize_wifi_key(w["key"])
if len(key) > 12:
too_long_keys = True
break
if not too_long_keys:
insert_wifi_measure.delay(measure_data, data["wifi"], userid=userid)
measure.wifi = dumps(data["wifi"])
return measure
def test_time(self):
app = self.app
# test two weeks ago and "now"
time = (datetime.utcnow() - timedelta(14)).replace(microsecond=0)
tstr = encode_datetime(time)
app.post_json('/v1/submit', {
"items": [
{
"lat": 1.0,
"lon": 2.0,
"wifi": [{
"key": "00aaaaaaaaaa"
}],
"time": tstr
},
{
"lat": 2.0,
"lon": 3.0,
"wifi": [{
"key": "00bbbbbbbbbb"
}]
},
]
},
status=204)
session = self.db_master_session
result = session.query(WifiMeasure).all()
self.assertEqual(len(result), 2)
wifis = dict([(w.key, (w.created, w.time)) for w in result])
today = datetime.utcnow().date()
month_rounded_tday = time.replace(day=1, hour=0, minute=0, second=0)
month_rounded_today = today.replace(day=1)
self.assertEqual(wifis['00aaaaaaaaaa'][0].date(), today)
self.assertEqual(wifis['00aaaaaaaaaa'][1], month_rounded_tday)
self.assertEqual(wifis['00bbbbbbbbbb'][0].date(), today)
self.assertEqual(wifis['00bbbbbbbbbb'][1].date(), month_rounded_today)
def process_measure(measure_id, data, session):
cell_measures = []
wifi_measures = []
measure_data = dict(
measure_id=measure_id,
lat=to_precise_int(data['lat']),
lon=to_precise_int(data['lon']),
time=encode_datetime(data['time']),
accuracy=data['accuracy'],
altitude=data['altitude'],
altitude_accuracy=data['altitude_accuracy'],
)
measure_radio = RADIO_TYPE.get(data['radio'], -1)
if data.get('cell'):
# flatten measure / cell data into a single dict
for c in data['cell']:
c.update(measure_data)
# use more specific cell type or
# fall back to less precise measure
if c['radio'] != '':
c['radio'] = RADIO_TYPE.get(c['radio'], -1)
else:
c['radio'] = measure_radio
cell_measures = data['cell']
if data.get('wifi'):
# filter out old-style sha1 hashes
invalid_wifi_key = False
for w in data['wifi']:
w['key'] = key = normalize_wifi_key(w['key'])
if not valid_wifi_pattern(key):
invalid_wifi_key = True
break
if not invalid_wifi_key:
# flatten measure / wifi data into a single dict
for w in data['wifi']:
w.update(measure_data)
wifi_measures = data['wifi']
return (cell_measures, wifi_measures)
def process_measure(measure_id, data, session):
cell_measures = []
wifi_measures = []
measure_data = dict(
measure_id=measure_id,
lat=to_precise_int(data['lat']),
lon=to_precise_int(data['lon']),
time=encode_datetime(data['time']),
accuracy=data['accuracy'],
altitude=data['altitude'],
altitude_accuracy=data['altitude_accuracy'],
)
measure_radio = RADIO_TYPE.get(data['radio'], -1)
if data.get('cell'):
# flatten measure / cell data into a single dict
for c in data['cell']:
c.update(measure_data)
# use more specific cell type or
# fall back to less precise measure
if c['radio'] != '':
c['radio'] = RADIO_TYPE.get(c['radio'], -1)
else:
c['radio'] = measure_radio
cell_measures = data['cell']
if data.get('wifi'):
# filter out old-style sha1 hashes
invalid_wifi_key = False
for w in data['wifi']:
w['key'] = key = normalize_wifi_key(w['key'])
if not valid_wifi_pattern(key):
invalid_wifi_key = True
break
if not invalid_wifi_key:
# flatten measure / wifi data into a single dict
for w in data['wifi']:
w.update(measure_data)
wifi_measures = data['wifi']
return (cell_measures, wifi_measures)
def test_cell(self):
from ichnaea.service.submit.tasks import insert_cell_measure
session = self.db_master_session
utcnow = datetime.utcnow()
session.add(
Cell(radio=0,
mcc=1,
mnc=2,
lac=3,
cid=4,
new_measures=2,
total_measures=5))
session.add(Score(userid=1, key=SCORE_TYPE['new_cell'], value=7))
session.flush()
measure = dict(
id=0,
created=encode_datetime(utcnow),
lat=10000000,
lon=20000000,
time=encode_datetime(utcnow),
accuracy=0,
altitude=0,
altitude_accuracy=0,
radio=0,
)
entries = [
{
"mcc": 1,
"mnc": 2,
"signal": -100
},
{
"mcc": 1,
"mnc": 2,
"lac": 3,
"cid": 4,
"psc": 5,
"asu": 8
},
{
"mcc": 1,
"mnc": 2,
"lac": 3,
"cid": 4,
"asu": 8
},
{
"mcc": 1,
"mnc": 2,
"lac": 3,
"cid": 4,
"asu": 15
},
{
"mcc": 1,
"mnc": 2,
"lac": 3,
"cid": 7
},
]
result = insert_cell_measure.delay(measure, entries, userid=1)
self.assertEqual(result.get(), 5)
measures = session.query(CellMeasure).all()
self.assertEqual(len(measures), 5)
self.assertEqual(set([m.mcc for m in measures]), set([1]))
self.assertEqual(set([m.mnc for m in measures]), set([2]))
self.assertEqual(set([m.asu for m in measures]), set([0, 8, 15]))
self.assertEqual(set([m.psc for m in measures]), set([0, 5]))
self.assertEqual(set([m.signal for m in measures]), set([0, -100]))
cells = session.query(Cell).all()
self.assertEqual(len(cells), 2)
self.assertEqual(set([c.mcc for c in cells]), set([1]))
self.assertEqual(set([c.mnc for c in cells]), set([2]))
self.assertEqual(set([c.lac for c in cells]), set([3]))
self.assertEqual(set([c.cid for c in cells]), set([4, 7]))
self.assertEqual(set([c.new_measures for c in cells]), set([1, 5]))
self.assertEqual(set([c.total_measures for c in cells]), set([1, 8]))
scores = session.query(Score).all()
self.assertEqual(len(scores), 1)
self.assertEqual(scores[0].key, SCORE_TYPE['new_cell'])
self.assertEqual(scores[0].value, 10)
# test duplicate execution
result = insert_cell_measure.delay(measure, entries, userid=1)
self.assertEqual(result.get(), 5)
# TODO this task isn't idempotent yet
measures = session.query(CellMeasure).all()
self.assertEqual(len(measures), 10)
def test_cell(self):
from ichnaea.service.submit.tasks import insert_cell_measures
session = self.db_master_session
time = datetime.utcnow().replace(microsecond=0) - timedelta(days=1)
session.add(Cell(radio=RADIO_TYPE["gsm"], mcc=1, mnc=2, lac=3, cid=4, psc=5, new_measures=2, total_measures=5))
session.add(Score(userid=1, key=SCORE_TYPE["new_cell"], value=7))
session.flush()
measure = dict(
id=0,
created=encode_datetime(time),
lat=10000000,
lon=20000000,
time=encode_datetime(time),
accuracy=0,
altitude=0,
altitude_accuracy=0,
radio=RADIO_TYPE["gsm"],
)
entries = [
# Note that this first entry will be skipped as it does
# not include (lac, cid) or (psc)
{"mcc": 1, "mnc": 2, "signal": -100},
{"mcc": 1, "mnc": 2, "lac": 3, "cid": 4, "psc": 5, "asu": 8},
{"mcc": 1, "mnc": 2, "lac": 3, "cid": 4, "psc": 5, "asu": 8},
{"mcc": 1, "mnc": 2, "lac": 3, "cid": 4, "psc": 5, "asu": 15},
{"mcc": 1, "mnc": 2, "lac": 3, "cid": 7, "psc": 5},
]
for e in entries:
e.update(measure)
result = insert_cell_measures.delay(entries, userid=1)
self.assertEqual(result.get(), 4)
measures = session.query(CellMeasure).all()
self.assertEqual(len(measures), 4)
self.assertEqual(set([m.mcc for m in measures]), set([1]))
self.assertEqual(set([m.mnc for m in measures]), set([2]))
self.assertEqual(set([m.asu for m in measures]), set([-1, 8, 15]))
self.assertEqual(set([m.psc for m in measures]), set([5]))
self.assertEqual(set([m.signal for m in measures]), set([0]))
cells = session.query(Cell).all()
self.assertEqual(len(cells), 2)
self.assertEqual(set([c.mcc for c in cells]), set([1]))
self.assertEqual(set([c.mnc for c in cells]), set([2]))
self.assertEqual(set([c.lac for c in cells]), set([3]))
self.assertEqual(set([c.cid for c in cells]), set([4, 7]))
self.assertEqual(set([c.psc for c in cells]), set([5]))
self.assertEqual(set([c.new_measures for c in cells]), set([1, 5]))
self.assertEqual(set([c.total_measures for c in cells]), set([1, 8]))
scores = session.query(Score).all()
self.assertEqual(len(scores), 1)
self.assertEqual(scores[0].key, SCORE_TYPE["new_cell"])
self.assertEqual(scores[0].value, 8)
# test duplicate execution
result = insert_cell_measures.delay(entries, userid=1)
self.assertEqual(result.get(), 4)
# TODO this task isn't idempotent yet
measures = session.query(CellMeasure).all()
self.assertEqual(len(measures), 8)
def test_ignore_unhelpful_incomplete_cells(self):
# Cell records must have MNC, MCC and at least one of (LAC, CID) or PSC
# values filled in.
from ichnaea.service.submit.tasks import insert_cell_measures
session = self.db_master_session
time = datetime.utcnow().replace(microsecond=0) - timedelta(days=1)
measure = dict(
id=0,
created=encode_datetime(time),
lat=10000000,
lon=20000000,
time=encode_datetime(time),
accuracy=0,
altitude=0,
altitude_accuracy=0,
radio=RADIO_TYPE['gsm'],
)
entries = [
# These records are valid
{
"mcc": 1,
"mnc": 2,
"lac": 3,
"cid": 4
},
{
"mcc": 1,
"mnc": 2,
"lac": 3,
"cid": 4,
"psc": 5
},
# This record is missing everything
{},
# These records fail the mcc check
{
"mnc": 2,
"lac": 3,
"cid": 4
},
{
"mcc": 0,
"mnc": 2,
"lac": 3,
"cid": 4
},
{
"mcc": -1,
"mnc": 2,
"lac": 3,
"cid": 4
},
{
"mcc": -2,
"mnc": 2,
"lac": 3,
"cid": 4
},
{
"mcc": 2000,
"mnc": 2,
"lac": 3,
"cid": 4
},
# These records fail the mnc check
{
"mcc": 1,
"lac": 3,
"cid": 4
},
{
"mcc": 1,
"mnc": -1,
"lac": 3,
"cid": 4
},
{
"mcc": 1,
"mnc": -2,
"lac": 3,
"cid": 4
},
{
"mcc": 1,
"mnc": 33000,
"lac": 3,
"cid": 4
},
# These records fail the lac check
{
"mcc": 1,
"mnc": 2,
"cid": 4
},
{
"mcc": 1,
"mnc": 2,
"lac": -1,
"cid": 4
},
{
"mcc": 1,
"mnc": 2,
"lac": -2,
"cid": 4
},
{
"mcc": 1,
"mnc": 2,
"lac": 65536,
"cid": 4
},
# These records fail the cid check
{
"mcc": 1,
"mnc": 2,
"lac": 3
},
{
"mcc": 1,
"mnc": 2,
"lac": 3,
"cid": -1
},
{
"mcc": 1,
"mnc": 2,
"lac": 3,
"cid": -2
},
{
"mcc": 1,
"mnc": 2,
"lac": 3,
"cid": 2**28
},
# These records fail the (lac or cid) and psc check
{
"mcc": 1,
"mnc": 2
},
{
"mcc": 1,
"mnc": 2,
"lac": 3
},
{
"mcc": 1,
"mnc": 2,
"cid": 4
},
]
for e in entries:
e.update(measure)
result = insert_cell_measures.delay(entries, userid=1)
self.assertEqual(result.get(), 2)
measures = session.query(CellMeasure).all()
self.assertEqual(len(measures), 2)
cells = session.query(Cell).all()
self.assertEqual(len(cells), 1)
entries = [
# These records are valid
{
"mcc": 1,
"mnc": 2,
"psc": 5
},
{
"mcc": 1,
"mnc": 2,
"lac": 3,
"psc": 5
},
{
"mcc": 1,
"mnc": 2,
"cid": 4,
"psc": 5
},
]
for e in entries:
e.update(measure)
result = insert_cell_measures.delay(entries, userid=1)
self.assertEqual(result.get(), 3)
measures = session.query(CellMeasure).all()
self.assertEqual(len(measures), 5)
cells = session.query(Cell).all()
self.assertEqual(len(cells), 1)
def test_ignore_unhelpful_incomplete_cells(self):
# Cell records must have MNC, MCC and at least one of (LAC, CID) or PSC
# values filled in.
from ichnaea.service.submit.tasks import insert_cell_measures
session = self.db_master_session
time = datetime.utcnow().replace(microsecond=0) - timedelta(days=1)
measure = dict(
id=0, created=encode_datetime(time), lat=10000000, lon=20000000,
time=encode_datetime(time), accuracy=0, altitude=0,
altitude_accuracy=0, radio=RADIO_TYPE['gsm'],
)
entries = [
# These records are valid
{"mcc": 1, "mnc": 2, "lac": 3, "cid": 4},
{"mcc": 1, "mnc": 2, "lac": 3, "cid": 4, "psc": 5},
# This record is missing everything
{},
# These records fail the mcc check
{"mnc": 2, "lac": 3, "cid": 4},
{"mcc": 0, "mnc": 2, "lac": 3, "cid": 4},
{"mcc": -1, "mnc": 2, "lac": 3, "cid": 4},
{"mcc": -2, "mnc": 2, "lac": 3, "cid": 4},
{"mcc": 2000, "mnc": 2, "lac": 3, "cid": 4},
# These records fail the mnc check
{"mcc": 1, "lac": 3, "cid": 4},
{"mcc": 1, "mnc": -1, "lac": 3, "cid": 4},
{"mcc": 1, "mnc": -2, "lac": 3, "cid": 4},
{"mcc": 1, "mnc": 33000, "lac": 3, "cid": 4},
# These records fail the lac check
{"mcc": 1, "mnc": 2, "cid": 4},
{"mcc": 1, "mnc": 2, "lac": -1, "cid": 4},
{"mcc": 1, "mnc": 2, "lac": -2, "cid": 4},
{"mcc": 1, "mnc": 2, "lac": 65536, "cid": 4},
# These records fail the cid check
{"mcc": 1, "mnc": 2, "lac": 3},
{"mcc": 1, "mnc": 2, "lac": 3, "cid": -1},
{"mcc": 1, "mnc": 2, "lac": 3, "cid": -2},
{"mcc": 1, "mnc": 2, "lac": 3, "cid": 2 ** 28},
# These records fail the (lac or cid) and psc check
{"mcc": 1, "mnc": 2},
{"mcc": 1, "mnc": 2, "lac": 3},
{"mcc": 1, "mnc": 2, "cid": 4},
]
for e in entries:
e.update(measure)
result = insert_cell_measures.delay(entries, userid=1)
self.assertEqual(result.get(), 2)
measures = session.query(CellMeasure).all()
self.assertEqual(len(measures), 2)
cells = session.query(Cell).all()
self.assertEqual(len(cells), 1)
entries = [
# These records are valid
{"mcc": 1, "mnc": 2, "psc": 5},
{"mcc": 1, "mnc": 2, "lac": 3, "psc": 5},
{"mcc": 1, "mnc": 2, "cid": 4, "psc": 5},
]
for e in entries:
e.update(measure)
result = insert_cell_measures.delay(entries, userid=1)
self.assertEqual(result.get(), 3)
measures = session.query(CellMeasure).all()
self.assertEqual(len(measures), 5)
cells = session.query(Cell).all()
self.assertEqual(len(cells), 1)
def test_cell(self):
from ichnaea.service.submit.tasks import insert_cell_measures
session = self.db_master_session
time = datetime.utcnow().replace(microsecond=0) - timedelta(days=1)
session.add(
Cell(radio=RADIO_TYPE['gsm'],
mcc=1,
mnc=2,
lac=3,
cid=4,
psc=5,
new_measures=2,
total_measures=5))
session.add(Score(userid=1, key=SCORE_TYPE['new_cell'], value=7))
session.flush()
measure = dict(
id=0,
created=encode_datetime(time),
lat=10000000,
lon=20000000,
time=encode_datetime(time),
accuracy=0,
altitude=0,
altitude_accuracy=0,
radio=RADIO_TYPE['gsm'],
)
entries = [
# Note that this first entry will be skipped as it does
# not include (lac, cid) or (psc)
{
"mcc": 1,
"mnc": 2,
"signal": -100
},
{
"mcc": 1,
"mnc": 2,
"lac": 3,
"cid": 4,
"psc": 5,
"asu": 8
},
{
"mcc": 1,
"mnc": 2,
"lac": 3,
"cid": 4,
"psc": 5,
"asu": 8
},
{
"mcc": 1,
"mnc": 2,
"lac": 3,
"cid": 4,
"psc": 5,
"asu": 15
},
{
"mcc": 1,
"mnc": 2,
"lac": 3,
"cid": 7,
"psc": 5
},
]
for e in entries:
e.update(measure)
result = insert_cell_measures.delay(entries, userid=1)
self.assertEqual(result.get(), 4)
measures = session.query(CellMeasure).all()
self.assertEqual(len(measures), 4)
self.assertEqual(set([m.mcc for m in measures]), set([1]))
self.assertEqual(set([m.mnc for m in measures]), set([2]))
self.assertEqual(set([m.asu for m in measures]), set([-1, 8, 15]))
self.assertEqual(set([m.psc for m in measures]), set([5]))
self.assertEqual(set([m.signal for m in measures]), set([0]))
cells = session.query(Cell).all()
self.assertEqual(len(cells), 2)
self.assertEqual(set([c.mcc for c in cells]), set([1]))
self.assertEqual(set([c.mnc for c in cells]), set([2]))
self.assertEqual(set([c.lac for c in cells]), set([3]))
self.assertEqual(set([c.cid for c in cells]), set([4, 7]))
self.assertEqual(set([c.psc for c in cells]), set([5]))
self.assertEqual(set([c.new_measures for c in cells]), set([1, 5]))
self.assertEqual(set([c.total_measures for c in cells]), set([1, 8]))
scores = session.query(Score).all()
self.assertEqual(len(scores), 1)
self.assertEqual(scores[0].key, SCORE_TYPE['new_cell'])
self.assertEqual(scores[0].value, 8)
# test duplicate execution
result = insert_cell_measures.delay(entries, userid=1)
self.assertEqual(result.get(), 4)
# TODO this task isn't idempotent yet
measures = session.query(CellMeasure).all()
self.assertEqual(len(measures), 8)
def test_wifi(self):
from ichnaea.service.submit.tasks import insert_wifi_measures
session = self.db_master_session
time = datetime.utcnow().replace(microsecond=0) - timedelta(days=1)
session.add(Wifi(key="ab12"))
session.add(Score(userid=1, key=SCORE_TYPE['new_wifi'], value=7))
session.flush()
measure = dict(
id=0,
created=encode_datetime(time),
lat=10000000,
lon=20000000,
time=encode_datetime(time),
accuracy=0,
altitude=0,
altitude_accuracy=0,
radio=-1,
)
entries = [
{
"key": "ab12",
"channel": 11,
"signal": -80
},
{
"key": "ab12",
"channel": 3,
"signal": -90
},
{
"key": "ab12",
"channel": 3,
"signal": -80
},
{
"key": "cd34",
"channel": 3,
"signal": -90
},
]
for e in entries:
e.update(measure)
result = insert_wifi_measures.delay(entries, userid=1)
self.assertEqual(result.get(), 4)
measures = session.query(WifiMeasure).all()
self.assertEqual(len(measures), 4)
self.assertEqual(set([m.key for m in measures]), set(["ab12", "cd34"]))
self.assertEqual(set([m.channel for m in measures]), set([3, 11]))
self.assertEqual(set([m.signal for m in measures]), set([-80, -90]))
wifis = session.query(Wifi).all()
self.assertEqual(len(wifis), 2)
self.assertEqual(set([w.key for w in wifis]), set(["ab12", "cd34"]))
self.assertEqual(set([w.new_measures for w in wifis]), set([1, 3]))
self.assertEqual(set([w.total_measures for w in wifis]), set([1, 3]))
scores = session.query(Score).all()
self.assertEqual(len(scores), 1)
self.assertEqual(scores[0].key, SCORE_TYPE['new_wifi'])
self.assertEqual(scores[0].value, 8)
# test duplicate execution
result = insert_wifi_measures.delay(entries, userid=1)
self.assertEqual(result.get(), 4)
# TODO this task isn't idempotent yet
measures = session.query(WifiMeasure).all()
self.assertEqual(len(measures), 8)