def test_cell_out_of_range_values(self): from ichnaea.service.submit.tasks import insert_cell_measures session = self.db_master_session time = datetime.utcnow().replace(microsecond=0) - timedelta(days=1) measure = dict( id=0, created=encode_datetime(time), lat=10000000, lon=20000000, time=encode_datetime(time), accuracy=0, altitude=0, altitude_accuracy=0, radio=RADIO_TYPE['gsm'], mcc=1, mnc=2, lac=3, cid=4) entries = [ {"asu": 8, "signal": -70, "ta": 32}, {"asu": -10, "signal": -300, "ta": -10}, {"asu": 256, "signal": 16, "ta": 128}, ] for e in entries: e.update(measure) result = insert_cell_measures.delay(entries) self.assertEqual(result.get(), 3) measures = session.query(CellMeasure).all() self.assertEqual(len(measures), 3) self.assertEqual(set([m.asu for m in measures]), set([-1, 8])) self.assertEqual(set([m.signal for m in measures]), set([0, -70])) self.assertEqual(set([m.ta for m in measures]), set([0, 32]))
def test_ignore_unhelpful_incomplete_cdma_cells(self): # CDMA cell records must have MNC, MCC, LAC and CID filled in from ichnaea.service.submit.tasks import insert_cell_measures session = self.db_master_session time = datetime.utcnow().replace(microsecond=0) - timedelta(days=1) measure = dict( id=0, created=encode_datetime(time), lat=10000000, lon=20000000, time=encode_datetime(time), accuracy=0, altitude=0, altitude_accuracy=0, radio=RADIO_TYPE['cdma'], ) entries = [ # This records is valid {"mcc": 1, "mnc": 2, "lac": 3, "cid": 4}, # This record should fail as it's missing CID {"mcc": 1, "mnc": 2, "lac": 3}, # This fails for missing lac {"mcc": 1, "mnc": 2, "cid": 4}, # Adding a psc doesn't change things {"mcc": 1, "mnc": 2, "psc": 5}, ] for e in entries: e.update(measure) result = insert_cell_measures.delay(entries, userid=1) self.assertEqual(result.get(), 1) measures = session.query(CellMeasure).all() self.assertEqual(len(measures), 1) cells = session.query(Cell).all() self.assertEqual(len(cells), 1)
def reprocess_cell_measure(self, measure_ids, userid=None): measures = [] try: with self.db_session() as session: measures = session.query(Measure).filter( Measure.id.in_(measure_ids)).filter( Measure.cell != sql_null).all() for measure in measures: measure_data = dict( id=measure.id, created=encode_datetime(measure.created), lat=measure.lat, lon=measure.lon, time=encode_datetime(measure.time), accuracy=measure.accuracy, altitude=measure.altitude, altitude_accuracy=measure.altitude_accuracy, radio=measure.radio, ) # adds data to this session process_cell_measure( session, measure_data, loads(measure.cell), userid=userid) session.commit() return len(measures) except IntegrityError as exc: # pragma: no cover logger.exception('error') return 0 except Exception as exc: # pragma: no cover raise self.retry(exc=exc)
def test_insert_invalid_lac(self): from ichnaea.service.submit.tasks import insert_cell_measures session = self.db_master_session time = datetime.utcnow().replace(microsecond=0) - timedelta(days=1) session.add( Cell(radio=RADIO_TYPE['gsm'], mcc=1, mnc=2, lac=3, cid=4, new_measures=2, total_measures=5)) session.add(Score(userid=1, key=SCORE_TYPE['new_cell'], value=7)) session.flush() measure = dict(id=0, created=encode_datetime(time), lat=10000000, lon=20000000, time=encode_datetime(time), accuracy=0, altitude=0, altitude_accuracy=0, radio=RADIO_TYPE['gsm']) entries = [ { "mcc": 1, "mnc": 2, "lac": 3147483647, "cid": 2147483647, "psc": 5, "asu": 8 }, { "mcc": 1, "mnc": 2, "lac": -1, "cid": -1, "psc": 5, "asu": 8 }, ] for e in entries: e.update(measure) result = insert_cell_measures.delay(entries, userid=1) self.assertEqual(result.get(), 2) measures = session.query(CellMeasure).all() self.assertEqual(len(measures), 2) self.assertEqual(set([m.lac for m in measures]), set([-1])) self.assertEqual(set([m.cid for m in measures]), set([-1])) # Nothing should change in the initially created Cell record cells = session.query(Cell).all() self.assertEqual(len(cells), 1) self.assertEqual(set([c.new_measures for c in cells]), set([2])) self.assertEqual(set([c.total_measures for c in cells]), set([5]))
def process_wifi(entries, measure): measure_data = dict( id=measure.id, created=encode_datetime(measure.created), lat=measure.lat, lon=measure.lon, time=encode_datetime(measure.time), accuracy=measure.accuracy, altitude=measure.altitude, altitude_accuracy=measure.altitude_accuracy, ) insert_wifi_measure.delay(measure_data, entries)
def test_ignore_unhelpful_incomplete_cdma_cells(self): # CDMA cell records must have MNC, MCC, LAC and CID filled in from ichnaea.service.submit.tasks import insert_cell_measures session = self.db_master_session time = datetime.utcnow().replace(microsecond=0) - timedelta(days=1) measure = dict( id=0, created=encode_datetime(time), lat=10000000, lon=20000000, time=encode_datetime(time), accuracy=0, altitude=0, altitude_accuracy=0, radio=RADIO_TYPE['cdma'], ) entries = [ # This records is valid { "mcc": 1, "mnc": 2, "lac": 3, "cid": 4 }, # This record should fail as it's missing CID { "mcc": 1, "mnc": 2, "lac": 3 }, # This fails for missing lac { "mcc": 1, "mnc": 2, "cid": 4 }, # Adding a psc doesn't change things { "mcc": 1, "mnc": 2, "psc": 5 }, ] for e in entries: e.update(measure) result = insert_cell_measures.delay(entries, userid=1) self.assertEqual(result.get(), 1) measures = session.query(CellMeasure).all() self.assertEqual(len(measures), 1) cells = session.query(Cell).all() self.assertEqual(len(cells), 1)
def test_wifi(self): from ichnaea.service.submit.tasks import insert_wifi_measures session = self.db_master_session time = datetime.utcnow().replace(microsecond=0) - timedelta(days=1) session.add(Wifi(key="ab1234567890")) session.add(Score(userid=1, key=SCORE_TYPE["new_wifi"], value=7)) session.flush() measure = dict( id=0, created=encode_datetime(time), lat=10000000, lon=20000000, time=encode_datetime(time), accuracy=0, altitude=0, altitude_accuracy=0, radio=-1, ) entries = [ {"key": "ab1234567890", "channel": 11, "signal": -80}, {"key": "ab1234567890", "channel": 3, "signal": -90}, {"key": "ab1234567890", "channel": 3, "signal": -80}, {"key": "cd3456789012", "channel": 3, "signal": -90}, ] for e in entries: e.update(measure) result = insert_wifi_measures.delay(entries, userid=1) self.assertEqual(result.get(), 4) measures = session.query(WifiMeasure).all() self.assertEqual(len(measures), 4) self.assertEqual(set([m.key for m in measures]), set(["ab1234567890", "cd3456789012"])) self.assertEqual(set([m.channel for m in measures]), set([3, 11])) self.assertEqual(set([m.signal for m in measures]), set([-80, -90])) wifis = session.query(Wifi).all() self.assertEqual(len(wifis), 2) self.assertEqual(set([w.key for w in wifis]), set(["ab1234567890", "cd3456789012"])) self.assertEqual(set([w.new_measures for w in wifis]), set([1, 3])) self.assertEqual(set([w.total_measures for w in wifis]), set([1, 3])) scores = session.query(Score).all() self.assertEqual(len(scores), 1) self.assertEqual(scores[0].key, SCORE_TYPE["new_wifi"]) self.assertEqual(scores[0].value, 8) # test duplicate execution result = insert_wifi_measures.delay(entries, userid=1) self.assertEqual(result.get(), 4) # TODO this task isn't idempotent yet measures = session.query(WifiMeasure).all() self.assertEqual(len(measures), 8)
def test_cell(self): from ichnaea.service.submit.tasks import insert_cell_measure session = self.db_master_session time = datetime.utcnow().replace(microsecond=0) - timedelta(days=1) session.add(Cell(radio=0, mcc=1, mnc=2, lac=3, cid=4, new_measures=2, total_measures=5)) session.add(Score(userid=1, key=SCORE_TYPE['new_cell'], value=7)) session.flush() measure = dict( id=0, created=encode_datetime(time), lat=10000000, lon=20000000, time=encode_datetime(time), accuracy=0, altitude=0, altitude_accuracy=0, radio=0, ) entries = [ {"mcc": 1, "mnc": 2, "signal": -100}, {"mcc": 1, "mnc": 2, "lac": 3, "cid": 4, "psc": 5, "asu": 8}, {"mcc": 1, "mnc": 2, "lac": 3, "cid": 4, "asu": 8}, {"mcc": 1, "mnc": 2, "lac": 3, "cid": 4, "asu": 15}, {"mcc": 1, "mnc": 2, "lac": 3, "cid": 7}, ] result = insert_cell_measure.delay(measure, entries, userid=1) self.assertEqual(result.get(), 5) measures = session.query(CellMeasure).all() self.assertEqual(len(measures), 5) self.assertEqual(set([m.mcc for m in measures]), set([1])) self.assertEqual(set([m.mnc for m in measures]), set([2])) self.assertEqual(set([m.asu for m in measures]), set([0, 8, 15])) self.assertEqual(set([m.psc for m in measures]), set([0, 5])) self.assertEqual(set([m.signal for m in measures]), set([0, -100])) cells = session.query(Cell).all() self.assertEqual(len(cells), 2) self.assertEqual(set([c.mcc for c in cells]), set([1])) self.assertEqual(set([c.mnc for c in cells]), set([2])) self.assertEqual(set([c.lac for c in cells]), set([3])) self.assertEqual(set([c.cid for c in cells]), set([4, 7])) self.assertEqual(set([c.new_measures for c in cells]), set([1, 5])) self.assertEqual(set([c.total_measures for c in cells]), set([1, 8])) scores = session.query(Score).all() self.assertEqual(len(scores), 1) self.assertEqual(scores[0].key, SCORE_TYPE['new_cell']) self.assertEqual(scores[0].value, 8) # test duplicate execution result = insert_cell_measure.delay(measure, entries, userid=1) self.assertEqual(result.get(), 5) # TODO this task isn't idempotent yet measures = session.query(CellMeasure).all() self.assertEqual(len(measures), 10)
def test_wifi(self): from ichnaea.service.submit.tasks import insert_wifi_measure session = self.db_master_session utcnow = datetime.utcnow() session.add(Wifi(key="ab12")) session.add(Score(userid=1, key=SCORE_TYPE['new_wifi'], value=7)) session.flush() measure = dict( id=0, created=encode_datetime(utcnow), lat=10000000, lon=20000000, time=encode_datetime(utcnow), accuracy=0, altitude=0, altitude_accuracy=0, radio=-1, ) entries = [ {"key": "ab12", "channel": 11, "signal": -80}, {"key": "ab12", "channel": 3, "signal": -90}, {"key": "ab12", "channel": 3, "signal": -80}, {"key": "cd34", "channel": 3, "signal": -90}, ] result = insert_wifi_measure.delay(measure, entries, userid=1) self.assertEqual(result.get(), 4) measures = session.query(WifiMeasure).all() self.assertEqual(len(measures), 4) self.assertEqual(set([m.key for m in measures]), set(["ab12", "cd34"])) self.assertEqual(set([m.channel for m in measures]), set([3, 11])) self.assertEqual(set([m.signal for m in measures]), set([-80, -90])) wifis = session.query(Wifi).all() self.assertEqual(len(wifis), 2) self.assertEqual(set([w.key for w in wifis]), set(["ab12", "cd34"])) self.assertEqual(set([w.new_measures for w in wifis]), set([1, 3])) self.assertEqual(set([w.total_measures for w in wifis]), set([1, 3])) scores = session.query(Score).all() self.assertEqual(len(scores), 1) self.assertEqual(scores[0].key, SCORE_TYPE['new_wifi']) self.assertEqual(scores[0].value, 9) # test duplicate execution result = insert_wifi_measure.delay(measure, entries, userid=1) self.assertEqual(result.get(), 4) # TODO this task isn't idempotent yet measures = session.query(WifiMeasure).all() self.assertEqual(len(measures), 8) # test error case entries[0]['id'] = measures[0].id result = insert_wifi_measure.delay(measure, entries) self.assertEqual(result.get(), 0)
def test_time(self): app = self.app # test two weeks ago and "now" time = (datetime.utcnow() - timedelta(14)).replace(microsecond=0) tstr = encode_datetime(time) app.post_json('/v1/submit', { "items": [ { "lat": 1.0, "lon": 2.0, "wifi": [{ "key": "a" }], "time": tstr }, { "lat": 2.0, "lon": 3.0, "wifi": [{ "key": "b" }] }, ] }, status=204) session = self.db_master_session result = session.query(Measure).all() self.assertEqual(len(result), 2) for item in result: if '"key": "a"' in item.wifi: self.assertEqual(item.time, time) else: self.assertEqual(item.time.date(), datetime.utcnow().date())
def test_time(self): app = self.app # test two weeks ago and "now" time = (datetime.utcnow() - timedelta(14)).replace(microsecond=0) tstr = encode_datetime(time) app.post_json( '/v1/submit', {"items": [ {"lat": 1.0, "lon": 2.0, "wifi": [{"key": "00aaaaaaaaaa"}], "time": tstr}, {"lat": 2.0, "lon": 3.0, "wifi": [{"key": "00bbbbbbbbbb"}]}, ]}, status=204) session = self.db_master_session result = session.query(WifiMeasure).all() self.assertEqual(len(result), 2) wifis = dict([(w.key, (w.created, w.time)) for w in result]) today = datetime.utcnow().date() month_rounded_tday = time.replace(day=1, hour=0, minute=0, second=0) month_rounded_today = today.replace(day=1) self.assertEqual(wifis['00aaaaaaaaaa'][0].date(), today) self.assertEqual(wifis['00aaaaaaaaaa'][1], month_rounded_tday) self.assertEqual(wifis['00bbbbbbbbbb'][0].date(), today) self.assertEqual(wifis['00bbbbbbbbbb'][1].date(), month_rounded_today)
def test_time(self): app = self.app # test two weeks ago and "now" time = (datetime.utcnow() - timedelta(14)).replace(microsecond=0) tday = time.replace(hour=0, minute=0, second=0) tstr = encode_datetime(time) app.post_json( '/v1/submit', {"items": [ {"lat": 1.0, "lon": 2.0, "wifi": [{"key": "a"}], "time": tstr}, {"lat": 2.0, "lon": 3.0, "wifi": [{"key": "b"}]}, ]}, status=204) session = self.db_master_session result = session.query(Measure).all() self.assertEqual(len(result), 2) today = datetime.utcnow().date() for item in result: self.assertEqual(item.created.date(), today) self.assertEqual(item.created.hour, 0) self.assertEqual(item.created.minute, 0) self.assertEqual(item.created.second, 0) if '"key": "a"' in item.wifi: self.assertEqual(item.time, tday) else: self.assertEqual(item.time.date(), today)
def test_cell_out_of_range_values(self): from ichnaea.service.submit.tasks import insert_cell_measures session = self.db_master_session time = datetime.utcnow().replace(microsecond=0) - timedelta(days=1) measure = dict(id=0, created=encode_datetime(time), lat=10000000, lon=20000000, time=encode_datetime(time), accuracy=0, altitude=0, altitude_accuracy=0, radio=RADIO_TYPE['gsm'], mcc=1, mnc=2, lac=3, cid=4) entries = [ { "asu": 8, "signal": -70, "ta": 32 }, { "asu": -10, "signal": -300, "ta": -10 }, { "asu": 256, "signal": 16, "ta": 128 }, ] for e in entries: e.update(measure) result = insert_cell_measures.delay(entries) self.assertEqual(result.get(), 3) measures = session.query(CellMeasure).all() self.assertEqual(len(measures), 3) self.assertEqual(set([m.asu for m in measures]), set([-1, 8])) self.assertEqual(set([m.signal for m in measures]), set([0, -70])) self.assertEqual(set([m.ta for m in measures]), set([0, 32]))
def test_insert_invalid_lac(self): from ichnaea.service.submit.tasks import insert_cell_measures session = self.db_master_session time = datetime.utcnow().replace(microsecond=0) - timedelta(days=1) session.add(Cell(radio=RADIO_TYPE["gsm"], mcc=1, mnc=2, lac=3, cid=4, new_measures=2, total_measures=5)) session.add(Score(userid=1, key=SCORE_TYPE["new_cell"], value=7)) session.flush() measure = dict( id=0, created=encode_datetime(time), lat=10000000, lon=20000000, time=encode_datetime(time), accuracy=0, altitude=0, altitude_accuracy=0, radio=RADIO_TYPE["gsm"], ) entries = [ {"mcc": 1, "mnc": 2, "lac": 3147483647, "cid": 2147483647, "psc": 5, "asu": 8}, {"mcc": 1, "mnc": 2, "lac": -1, "cid": -1, "psc": 5, "asu": 8}, ] for e in entries: e.update(measure) result = insert_cell_measures.delay(entries, userid=1) self.assertEqual(result.get(), 2) measures = session.query(CellMeasure).all() self.assertEqual(len(measures), 2) self.assertEqual(set([m.lac for m in measures]), set([-1])) self.assertEqual(set([m.cid for m in measures]), set([-1])) # Nothing should change in the initially created Cell record cells = session.query(Cell).all() self.assertEqual(len(cells), 1) self.assertEqual(set([c.new_measures for c in cells]), set([2])) self.assertEqual(set([c.total_measures for c in cells]), set([5]))
def process_measure(data, utcnow, session, userid=None): measure = Measure() measure.created = utcnow measure.time = data["time"] measure.lat = to_precise_int(data["lat"]) measure.lon = to_precise_int(data["lon"]) measure.accuracy = data["accuracy"] measure.altitude = data["altitude"] measure.altitude_accuracy = data["altitude_accuracy"] measure.radio = RADIO_TYPE.get(data["radio"], -1) # get measure.id set session.add(measure) session.flush() measure_data = dict( id=measure.id, created=encode_datetime(measure.created), lat=measure.lat, lon=measure.lon, time=encode_datetime(measure.time), accuracy=measure.accuracy, altitude=measure.altitude, altitude_accuracy=measure.altitude_accuracy, radio=measure.radio, ) if data.get("cell"): insert_cell_measure.delay(measure_data, data["cell"], userid=userid) measure.cell = dumps(data["cell"]) if data.get("wifi"): # filter out old-style sha1 hashes too_long_keys = False for w in data["wifi"]: w["key"] = key = normalize_wifi_key(w["key"]) if len(key) > 12: too_long_keys = True break if not too_long_keys: insert_wifi_measure.delay(measure_data, data["wifi"], userid=userid) measure.wifi = dumps(data["wifi"]) return measure
def test_time(self): app = self.app # test two weeks ago and "now" time = (datetime.utcnow() - timedelta(14)).replace(microsecond=0) tstr = encode_datetime(time) app.post_json('/v1/submit', { "items": [ { "lat": 1.0, "lon": 2.0, "wifi": [{ "key": "00aaaaaaaaaa" }], "time": tstr }, { "lat": 2.0, "lon": 3.0, "wifi": [{ "key": "00bbbbbbbbbb" }] }, ] }, status=204) session = self.db_master_session result = session.query(WifiMeasure).all() self.assertEqual(len(result), 2) wifis = dict([(w.key, (w.created, w.time)) for w in result]) today = datetime.utcnow().date() month_rounded_tday = time.replace(day=1, hour=0, minute=0, second=0) month_rounded_today = today.replace(day=1) self.assertEqual(wifis['00aaaaaaaaaa'][0].date(), today) self.assertEqual(wifis['00aaaaaaaaaa'][1], month_rounded_tday) self.assertEqual(wifis['00bbbbbbbbbb'][0].date(), today) self.assertEqual(wifis['00bbbbbbbbbb'][1].date(), month_rounded_today)
def process_measure(measure_id, data, session): cell_measures = [] wifi_measures = [] measure_data = dict( measure_id=measure_id, lat=to_precise_int(data['lat']), lon=to_precise_int(data['lon']), time=encode_datetime(data['time']), accuracy=data['accuracy'], altitude=data['altitude'], altitude_accuracy=data['altitude_accuracy'], ) measure_radio = RADIO_TYPE.get(data['radio'], -1) if data.get('cell'): # flatten measure / cell data into a single dict for c in data['cell']: c.update(measure_data) # use more specific cell type or # fall back to less precise measure if c['radio'] != '': c['radio'] = RADIO_TYPE.get(c['radio'], -1) else: c['radio'] = measure_radio cell_measures = data['cell'] if data.get('wifi'): # filter out old-style sha1 hashes invalid_wifi_key = False for w in data['wifi']: w['key'] = key = normalize_wifi_key(w['key']) if not valid_wifi_pattern(key): invalid_wifi_key = True break if not invalid_wifi_key: # flatten measure / wifi data into a single dict for w in data['wifi']: w.update(measure_data) wifi_measures = data['wifi'] return (cell_measures, wifi_measures)
def test_cell(self): from ichnaea.service.submit.tasks import insert_cell_measure session = self.db_master_session utcnow = datetime.utcnow() session.add( Cell(radio=0, mcc=1, mnc=2, lac=3, cid=4, new_measures=2, total_measures=5)) session.add(Score(userid=1, key=SCORE_TYPE['new_cell'], value=7)) session.flush() measure = dict( id=0, created=encode_datetime(utcnow), lat=10000000, lon=20000000, time=encode_datetime(utcnow), accuracy=0, altitude=0, altitude_accuracy=0, radio=0, ) entries = [ { "mcc": 1, "mnc": 2, "signal": -100 }, { "mcc": 1, "mnc": 2, "lac": 3, "cid": 4, "psc": 5, "asu": 8 }, { "mcc": 1, "mnc": 2, "lac": 3, "cid": 4, "asu": 8 }, { "mcc": 1, "mnc": 2, "lac": 3, "cid": 4, "asu": 15 }, { "mcc": 1, "mnc": 2, "lac": 3, "cid": 7 }, ] result = insert_cell_measure.delay(measure, entries, userid=1) self.assertEqual(result.get(), 5) measures = session.query(CellMeasure).all() self.assertEqual(len(measures), 5) self.assertEqual(set([m.mcc for m in measures]), set([1])) self.assertEqual(set([m.mnc for m in measures]), set([2])) self.assertEqual(set([m.asu for m in measures]), set([0, 8, 15])) self.assertEqual(set([m.psc for m in measures]), set([0, 5])) self.assertEqual(set([m.signal for m in measures]), set([0, -100])) cells = session.query(Cell).all() self.assertEqual(len(cells), 2) self.assertEqual(set([c.mcc for c in cells]), set([1])) self.assertEqual(set([c.mnc for c in cells]), set([2])) self.assertEqual(set([c.lac for c in cells]), set([3])) self.assertEqual(set([c.cid for c in cells]), set([4, 7])) self.assertEqual(set([c.new_measures for c in cells]), set([1, 5])) self.assertEqual(set([c.total_measures for c in cells]), set([1, 8])) scores = session.query(Score).all() self.assertEqual(len(scores), 1) self.assertEqual(scores[0].key, SCORE_TYPE['new_cell']) self.assertEqual(scores[0].value, 10) # test duplicate execution result = insert_cell_measure.delay(measure, entries, userid=1) self.assertEqual(result.get(), 5) # TODO this task isn't idempotent yet measures = session.query(CellMeasure).all() self.assertEqual(len(measures), 10)
def test_cell(self): from ichnaea.service.submit.tasks import insert_cell_measures session = self.db_master_session time = datetime.utcnow().replace(microsecond=0) - timedelta(days=1) session.add(Cell(radio=RADIO_TYPE["gsm"], mcc=1, mnc=2, lac=3, cid=4, psc=5, new_measures=2, total_measures=5)) session.add(Score(userid=1, key=SCORE_TYPE["new_cell"], value=7)) session.flush() measure = dict( id=0, created=encode_datetime(time), lat=10000000, lon=20000000, time=encode_datetime(time), accuracy=0, altitude=0, altitude_accuracy=0, radio=RADIO_TYPE["gsm"], ) entries = [ # Note that this first entry will be skipped as it does # not include (lac, cid) or (psc) {"mcc": 1, "mnc": 2, "signal": -100}, {"mcc": 1, "mnc": 2, "lac": 3, "cid": 4, "psc": 5, "asu": 8}, {"mcc": 1, "mnc": 2, "lac": 3, "cid": 4, "psc": 5, "asu": 8}, {"mcc": 1, "mnc": 2, "lac": 3, "cid": 4, "psc": 5, "asu": 15}, {"mcc": 1, "mnc": 2, "lac": 3, "cid": 7, "psc": 5}, ] for e in entries: e.update(measure) result = insert_cell_measures.delay(entries, userid=1) self.assertEqual(result.get(), 4) measures = session.query(CellMeasure).all() self.assertEqual(len(measures), 4) self.assertEqual(set([m.mcc for m in measures]), set([1])) self.assertEqual(set([m.mnc for m in measures]), set([2])) self.assertEqual(set([m.asu for m in measures]), set([-1, 8, 15])) self.assertEqual(set([m.psc for m in measures]), set([5])) self.assertEqual(set([m.signal for m in measures]), set([0])) cells = session.query(Cell).all() self.assertEqual(len(cells), 2) self.assertEqual(set([c.mcc for c in cells]), set([1])) self.assertEqual(set([c.mnc for c in cells]), set([2])) self.assertEqual(set([c.lac for c in cells]), set([3])) self.assertEqual(set([c.cid for c in cells]), set([4, 7])) self.assertEqual(set([c.psc for c in cells]), set([5])) self.assertEqual(set([c.new_measures for c in cells]), set([1, 5])) self.assertEqual(set([c.total_measures for c in cells]), set([1, 8])) scores = session.query(Score).all() self.assertEqual(len(scores), 1) self.assertEqual(scores[0].key, SCORE_TYPE["new_cell"]) self.assertEqual(scores[0].value, 8) # test duplicate execution result = insert_cell_measures.delay(entries, userid=1) self.assertEqual(result.get(), 4) # TODO this task isn't idempotent yet measures = session.query(CellMeasure).all() self.assertEqual(len(measures), 8)
def test_ignore_unhelpful_incomplete_cells(self): # Cell records must have MNC, MCC and at least one of (LAC, CID) or PSC # values filled in. from ichnaea.service.submit.tasks import insert_cell_measures session = self.db_master_session time = datetime.utcnow().replace(microsecond=0) - timedelta(days=1) measure = dict( id=0, created=encode_datetime(time), lat=10000000, lon=20000000, time=encode_datetime(time), accuracy=0, altitude=0, altitude_accuracy=0, radio=RADIO_TYPE['gsm'], ) entries = [ # These records are valid { "mcc": 1, "mnc": 2, "lac": 3, "cid": 4 }, { "mcc": 1, "mnc": 2, "lac": 3, "cid": 4, "psc": 5 }, # This record is missing everything {}, # These records fail the mcc check { "mnc": 2, "lac": 3, "cid": 4 }, { "mcc": 0, "mnc": 2, "lac": 3, "cid": 4 }, { "mcc": -1, "mnc": 2, "lac": 3, "cid": 4 }, { "mcc": -2, "mnc": 2, "lac": 3, "cid": 4 }, { "mcc": 2000, "mnc": 2, "lac": 3, "cid": 4 }, # These records fail the mnc check { "mcc": 1, "lac": 3, "cid": 4 }, { "mcc": 1, "mnc": -1, "lac": 3, "cid": 4 }, { "mcc": 1, "mnc": -2, "lac": 3, "cid": 4 }, { "mcc": 1, "mnc": 33000, "lac": 3, "cid": 4 }, # These records fail the lac check { "mcc": 1, "mnc": 2, "cid": 4 }, { "mcc": 1, "mnc": 2, "lac": -1, "cid": 4 }, { "mcc": 1, "mnc": 2, "lac": -2, "cid": 4 }, { "mcc": 1, "mnc": 2, "lac": 65536, "cid": 4 }, # These records fail the cid check { "mcc": 1, "mnc": 2, "lac": 3 }, { "mcc": 1, "mnc": 2, "lac": 3, "cid": -1 }, { "mcc": 1, "mnc": 2, "lac": 3, "cid": -2 }, { "mcc": 1, "mnc": 2, "lac": 3, "cid": 2**28 }, # These records fail the (lac or cid) and psc check { "mcc": 1, "mnc": 2 }, { "mcc": 1, "mnc": 2, "lac": 3 }, { "mcc": 1, "mnc": 2, "cid": 4 }, ] for e in entries: e.update(measure) result = insert_cell_measures.delay(entries, userid=1) self.assertEqual(result.get(), 2) measures = session.query(CellMeasure).all() self.assertEqual(len(measures), 2) cells = session.query(Cell).all() self.assertEqual(len(cells), 1) entries = [ # These records are valid { "mcc": 1, "mnc": 2, "psc": 5 }, { "mcc": 1, "mnc": 2, "lac": 3, "psc": 5 }, { "mcc": 1, "mnc": 2, "cid": 4, "psc": 5 }, ] for e in entries: e.update(measure) result = insert_cell_measures.delay(entries, userid=1) self.assertEqual(result.get(), 3) measures = session.query(CellMeasure).all() self.assertEqual(len(measures), 5) cells = session.query(Cell).all() self.assertEqual(len(cells), 1)
def test_ignore_unhelpful_incomplete_cells(self): # Cell records must have MNC, MCC and at least one of (LAC, CID) or PSC # values filled in. from ichnaea.service.submit.tasks import insert_cell_measures session = self.db_master_session time = datetime.utcnow().replace(microsecond=0) - timedelta(days=1) measure = dict( id=0, created=encode_datetime(time), lat=10000000, lon=20000000, time=encode_datetime(time), accuracy=0, altitude=0, altitude_accuracy=0, radio=RADIO_TYPE['gsm'], ) entries = [ # These records are valid {"mcc": 1, "mnc": 2, "lac": 3, "cid": 4}, {"mcc": 1, "mnc": 2, "lac": 3, "cid": 4, "psc": 5}, # This record is missing everything {}, # These records fail the mcc check {"mnc": 2, "lac": 3, "cid": 4}, {"mcc": 0, "mnc": 2, "lac": 3, "cid": 4}, {"mcc": -1, "mnc": 2, "lac": 3, "cid": 4}, {"mcc": -2, "mnc": 2, "lac": 3, "cid": 4}, {"mcc": 2000, "mnc": 2, "lac": 3, "cid": 4}, # These records fail the mnc check {"mcc": 1, "lac": 3, "cid": 4}, {"mcc": 1, "mnc": -1, "lac": 3, "cid": 4}, {"mcc": 1, "mnc": -2, "lac": 3, "cid": 4}, {"mcc": 1, "mnc": 33000, "lac": 3, "cid": 4}, # These records fail the lac check {"mcc": 1, "mnc": 2, "cid": 4}, {"mcc": 1, "mnc": 2, "lac": -1, "cid": 4}, {"mcc": 1, "mnc": 2, "lac": -2, "cid": 4}, {"mcc": 1, "mnc": 2, "lac": 65536, "cid": 4}, # These records fail the cid check {"mcc": 1, "mnc": 2, "lac": 3}, {"mcc": 1, "mnc": 2, "lac": 3, "cid": -1}, {"mcc": 1, "mnc": 2, "lac": 3, "cid": -2}, {"mcc": 1, "mnc": 2, "lac": 3, "cid": 2 ** 28}, # These records fail the (lac or cid) and psc check {"mcc": 1, "mnc": 2}, {"mcc": 1, "mnc": 2, "lac": 3}, {"mcc": 1, "mnc": 2, "cid": 4}, ] for e in entries: e.update(measure) result = insert_cell_measures.delay(entries, userid=1) self.assertEqual(result.get(), 2) measures = session.query(CellMeasure).all() self.assertEqual(len(measures), 2) cells = session.query(Cell).all() self.assertEqual(len(cells), 1) entries = [ # These records are valid {"mcc": 1, "mnc": 2, "psc": 5}, {"mcc": 1, "mnc": 2, "lac": 3, "psc": 5}, {"mcc": 1, "mnc": 2, "cid": 4, "psc": 5}, ] for e in entries: e.update(measure) result = insert_cell_measures.delay(entries, userid=1) self.assertEqual(result.get(), 3) measures = session.query(CellMeasure).all() self.assertEqual(len(measures), 5) cells = session.query(Cell).all() self.assertEqual(len(cells), 1)
def test_cell(self): from ichnaea.service.submit.tasks import insert_cell_measures session = self.db_master_session time = datetime.utcnow().replace(microsecond=0) - timedelta(days=1) session.add( Cell(radio=RADIO_TYPE['gsm'], mcc=1, mnc=2, lac=3, cid=4, psc=5, new_measures=2, total_measures=5)) session.add(Score(userid=1, key=SCORE_TYPE['new_cell'], value=7)) session.flush() measure = dict( id=0, created=encode_datetime(time), lat=10000000, lon=20000000, time=encode_datetime(time), accuracy=0, altitude=0, altitude_accuracy=0, radio=RADIO_TYPE['gsm'], ) entries = [ # Note that this first entry will be skipped as it does # not include (lac, cid) or (psc) { "mcc": 1, "mnc": 2, "signal": -100 }, { "mcc": 1, "mnc": 2, "lac": 3, "cid": 4, "psc": 5, "asu": 8 }, { "mcc": 1, "mnc": 2, "lac": 3, "cid": 4, "psc": 5, "asu": 8 }, { "mcc": 1, "mnc": 2, "lac": 3, "cid": 4, "psc": 5, "asu": 15 }, { "mcc": 1, "mnc": 2, "lac": 3, "cid": 7, "psc": 5 }, ] for e in entries: e.update(measure) result = insert_cell_measures.delay(entries, userid=1) self.assertEqual(result.get(), 4) measures = session.query(CellMeasure).all() self.assertEqual(len(measures), 4) self.assertEqual(set([m.mcc for m in measures]), set([1])) self.assertEqual(set([m.mnc for m in measures]), set([2])) self.assertEqual(set([m.asu for m in measures]), set([-1, 8, 15])) self.assertEqual(set([m.psc for m in measures]), set([5])) self.assertEqual(set([m.signal for m in measures]), set([0])) cells = session.query(Cell).all() self.assertEqual(len(cells), 2) self.assertEqual(set([c.mcc for c in cells]), set([1])) self.assertEqual(set([c.mnc for c in cells]), set([2])) self.assertEqual(set([c.lac for c in cells]), set([3])) self.assertEqual(set([c.cid for c in cells]), set([4, 7])) self.assertEqual(set([c.psc for c in cells]), set([5])) self.assertEqual(set([c.new_measures for c in cells]), set([1, 5])) self.assertEqual(set([c.total_measures for c in cells]), set([1, 8])) scores = session.query(Score).all() self.assertEqual(len(scores), 1) self.assertEqual(scores[0].key, SCORE_TYPE['new_cell']) self.assertEqual(scores[0].value, 8) # test duplicate execution result = insert_cell_measures.delay(entries, userid=1) self.assertEqual(result.get(), 4) # TODO this task isn't idempotent yet measures = session.query(CellMeasure).all() self.assertEqual(len(measures), 8)
def test_wifi(self): from ichnaea.service.submit.tasks import insert_wifi_measures session = self.db_master_session time = datetime.utcnow().replace(microsecond=0) - timedelta(days=1) session.add(Wifi(key="ab12")) session.add(Score(userid=1, key=SCORE_TYPE['new_wifi'], value=7)) session.flush() measure = dict( id=0, created=encode_datetime(time), lat=10000000, lon=20000000, time=encode_datetime(time), accuracy=0, altitude=0, altitude_accuracy=0, radio=-1, ) entries = [ { "key": "ab12", "channel": 11, "signal": -80 }, { "key": "ab12", "channel": 3, "signal": -90 }, { "key": "ab12", "channel": 3, "signal": -80 }, { "key": "cd34", "channel": 3, "signal": -90 }, ] for e in entries: e.update(measure) result = insert_wifi_measures.delay(entries, userid=1) self.assertEqual(result.get(), 4) measures = session.query(WifiMeasure).all() self.assertEqual(len(measures), 4) self.assertEqual(set([m.key for m in measures]), set(["ab12", "cd34"])) self.assertEqual(set([m.channel for m in measures]), set([3, 11])) self.assertEqual(set([m.signal for m in measures]), set([-80, -90])) wifis = session.query(Wifi).all() self.assertEqual(len(wifis), 2) self.assertEqual(set([w.key for w in wifis]), set(["ab12", "cd34"])) self.assertEqual(set([w.new_measures for w in wifis]), set([1, 3])) self.assertEqual(set([w.total_measures for w in wifis]), set([1, 3])) scores = session.query(Score).all() self.assertEqual(len(scores), 1) self.assertEqual(scores[0].key, SCORE_TYPE['new_wifi']) self.assertEqual(scores[0].value, 8) # test duplicate execution result = insert_wifi_measures.delay(entries, userid=1) self.assertEqual(result.get(), 4) # TODO this task isn't idempotent yet measures = session.query(WifiMeasure).all() self.assertEqual(len(measures), 8)