Exemple #1
0
    def test_cell_out_of_range_values(self):
        from ichnaea.service.submit.tasks import insert_cell_measures
        session = self.db_master_session
        time = datetime.utcnow().replace(microsecond=0) - timedelta(days=1)

        measure = dict(
            id=0, created=encode_datetime(time), lat=10000000, lon=20000000,
            time=encode_datetime(time), accuracy=0, altitude=0,
            altitude_accuracy=0, radio=RADIO_TYPE['gsm'], mcc=1,
            mnc=2, lac=3, cid=4)
        entries = [
            {"asu": 8, "signal": -70, "ta": 32},
            {"asu": -10, "signal": -300, "ta": -10},
            {"asu": 256, "signal": 16, "ta": 128},
        ]
        for e in entries:
            e.update(measure)

        result = insert_cell_measures.delay(entries)
        self.assertEqual(result.get(), 3)

        measures = session.query(CellMeasure).all()
        self.assertEqual(len(measures), 3)
        self.assertEqual(set([m.asu for m in measures]), set([-1, 8]))
        self.assertEqual(set([m.signal for m in measures]), set([0, -70]))
        self.assertEqual(set([m.ta for m in measures]), set([0, 32]))
Exemple #2
0
    def test_ignore_unhelpful_incomplete_cdma_cells(self):
        # CDMA cell records must have MNC, MCC, LAC and CID filled in
        from ichnaea.service.submit.tasks import insert_cell_measures
        session = self.db_master_session
        time = datetime.utcnow().replace(microsecond=0) - timedelta(days=1)

        measure = dict(
            id=0, created=encode_datetime(time), lat=10000000, lon=20000000,
            time=encode_datetime(time), accuracy=0, altitude=0,
            altitude_accuracy=0, radio=RADIO_TYPE['cdma'],
        )
        entries = [
            # This records is valid
            {"mcc": 1, "mnc": 2, "lac": 3, "cid": 4},

            # This record should fail as it's missing CID
            {"mcc": 1, "mnc": 2, "lac": 3},

            # This fails for missing lac
            {"mcc": 1, "mnc": 2, "cid": 4},

            # Adding a psc doesn't change things
            {"mcc": 1, "mnc": 2, "psc": 5},
        ]

        for e in entries:
            e.update(measure)
        result = insert_cell_measures.delay(entries, userid=1)

        self.assertEqual(result.get(), 1)
        measures = session.query(CellMeasure).all()
        self.assertEqual(len(measures), 1)
        cells = session.query(Cell).all()
        self.assertEqual(len(cells), 1)
Exemple #3
0
def reprocess_cell_measure(self, measure_ids, userid=None):
    measures = []
    try:
        with self.db_session() as session:
            measures = session.query(Measure).filter(
                Measure.id.in_(measure_ids)).filter(
                Measure.cell != sql_null).all()
            for measure in measures:
                measure_data = dict(
                    id=measure.id, created=encode_datetime(measure.created),
                    lat=measure.lat, lon=measure.lon,
                    time=encode_datetime(measure.time),
                    accuracy=measure.accuracy, altitude=measure.altitude,
                    altitude_accuracy=measure.altitude_accuracy,
                    radio=measure.radio,
                )
                # adds data to this session
                process_cell_measure(
                    session, measure_data, loads(measure.cell), userid=userid)
            session.commit()
        return len(measures)
    except IntegrityError as exc:  # pragma: no cover
        logger.exception('error')
        return 0
    except Exception as exc:  # pragma: no cover
        raise self.retry(exc=exc)
Exemple #4
0
    def test_insert_invalid_lac(self):
        from ichnaea.service.submit.tasks import insert_cell_measures
        session = self.db_master_session
        time = datetime.utcnow().replace(microsecond=0) - timedelta(days=1)

        session.add(
            Cell(radio=RADIO_TYPE['gsm'],
                 mcc=1,
                 mnc=2,
                 lac=3,
                 cid=4,
                 new_measures=2,
                 total_measures=5))
        session.add(Score(userid=1, key=SCORE_TYPE['new_cell'], value=7))
        session.flush()

        measure = dict(id=0,
                       created=encode_datetime(time),
                       lat=10000000,
                       lon=20000000,
                       time=encode_datetime(time),
                       accuracy=0,
                       altitude=0,
                       altitude_accuracy=0,
                       radio=RADIO_TYPE['gsm'])
        entries = [
            {
                "mcc": 1,
                "mnc": 2,
                "lac": 3147483647,
                "cid": 2147483647,
                "psc": 5,
                "asu": 8
            },
            {
                "mcc": 1,
                "mnc": 2,
                "lac": -1,
                "cid": -1,
                "psc": 5,
                "asu": 8
            },
        ]
        for e in entries:
            e.update(measure)

        result = insert_cell_measures.delay(entries, userid=1)
        self.assertEqual(result.get(), 2)

        measures = session.query(CellMeasure).all()
        self.assertEqual(len(measures), 2)
        self.assertEqual(set([m.lac for m in measures]), set([-1]))
        self.assertEqual(set([m.cid for m in measures]), set([-1]))

        # Nothing should change in the initially created Cell record
        cells = session.query(Cell).all()
        self.assertEqual(len(cells), 1)
        self.assertEqual(set([c.new_measures for c in cells]), set([2]))
        self.assertEqual(set([c.total_measures for c in cells]), set([5]))
Exemple #5
0
def process_wifi(entries, measure):
    measure_data = dict(
        id=measure.id, created=encode_datetime(measure.created),
        lat=measure.lat, lon=measure.lon, time=encode_datetime(measure.time),
        accuracy=measure.accuracy, altitude=measure.altitude,
        altitude_accuracy=measure.altitude_accuracy,
    )
    insert_wifi_measure.delay(measure_data, entries)
Exemple #6
0
    def test_ignore_unhelpful_incomplete_cdma_cells(self):
        # CDMA cell records must have MNC, MCC, LAC and CID filled in
        from ichnaea.service.submit.tasks import insert_cell_measures
        session = self.db_master_session
        time = datetime.utcnow().replace(microsecond=0) - timedelta(days=1)

        measure = dict(
            id=0,
            created=encode_datetime(time),
            lat=10000000,
            lon=20000000,
            time=encode_datetime(time),
            accuracy=0,
            altitude=0,
            altitude_accuracy=0,
            radio=RADIO_TYPE['cdma'],
        )
        entries = [
            # This records is valid
            {
                "mcc": 1,
                "mnc": 2,
                "lac": 3,
                "cid": 4
            },

            # This record should fail as it's missing CID
            {
                "mcc": 1,
                "mnc": 2,
                "lac": 3
            },

            # This fails for missing lac
            {
                "mcc": 1,
                "mnc": 2,
                "cid": 4
            },

            # Adding a psc doesn't change things
            {
                "mcc": 1,
                "mnc": 2,
                "psc": 5
            },
        ]

        for e in entries:
            e.update(measure)
        result = insert_cell_measures.delay(entries, userid=1)

        self.assertEqual(result.get(), 1)
        measures = session.query(CellMeasure).all()
        self.assertEqual(len(measures), 1)
        cells = session.query(Cell).all()
        self.assertEqual(len(cells), 1)
Exemple #7
0
    def test_wifi(self):
        from ichnaea.service.submit.tasks import insert_wifi_measures

        session = self.db_master_session
        time = datetime.utcnow().replace(microsecond=0) - timedelta(days=1)

        session.add(Wifi(key="ab1234567890"))
        session.add(Score(userid=1, key=SCORE_TYPE["new_wifi"], value=7))
        session.flush()

        measure = dict(
            id=0,
            created=encode_datetime(time),
            lat=10000000,
            lon=20000000,
            time=encode_datetime(time),
            accuracy=0,
            altitude=0,
            altitude_accuracy=0,
            radio=-1,
        )
        entries = [
            {"key": "ab1234567890", "channel": 11, "signal": -80},
            {"key": "ab1234567890", "channel": 3, "signal": -90},
            {"key": "ab1234567890", "channel": 3, "signal": -80},
            {"key": "cd3456789012", "channel": 3, "signal": -90},
        ]
        for e in entries:
            e.update(measure)
        result = insert_wifi_measures.delay(entries, userid=1)
        self.assertEqual(result.get(), 4)

        measures = session.query(WifiMeasure).all()
        self.assertEqual(len(measures), 4)
        self.assertEqual(set([m.key for m in measures]), set(["ab1234567890", "cd3456789012"]))
        self.assertEqual(set([m.channel for m in measures]), set([3, 11]))
        self.assertEqual(set([m.signal for m in measures]), set([-80, -90]))

        wifis = session.query(Wifi).all()
        self.assertEqual(len(wifis), 2)
        self.assertEqual(set([w.key for w in wifis]), set(["ab1234567890", "cd3456789012"]))
        self.assertEqual(set([w.new_measures for w in wifis]), set([1, 3]))
        self.assertEqual(set([w.total_measures for w in wifis]), set([1, 3]))

        scores = session.query(Score).all()
        self.assertEqual(len(scores), 1)
        self.assertEqual(scores[0].key, SCORE_TYPE["new_wifi"])
        self.assertEqual(scores[0].value, 8)

        # test duplicate execution
        result = insert_wifi_measures.delay(entries, userid=1)
        self.assertEqual(result.get(), 4)
        # TODO this task isn't idempotent yet
        measures = session.query(WifiMeasure).all()
        self.assertEqual(len(measures), 8)
Exemple #8
0
    def test_cell(self):
        from ichnaea.service.submit.tasks import insert_cell_measure
        session = self.db_master_session
        time = datetime.utcnow().replace(microsecond=0) - timedelta(days=1)

        session.add(Cell(radio=0, mcc=1, mnc=2, lac=3, cid=4,
                         new_measures=2, total_measures=5))
        session.add(Score(userid=1, key=SCORE_TYPE['new_cell'], value=7))
        session.flush()

        measure = dict(
            id=0, created=encode_datetime(time), lat=10000000, lon=20000000,
            time=encode_datetime(time), accuracy=0, altitude=0,
            altitude_accuracy=0, radio=0,
        )
        entries = [
            {"mcc": 1, "mnc": 2, "signal": -100},
            {"mcc": 1, "mnc": 2, "lac": 3, "cid": 4, "psc": 5, "asu": 8},
            {"mcc": 1, "mnc": 2, "lac": 3, "cid": 4, "asu": 8},
            {"mcc": 1, "mnc": 2, "lac": 3, "cid": 4, "asu": 15},
            {"mcc": 1, "mnc": 2, "lac": 3, "cid": 7},
        ]
        result = insert_cell_measure.delay(measure, entries, userid=1)
        self.assertEqual(result.get(), 5)

        measures = session.query(CellMeasure).all()
        self.assertEqual(len(measures), 5)
        self.assertEqual(set([m.mcc for m in measures]), set([1]))
        self.assertEqual(set([m.mnc for m in measures]), set([2]))
        self.assertEqual(set([m.asu for m in measures]), set([0, 8, 15]))
        self.assertEqual(set([m.psc for m in measures]), set([0, 5]))
        self.assertEqual(set([m.signal for m in measures]), set([0, -100]))

        cells = session.query(Cell).all()
        self.assertEqual(len(cells), 2)
        self.assertEqual(set([c.mcc for c in cells]), set([1]))
        self.assertEqual(set([c.mnc for c in cells]), set([2]))
        self.assertEqual(set([c.lac for c in cells]), set([3]))
        self.assertEqual(set([c.cid for c in cells]), set([4, 7]))
        self.assertEqual(set([c.new_measures for c in cells]), set([1, 5]))
        self.assertEqual(set([c.total_measures for c in cells]), set([1, 8]))

        scores = session.query(Score).all()
        self.assertEqual(len(scores), 1)
        self.assertEqual(scores[0].key, SCORE_TYPE['new_cell'])
        self.assertEqual(scores[0].value, 8)

        # test duplicate execution
        result = insert_cell_measure.delay(measure, entries, userid=1)
        self.assertEqual(result.get(), 5)
        # TODO this task isn't idempotent yet
        measures = session.query(CellMeasure).all()
        self.assertEqual(len(measures), 10)
Exemple #9
0
    def test_wifi(self):
        from ichnaea.service.submit.tasks import insert_wifi_measure
        session = self.db_master_session
        utcnow = datetime.utcnow()

        session.add(Wifi(key="ab12"))
        session.add(Score(userid=1, key=SCORE_TYPE['new_wifi'], value=7))
        session.flush()

        measure = dict(
            id=0, created=encode_datetime(utcnow), lat=10000000, lon=20000000,
            time=encode_datetime(utcnow), accuracy=0, altitude=0,
            altitude_accuracy=0, radio=-1,
        )
        entries = [
            {"key": "ab12", "channel": 11, "signal": -80},
            {"key": "ab12", "channel": 3, "signal": -90},
            {"key": "ab12", "channel": 3, "signal": -80},
            {"key": "cd34", "channel": 3, "signal": -90},
        ]
        result = insert_wifi_measure.delay(measure, entries, userid=1)
        self.assertEqual(result.get(), 4)

        measures = session.query(WifiMeasure).all()
        self.assertEqual(len(measures), 4)
        self.assertEqual(set([m.key for m in measures]), set(["ab12", "cd34"]))
        self.assertEqual(set([m.channel for m in measures]), set([3, 11]))
        self.assertEqual(set([m.signal for m in measures]), set([-80, -90]))

        wifis = session.query(Wifi).all()
        self.assertEqual(len(wifis), 2)
        self.assertEqual(set([w.key for w in wifis]), set(["ab12", "cd34"]))
        self.assertEqual(set([w.new_measures for w in wifis]), set([1, 3]))
        self.assertEqual(set([w.total_measures for w in wifis]), set([1, 3]))

        scores = session.query(Score).all()
        self.assertEqual(len(scores), 1)
        self.assertEqual(scores[0].key, SCORE_TYPE['new_wifi'])
        self.assertEqual(scores[0].value, 9)

        # test duplicate execution
        result = insert_wifi_measure.delay(measure, entries, userid=1)
        self.assertEqual(result.get(), 4)
        # TODO this task isn't idempotent yet
        measures = session.query(WifiMeasure).all()
        self.assertEqual(len(measures), 8)

        # test error case
        entries[0]['id'] = measures[0].id
        result = insert_wifi_measure.delay(measure, entries)
        self.assertEqual(result.get(), 0)
Exemple #10
0
 def test_time(self):
     app = self.app
     # test two weeks ago and "now"
     time = (datetime.utcnow() - timedelta(14)).replace(microsecond=0)
     tstr = encode_datetime(time)
     app.post_json('/v1/submit', {
         "items": [
             {
                 "lat": 1.0,
                 "lon": 2.0,
                 "wifi": [{
                     "key": "a"
                 }],
                 "time": tstr
             },
             {
                 "lat": 2.0,
                 "lon": 3.0,
                 "wifi": [{
                     "key": "b"
                 }]
             },
         ]
     },
                   status=204)
     session = self.db_master_session
     result = session.query(Measure).all()
     self.assertEqual(len(result), 2)
     for item in result:
         if '"key": "a"' in item.wifi:
             self.assertEqual(item.time, time)
         else:
             self.assertEqual(item.time.date(), datetime.utcnow().date())
Exemple #11
0
    def test_time(self):
        app = self.app
        # test two weeks ago and "now"
        time = (datetime.utcnow() - timedelta(14)).replace(microsecond=0)
        tstr = encode_datetime(time)
        app.post_json(
            '/v1/submit', {"items": [
                {"lat": 1.0,
                 "lon": 2.0,
                 "wifi": [{"key": "00aaaaaaaaaa"}],
                 "time": tstr},
                {"lat": 2.0,
                 "lon": 3.0,
                 "wifi": [{"key": "00bbbbbbbbbb"}]},
            ]},
            status=204)
        session = self.db_master_session
        result = session.query(WifiMeasure).all()
        self.assertEqual(len(result), 2)

        wifis = dict([(w.key, (w.created, w.time)) for w in result])
        today = datetime.utcnow().date()

        month_rounded_tday = time.replace(day=1, hour=0, minute=0, second=0)
        month_rounded_today = today.replace(day=1)

        self.assertEqual(wifis['00aaaaaaaaaa'][0].date(), today)
        self.assertEqual(wifis['00aaaaaaaaaa'][1], month_rounded_tday)

        self.assertEqual(wifis['00bbbbbbbbbb'][0].date(), today)
        self.assertEqual(wifis['00bbbbbbbbbb'][1].date(), month_rounded_today)
Exemple #12
0
 def test_time(self):
     app = self.app
     # test two weeks ago and "now"
     time = (datetime.utcnow() - timedelta(14)).replace(microsecond=0)
     tday = time.replace(hour=0, minute=0, second=0)
     tstr = encode_datetime(time)
     app.post_json(
         '/v1/submit', {"items": [
             {"lat": 1.0, "lon": 2.0, "wifi": [{"key": "a"}], "time": tstr},
             {"lat": 2.0, "lon": 3.0, "wifi": [{"key": "b"}]},
         ]},
         status=204)
     session = self.db_master_session
     result = session.query(Measure).all()
     self.assertEqual(len(result), 2)
     today = datetime.utcnow().date()
     for item in result:
         self.assertEqual(item.created.date(), today)
         self.assertEqual(item.created.hour, 0)
         self.assertEqual(item.created.minute, 0)
         self.assertEqual(item.created.second, 0)
         if '"key": "a"' in item.wifi:
             self.assertEqual(item.time, tday)
         else:
             self.assertEqual(item.time.date(), today)
Exemple #13
0
    def test_cell_out_of_range_values(self):
        from ichnaea.service.submit.tasks import insert_cell_measures
        session = self.db_master_session
        time = datetime.utcnow().replace(microsecond=0) - timedelta(days=1)

        measure = dict(id=0,
                       created=encode_datetime(time),
                       lat=10000000,
                       lon=20000000,
                       time=encode_datetime(time),
                       accuracy=0,
                       altitude=0,
                       altitude_accuracy=0,
                       radio=RADIO_TYPE['gsm'],
                       mcc=1,
                       mnc=2,
                       lac=3,
                       cid=4)
        entries = [
            {
                "asu": 8,
                "signal": -70,
                "ta": 32
            },
            {
                "asu": -10,
                "signal": -300,
                "ta": -10
            },
            {
                "asu": 256,
                "signal": 16,
                "ta": 128
            },
        ]
        for e in entries:
            e.update(measure)

        result = insert_cell_measures.delay(entries)
        self.assertEqual(result.get(), 3)

        measures = session.query(CellMeasure).all()
        self.assertEqual(len(measures), 3)
        self.assertEqual(set([m.asu for m in measures]), set([-1, 8]))
        self.assertEqual(set([m.signal for m in measures]), set([0, -70]))
        self.assertEqual(set([m.ta for m in measures]), set([0, 32]))
Exemple #14
0
    def test_insert_invalid_lac(self):
        from ichnaea.service.submit.tasks import insert_cell_measures

        session = self.db_master_session
        time = datetime.utcnow().replace(microsecond=0) - timedelta(days=1)

        session.add(Cell(radio=RADIO_TYPE["gsm"], mcc=1, mnc=2, lac=3, cid=4, new_measures=2, total_measures=5))
        session.add(Score(userid=1, key=SCORE_TYPE["new_cell"], value=7))
        session.flush()

        measure = dict(
            id=0,
            created=encode_datetime(time),
            lat=10000000,
            lon=20000000,
            time=encode_datetime(time),
            accuracy=0,
            altitude=0,
            altitude_accuracy=0,
            radio=RADIO_TYPE["gsm"],
        )
        entries = [
            {"mcc": 1, "mnc": 2, "lac": 3147483647, "cid": 2147483647, "psc": 5, "asu": 8},
            {"mcc": 1, "mnc": 2, "lac": -1, "cid": -1, "psc": 5, "asu": 8},
        ]
        for e in entries:
            e.update(measure)

        result = insert_cell_measures.delay(entries, userid=1)
        self.assertEqual(result.get(), 2)

        measures = session.query(CellMeasure).all()
        self.assertEqual(len(measures), 2)
        self.assertEqual(set([m.lac for m in measures]), set([-1]))
        self.assertEqual(set([m.cid for m in measures]), set([-1]))

        # Nothing should change in the initially created Cell record
        cells = session.query(Cell).all()
        self.assertEqual(len(cells), 1)
        self.assertEqual(set([c.new_measures for c in cells]), set([2]))
        self.assertEqual(set([c.total_measures for c in cells]), set([5]))
Exemple #15
0
def process_measure(data, utcnow, session, userid=None):
    measure = Measure()
    measure.created = utcnow
    measure.time = data["time"]
    measure.lat = to_precise_int(data["lat"])
    measure.lon = to_precise_int(data["lon"])
    measure.accuracy = data["accuracy"]
    measure.altitude = data["altitude"]
    measure.altitude_accuracy = data["altitude_accuracy"]
    measure.radio = RADIO_TYPE.get(data["radio"], -1)
    # get measure.id set
    session.add(measure)
    session.flush()
    measure_data = dict(
        id=measure.id,
        created=encode_datetime(measure.created),
        lat=measure.lat,
        lon=measure.lon,
        time=encode_datetime(measure.time),
        accuracy=measure.accuracy,
        altitude=measure.altitude,
        altitude_accuracy=measure.altitude_accuracy,
        radio=measure.radio,
    )
    if data.get("cell"):
        insert_cell_measure.delay(measure_data, data["cell"], userid=userid)
        measure.cell = dumps(data["cell"])
    if data.get("wifi"):
        # filter out old-style sha1 hashes
        too_long_keys = False
        for w in data["wifi"]:
            w["key"] = key = normalize_wifi_key(w["key"])
            if len(key) > 12:
                too_long_keys = True
                break
        if not too_long_keys:
            insert_wifi_measure.delay(measure_data, data["wifi"], userid=userid)
            measure.wifi = dumps(data["wifi"])
    return measure
Exemple #16
0
    def test_time(self):
        app = self.app
        # test two weeks ago and "now"
        time = (datetime.utcnow() - timedelta(14)).replace(microsecond=0)
        tstr = encode_datetime(time)
        app.post_json('/v1/submit', {
            "items": [
                {
                    "lat": 1.0,
                    "lon": 2.0,
                    "wifi": [{
                        "key": "00aaaaaaaaaa"
                    }],
                    "time": tstr
                },
                {
                    "lat": 2.0,
                    "lon": 3.0,
                    "wifi": [{
                        "key": "00bbbbbbbbbb"
                    }]
                },
            ]
        },
                      status=204)
        session = self.db_master_session
        result = session.query(WifiMeasure).all()
        self.assertEqual(len(result), 2)

        wifis = dict([(w.key, (w.created, w.time)) for w in result])
        today = datetime.utcnow().date()

        month_rounded_tday = time.replace(day=1, hour=0, minute=0, second=0)
        month_rounded_today = today.replace(day=1)

        self.assertEqual(wifis['00aaaaaaaaaa'][0].date(), today)
        self.assertEqual(wifis['00aaaaaaaaaa'][1], month_rounded_tday)

        self.assertEqual(wifis['00bbbbbbbbbb'][0].date(), today)
        self.assertEqual(wifis['00bbbbbbbbbb'][1].date(), month_rounded_today)
Exemple #17
0
def process_measure(measure_id, data, session):
    cell_measures = []
    wifi_measures = []
    measure_data = dict(
        measure_id=measure_id,
        lat=to_precise_int(data['lat']),
        lon=to_precise_int(data['lon']),
        time=encode_datetime(data['time']),
        accuracy=data['accuracy'],
        altitude=data['altitude'],
        altitude_accuracy=data['altitude_accuracy'],
    )
    measure_radio = RADIO_TYPE.get(data['radio'], -1)
    if data.get('cell'):
        # flatten measure / cell data into a single dict
        for c in data['cell']:
            c.update(measure_data)
            # use more specific cell type or
            # fall back to less precise measure
            if c['radio'] != '':
                c['radio'] = RADIO_TYPE.get(c['radio'], -1)
            else:
                c['radio'] = measure_radio
        cell_measures = data['cell']
    if data.get('wifi'):
        # filter out old-style sha1 hashes
        invalid_wifi_key = False
        for w in data['wifi']:
            w['key'] = key = normalize_wifi_key(w['key'])
            if not valid_wifi_pattern(key):
                invalid_wifi_key = True
                break

        if not invalid_wifi_key:
            # flatten measure / wifi data into a single dict
            for w in data['wifi']:
                w.update(measure_data)
            wifi_measures = data['wifi']
    return (cell_measures, wifi_measures)
Exemple #18
0
def process_measure(measure_id, data, session):
    cell_measures = []
    wifi_measures = []
    measure_data = dict(
        measure_id=measure_id,
        lat=to_precise_int(data['lat']),
        lon=to_precise_int(data['lon']),
        time=encode_datetime(data['time']),
        accuracy=data['accuracy'],
        altitude=data['altitude'],
        altitude_accuracy=data['altitude_accuracy'],
    )
    measure_radio = RADIO_TYPE.get(data['radio'], -1)
    if data.get('cell'):
        # flatten measure / cell data into a single dict
        for c in data['cell']:
            c.update(measure_data)
            # use more specific cell type or
            # fall back to less precise measure
            if c['radio'] != '':
                c['radio'] = RADIO_TYPE.get(c['radio'], -1)
            else:
                c['radio'] = measure_radio
        cell_measures = data['cell']
    if data.get('wifi'):
        # filter out old-style sha1 hashes
        invalid_wifi_key = False
        for w in data['wifi']:
            w['key'] = key = normalize_wifi_key(w['key'])
            if not valid_wifi_pattern(key):
                invalid_wifi_key = True
                break

        if not invalid_wifi_key:
            # flatten measure / wifi data into a single dict
            for w in data['wifi']:
                w.update(measure_data)
            wifi_measures = data['wifi']
    return (cell_measures, wifi_measures)
Exemple #19
0
    def test_cell(self):
        from ichnaea.service.submit.tasks import insert_cell_measure
        session = self.db_master_session
        utcnow = datetime.utcnow()

        session.add(
            Cell(radio=0,
                 mcc=1,
                 mnc=2,
                 lac=3,
                 cid=4,
                 new_measures=2,
                 total_measures=5))
        session.add(Score(userid=1, key=SCORE_TYPE['new_cell'], value=7))
        session.flush()

        measure = dict(
            id=0,
            created=encode_datetime(utcnow),
            lat=10000000,
            lon=20000000,
            time=encode_datetime(utcnow),
            accuracy=0,
            altitude=0,
            altitude_accuracy=0,
            radio=0,
        )
        entries = [
            {
                "mcc": 1,
                "mnc": 2,
                "signal": -100
            },
            {
                "mcc": 1,
                "mnc": 2,
                "lac": 3,
                "cid": 4,
                "psc": 5,
                "asu": 8
            },
            {
                "mcc": 1,
                "mnc": 2,
                "lac": 3,
                "cid": 4,
                "asu": 8
            },
            {
                "mcc": 1,
                "mnc": 2,
                "lac": 3,
                "cid": 4,
                "asu": 15
            },
            {
                "mcc": 1,
                "mnc": 2,
                "lac": 3,
                "cid": 7
            },
        ]
        result = insert_cell_measure.delay(measure, entries, userid=1)
        self.assertEqual(result.get(), 5)

        measures = session.query(CellMeasure).all()
        self.assertEqual(len(measures), 5)
        self.assertEqual(set([m.mcc for m in measures]), set([1]))
        self.assertEqual(set([m.mnc for m in measures]), set([2]))
        self.assertEqual(set([m.asu for m in measures]), set([0, 8, 15]))
        self.assertEqual(set([m.psc for m in measures]), set([0, 5]))
        self.assertEqual(set([m.signal for m in measures]), set([0, -100]))

        cells = session.query(Cell).all()
        self.assertEqual(len(cells), 2)
        self.assertEqual(set([c.mcc for c in cells]), set([1]))
        self.assertEqual(set([c.mnc for c in cells]), set([2]))
        self.assertEqual(set([c.lac for c in cells]), set([3]))
        self.assertEqual(set([c.cid for c in cells]), set([4, 7]))
        self.assertEqual(set([c.new_measures for c in cells]), set([1, 5]))
        self.assertEqual(set([c.total_measures for c in cells]), set([1, 8]))

        scores = session.query(Score).all()
        self.assertEqual(len(scores), 1)
        self.assertEqual(scores[0].key, SCORE_TYPE['new_cell'])
        self.assertEqual(scores[0].value, 10)

        # test duplicate execution
        result = insert_cell_measure.delay(measure, entries, userid=1)
        self.assertEqual(result.get(), 5)
        # TODO this task isn't idempotent yet
        measures = session.query(CellMeasure).all()
        self.assertEqual(len(measures), 10)
Exemple #20
0
    def test_cell(self):
        from ichnaea.service.submit.tasks import insert_cell_measures

        session = self.db_master_session
        time = datetime.utcnow().replace(microsecond=0) - timedelta(days=1)

        session.add(Cell(radio=RADIO_TYPE["gsm"], mcc=1, mnc=2, lac=3, cid=4, psc=5, new_measures=2, total_measures=5))
        session.add(Score(userid=1, key=SCORE_TYPE["new_cell"], value=7))
        session.flush()

        measure = dict(
            id=0,
            created=encode_datetime(time),
            lat=10000000,
            lon=20000000,
            time=encode_datetime(time),
            accuracy=0,
            altitude=0,
            altitude_accuracy=0,
            radio=RADIO_TYPE["gsm"],
        )
        entries = [
            # Note that this first entry will be skipped as it does
            # not include (lac, cid) or (psc)
            {"mcc": 1, "mnc": 2, "signal": -100},
            {"mcc": 1, "mnc": 2, "lac": 3, "cid": 4, "psc": 5, "asu": 8},
            {"mcc": 1, "mnc": 2, "lac": 3, "cid": 4, "psc": 5, "asu": 8},
            {"mcc": 1, "mnc": 2, "lac": 3, "cid": 4, "psc": 5, "asu": 15},
            {"mcc": 1, "mnc": 2, "lac": 3, "cid": 7, "psc": 5},
        ]
        for e in entries:
            e.update(measure)

        result = insert_cell_measures.delay(entries, userid=1)

        self.assertEqual(result.get(), 4)
        measures = session.query(CellMeasure).all()
        self.assertEqual(len(measures), 4)
        self.assertEqual(set([m.mcc for m in measures]), set([1]))
        self.assertEqual(set([m.mnc for m in measures]), set([2]))
        self.assertEqual(set([m.asu for m in measures]), set([-1, 8, 15]))
        self.assertEqual(set([m.psc for m in measures]), set([5]))
        self.assertEqual(set([m.signal for m in measures]), set([0]))

        cells = session.query(Cell).all()
        self.assertEqual(len(cells), 2)
        self.assertEqual(set([c.mcc for c in cells]), set([1]))
        self.assertEqual(set([c.mnc for c in cells]), set([2]))
        self.assertEqual(set([c.lac for c in cells]), set([3]))
        self.assertEqual(set([c.cid for c in cells]), set([4, 7]))
        self.assertEqual(set([c.psc for c in cells]), set([5]))
        self.assertEqual(set([c.new_measures for c in cells]), set([1, 5]))
        self.assertEqual(set([c.total_measures for c in cells]), set([1, 8]))

        scores = session.query(Score).all()
        self.assertEqual(len(scores), 1)
        self.assertEqual(scores[0].key, SCORE_TYPE["new_cell"])
        self.assertEqual(scores[0].value, 8)

        # test duplicate execution
        result = insert_cell_measures.delay(entries, userid=1)
        self.assertEqual(result.get(), 4)
        # TODO this task isn't idempotent yet
        measures = session.query(CellMeasure).all()
        self.assertEqual(len(measures), 8)
Exemple #21
0
    def test_ignore_unhelpful_incomplete_cells(self):
        # Cell records must have MNC, MCC and at least one of (LAC, CID) or PSC
        # values filled in.
        from ichnaea.service.submit.tasks import insert_cell_measures
        session = self.db_master_session
        time = datetime.utcnow().replace(microsecond=0) - timedelta(days=1)

        measure = dict(
            id=0,
            created=encode_datetime(time),
            lat=10000000,
            lon=20000000,
            time=encode_datetime(time),
            accuracy=0,
            altitude=0,
            altitude_accuracy=0,
            radio=RADIO_TYPE['gsm'],
        )
        entries = [
            # These records are valid
            {
                "mcc": 1,
                "mnc": 2,
                "lac": 3,
                "cid": 4
            },
            {
                "mcc": 1,
                "mnc": 2,
                "lac": 3,
                "cid": 4,
                "psc": 5
            },

            # This record is missing everything
            {},

            # These records fail the mcc check
            {
                "mnc": 2,
                "lac": 3,
                "cid": 4
            },
            {
                "mcc": 0,
                "mnc": 2,
                "lac": 3,
                "cid": 4
            },
            {
                "mcc": -1,
                "mnc": 2,
                "lac": 3,
                "cid": 4
            },
            {
                "mcc": -2,
                "mnc": 2,
                "lac": 3,
                "cid": 4
            },
            {
                "mcc": 2000,
                "mnc": 2,
                "lac": 3,
                "cid": 4
            },

            # These records fail the mnc check
            {
                "mcc": 1,
                "lac": 3,
                "cid": 4
            },
            {
                "mcc": 1,
                "mnc": -1,
                "lac": 3,
                "cid": 4
            },
            {
                "mcc": 1,
                "mnc": -2,
                "lac": 3,
                "cid": 4
            },
            {
                "mcc": 1,
                "mnc": 33000,
                "lac": 3,
                "cid": 4
            },

            # These records fail the lac check
            {
                "mcc": 1,
                "mnc": 2,
                "cid": 4
            },
            {
                "mcc": 1,
                "mnc": 2,
                "lac": -1,
                "cid": 4
            },
            {
                "mcc": 1,
                "mnc": 2,
                "lac": -2,
                "cid": 4
            },
            {
                "mcc": 1,
                "mnc": 2,
                "lac": 65536,
                "cid": 4
            },

            # These records fail the cid check
            {
                "mcc": 1,
                "mnc": 2,
                "lac": 3
            },
            {
                "mcc": 1,
                "mnc": 2,
                "lac": 3,
                "cid": -1
            },
            {
                "mcc": 1,
                "mnc": 2,
                "lac": 3,
                "cid": -2
            },
            {
                "mcc": 1,
                "mnc": 2,
                "lac": 3,
                "cid": 2**28
            },

            # These records fail the (lac or cid) and psc check
            {
                "mcc": 1,
                "mnc": 2
            },
            {
                "mcc": 1,
                "mnc": 2,
                "lac": 3
            },
            {
                "mcc": 1,
                "mnc": 2,
                "cid": 4
            },
        ]

        for e in entries:
            e.update(measure)
        result = insert_cell_measures.delay(entries, userid=1)

        self.assertEqual(result.get(), 2)
        measures = session.query(CellMeasure).all()
        self.assertEqual(len(measures), 2)
        cells = session.query(Cell).all()
        self.assertEqual(len(cells), 1)

        entries = [
            # These records are valid
            {
                "mcc": 1,
                "mnc": 2,
                "psc": 5
            },
            {
                "mcc": 1,
                "mnc": 2,
                "lac": 3,
                "psc": 5
            },
            {
                "mcc": 1,
                "mnc": 2,
                "cid": 4,
                "psc": 5
            },
        ]
        for e in entries:
            e.update(measure)
        result = insert_cell_measures.delay(entries, userid=1)

        self.assertEqual(result.get(), 3)
        measures = session.query(CellMeasure).all()
        self.assertEqual(len(measures), 5)
        cells = session.query(Cell).all()
        self.assertEqual(len(cells), 1)
Exemple #22
0
    def test_ignore_unhelpful_incomplete_cells(self):
        # Cell records must have MNC, MCC and at least one of (LAC, CID) or PSC
        # values filled in.
        from ichnaea.service.submit.tasks import insert_cell_measures
        session = self.db_master_session
        time = datetime.utcnow().replace(microsecond=0) - timedelta(days=1)

        measure = dict(
            id=0, created=encode_datetime(time), lat=10000000, lon=20000000,
            time=encode_datetime(time), accuracy=0, altitude=0,
            altitude_accuracy=0, radio=RADIO_TYPE['gsm'],
        )
        entries = [
            # These records are valid
            {"mcc": 1, "mnc": 2, "lac": 3, "cid": 4},
            {"mcc": 1, "mnc": 2, "lac": 3, "cid": 4, "psc": 5},

            # This record is missing everything
            {},

            # These records fail the mcc check
            {"mnc": 2, "lac": 3, "cid": 4},
            {"mcc": 0, "mnc": 2, "lac": 3, "cid": 4},
            {"mcc": -1, "mnc": 2, "lac": 3, "cid": 4},
            {"mcc": -2, "mnc": 2, "lac": 3, "cid": 4},
            {"mcc": 2000, "mnc": 2, "lac": 3, "cid": 4},

            # These records fail the mnc check
            {"mcc": 1, "lac": 3, "cid": 4},
            {"mcc": 1, "mnc": -1, "lac": 3, "cid": 4},
            {"mcc": 1, "mnc": -2, "lac": 3, "cid": 4},
            {"mcc": 1, "mnc": 33000, "lac": 3, "cid": 4},

            # These records fail the lac check
            {"mcc": 1, "mnc": 2, "cid": 4},
            {"mcc": 1, "mnc": 2, "lac": -1, "cid": 4},
            {"mcc": 1, "mnc": 2, "lac": -2, "cid": 4},
            {"mcc": 1, "mnc": 2, "lac": 65536, "cid": 4},

            # These records fail the cid check
            {"mcc": 1, "mnc": 2, "lac": 3},
            {"mcc": 1, "mnc": 2, "lac": 3, "cid": -1},
            {"mcc": 1, "mnc": 2, "lac": 3, "cid": -2},
            {"mcc": 1, "mnc": 2, "lac": 3, "cid": 2 ** 28},

            # These records fail the (lac or cid) and psc check
            {"mcc": 1, "mnc": 2},
            {"mcc": 1, "mnc": 2, "lac": 3},
            {"mcc": 1, "mnc": 2, "cid": 4},
        ]

        for e in entries:
            e.update(measure)
        result = insert_cell_measures.delay(entries, userid=1)

        self.assertEqual(result.get(), 2)
        measures = session.query(CellMeasure).all()
        self.assertEqual(len(measures), 2)
        cells = session.query(Cell).all()
        self.assertEqual(len(cells), 1)

        entries = [
            # These records are valid
            {"mcc": 1, "mnc": 2, "psc": 5},
            {"mcc": 1, "mnc": 2, "lac": 3, "psc": 5},
            {"mcc": 1, "mnc": 2, "cid": 4, "psc": 5},
        ]
        for e in entries:
            e.update(measure)
        result = insert_cell_measures.delay(entries, userid=1)

        self.assertEqual(result.get(), 3)
        measures = session.query(CellMeasure).all()
        self.assertEqual(len(measures), 5)
        cells = session.query(Cell).all()
        self.assertEqual(len(cells), 1)
Exemple #23
0
    def test_cell(self):
        from ichnaea.service.submit.tasks import insert_cell_measures
        session = self.db_master_session
        time = datetime.utcnow().replace(microsecond=0) - timedelta(days=1)

        session.add(
            Cell(radio=RADIO_TYPE['gsm'],
                 mcc=1,
                 mnc=2,
                 lac=3,
                 cid=4,
                 psc=5,
                 new_measures=2,
                 total_measures=5))
        session.add(Score(userid=1, key=SCORE_TYPE['new_cell'], value=7))
        session.flush()

        measure = dict(
            id=0,
            created=encode_datetime(time),
            lat=10000000,
            lon=20000000,
            time=encode_datetime(time),
            accuracy=0,
            altitude=0,
            altitude_accuracy=0,
            radio=RADIO_TYPE['gsm'],
        )
        entries = [
            # Note that this first entry will be skipped as it does
            # not include (lac, cid) or (psc)
            {
                "mcc": 1,
                "mnc": 2,
                "signal": -100
            },
            {
                "mcc": 1,
                "mnc": 2,
                "lac": 3,
                "cid": 4,
                "psc": 5,
                "asu": 8
            },
            {
                "mcc": 1,
                "mnc": 2,
                "lac": 3,
                "cid": 4,
                "psc": 5,
                "asu": 8
            },
            {
                "mcc": 1,
                "mnc": 2,
                "lac": 3,
                "cid": 4,
                "psc": 5,
                "asu": 15
            },
            {
                "mcc": 1,
                "mnc": 2,
                "lac": 3,
                "cid": 7,
                "psc": 5
            },
        ]
        for e in entries:
            e.update(measure)

        result = insert_cell_measures.delay(entries, userid=1)

        self.assertEqual(result.get(), 4)
        measures = session.query(CellMeasure).all()
        self.assertEqual(len(measures), 4)
        self.assertEqual(set([m.mcc for m in measures]), set([1]))
        self.assertEqual(set([m.mnc for m in measures]), set([2]))
        self.assertEqual(set([m.asu for m in measures]), set([-1, 8, 15]))
        self.assertEqual(set([m.psc for m in measures]), set([5]))
        self.assertEqual(set([m.signal for m in measures]), set([0]))

        cells = session.query(Cell).all()
        self.assertEqual(len(cells), 2)
        self.assertEqual(set([c.mcc for c in cells]), set([1]))
        self.assertEqual(set([c.mnc for c in cells]), set([2]))
        self.assertEqual(set([c.lac for c in cells]), set([3]))
        self.assertEqual(set([c.cid for c in cells]), set([4, 7]))
        self.assertEqual(set([c.psc for c in cells]), set([5]))
        self.assertEqual(set([c.new_measures for c in cells]), set([1, 5]))
        self.assertEqual(set([c.total_measures for c in cells]), set([1, 8]))

        scores = session.query(Score).all()
        self.assertEqual(len(scores), 1)
        self.assertEqual(scores[0].key, SCORE_TYPE['new_cell'])
        self.assertEqual(scores[0].value, 8)

        # test duplicate execution
        result = insert_cell_measures.delay(entries, userid=1)
        self.assertEqual(result.get(), 4)
        # TODO this task isn't idempotent yet
        measures = session.query(CellMeasure).all()
        self.assertEqual(len(measures), 8)
Exemple #24
0
    def test_wifi(self):
        from ichnaea.service.submit.tasks import insert_wifi_measures
        session = self.db_master_session
        time = datetime.utcnow().replace(microsecond=0) - timedelta(days=1)

        session.add(Wifi(key="ab12"))
        session.add(Score(userid=1, key=SCORE_TYPE['new_wifi'], value=7))
        session.flush()

        measure = dict(
            id=0,
            created=encode_datetime(time),
            lat=10000000,
            lon=20000000,
            time=encode_datetime(time),
            accuracy=0,
            altitude=0,
            altitude_accuracy=0,
            radio=-1,
        )
        entries = [
            {
                "key": "ab12",
                "channel": 11,
                "signal": -80
            },
            {
                "key": "ab12",
                "channel": 3,
                "signal": -90
            },
            {
                "key": "ab12",
                "channel": 3,
                "signal": -80
            },
            {
                "key": "cd34",
                "channel": 3,
                "signal": -90
            },
        ]
        for e in entries:
            e.update(measure)
        result = insert_wifi_measures.delay(entries, userid=1)
        self.assertEqual(result.get(), 4)

        measures = session.query(WifiMeasure).all()
        self.assertEqual(len(measures), 4)
        self.assertEqual(set([m.key for m in measures]), set(["ab12", "cd34"]))
        self.assertEqual(set([m.channel for m in measures]), set([3, 11]))
        self.assertEqual(set([m.signal for m in measures]), set([-80, -90]))

        wifis = session.query(Wifi).all()
        self.assertEqual(len(wifis), 2)
        self.assertEqual(set([w.key for w in wifis]), set(["ab12", "cd34"]))
        self.assertEqual(set([w.new_measures for w in wifis]), set([1, 3]))
        self.assertEqual(set([w.total_measures for w in wifis]), set([1, 3]))

        scores = session.query(Score).all()
        self.assertEqual(len(scores), 1)
        self.assertEqual(scores[0].key, SCORE_TYPE['new_wifi'])
        self.assertEqual(scores[0].value, 8)

        # test duplicate execution
        result = insert_wifi_measures.delay(entries, userid=1)
        self.assertEqual(result.get(), 4)
        # TODO this task isn't idempotent yet
        measures = session.query(WifiMeasure).all()
        self.assertEqual(len(measures), 8)