def test_cell_overflow(self): from ichnaea.service.submit.tasks import insert_cell_measures session = self.db_master_session measures = [ dict(mcc=1, mnc=2, lac=3, cid=4, psc=5, radio=RADIO_TYPE['gsm'], id=0, lat=10000000 + i, lon=20000000 + i) for i in range(3) ] result = insert_cell_measures.delay(measures) self.assertEqual(result.get(), 3) result = insert_cell_measures.delay(measures, max_measures_per_cell=3) self.assertEqual(result.get(), 0) result = insert_cell_measures.delay(measures, max_measures_per_cell=10) self.assertEqual(result.get(), 3) result = insert_cell_measures.delay(measures, max_measures_per_cell=3) self.assertEqual(result.get(), 0) measures = session.query(CellMeasure).all() self.assertEqual(len(measures), 6) cells = session.query(Cell).all() self.assertEqual(len(cells), 1) self.assertEqual(cells[0].total_measures, 6)
def test_cell_overflow(self): from ichnaea.service.submit.tasks import insert_cell_measures session = self.db_master_session measures = [dict(mcc=1, mnc=2, lac=3, cid=4, psc=5, radio=RADIO_TYPE['gsm'], id=0, lat=10000000+i, lon=20000000+i) for i in range(3)] result = insert_cell_measures.delay(measures) self.assertEqual(result.get(), 3) result = insert_cell_measures.delay(measures, max_measures_per_cell=3) self.assertEqual(result.get(), 0) result = insert_cell_measures.delay(measures, max_measures_per_cell=10) self.assertEqual(result.get(), 3) result = insert_cell_measures.delay(measures, max_measures_per_cell=3) self.assertEqual(result.get(), 0) measures = session.query(CellMeasure).all() self.assertEqual(len(measures), 6) cells = session.query(Cell).all() self.assertEqual(len(cells), 1) self.assertEqual(cells[0].total_measures, 6)
def test_cell_blacklist(self): from ichnaea.service.submit.tasks import insert_cell_measures session = self.db_master_session measures = [ dict(mcc=1, mnc=2, lac=3, cid=i, psc=5, radio=RADIO_TYPE['gsm'], id=0, lat=10000000 + i, lon=20000000 + i) for i in range(3) ] black = CellBlacklist( mcc=1, mnc=2, lac=3, cid=1, radio=RADIO_TYPE['gsm'], ) session.add(black) session.flush() result = insert_cell_measures.delay(measures) self.assertEqual(result.get(), 3) measures = session.query(CellMeasure).all() self.assertEqual(len(measures), 3) cells = session.query(Cell).all() self.assertEqual(len(cells), 2)
def test_ignore_unhelpful_incomplete_cdma_cells(self): # CDMA cell records must have MNC, MCC, LAC and CID filled in from ichnaea.service.submit.tasks import insert_cell_measures session = self.db_master_session time = datetime.utcnow().replace(microsecond=0) - timedelta(days=1) measure = dict( id=0, created=encode_datetime(time), lat=10000000, lon=20000000, time=encode_datetime(time), accuracy=0, altitude=0, altitude_accuracy=0, radio=RADIO_TYPE['cdma'], ) entries = [ # This records is valid {"mcc": 1, "mnc": 2, "lac": 3, "cid": 4}, # This record should fail as it's missing CID {"mcc": 1, "mnc": 2, "lac": 3}, # This fails for missing lac {"mcc": 1, "mnc": 2, "cid": 4}, # Adding a psc doesn't change things {"mcc": 1, "mnc": 2, "psc": 5}, ] for e in entries: e.update(measure) result = insert_cell_measures.delay(entries, userid=1) self.assertEqual(result.get(), 1) measures = session.query(CellMeasure).all() self.assertEqual(len(measures), 1) cells = session.query(Cell).all() self.assertEqual(len(cells), 1)
def test_cell_out_of_range_values(self): from ichnaea.service.submit.tasks import insert_cell_measures session = self.db_master_session time = datetime.utcnow().replace(microsecond=0) - timedelta(days=1) measure = dict( id=0, created=encode_datetime(time), lat=10000000, lon=20000000, time=encode_datetime(time), accuracy=0, altitude=0, altitude_accuracy=0, radio=RADIO_TYPE['gsm'], mcc=1, mnc=2, lac=3, cid=4) entries = [ {"asu": 8, "signal": -70, "ta": 32}, {"asu": -10, "signal": -300, "ta": -10}, {"asu": 256, "signal": 16, "ta": 128}, ] for e in entries: e.update(measure) result = insert_cell_measures.delay(entries) self.assertEqual(result.get(), 3) measures = session.query(CellMeasure).all() self.assertEqual(len(measures), 3) self.assertEqual(set([m.asu for m in measures]), set([-1, 8])) self.assertEqual(set([m.signal for m in measures]), set([0, -70])) self.assertEqual(set([m.ta for m in measures]), set([0, 32]))
def test_cell_blacklist(self): from ichnaea.service.submit.tasks import insert_cell_measures session = self.db_master_session measures = [dict(mcc=1, mnc=2, lac=3, cid=i, psc=5, radio=RADIO_TYPE['gsm'], id=0, lat=10000000 + i, lon=20000000 + i) for i in range(3)] black = CellBlacklist( mcc=1, mnc=2, lac=3, cid=1, radio=RADIO_TYPE['gsm'], ) session.add(black) session.flush() result = insert_cell_measures.delay(measures) self.assertEqual(result.get(), 3) measures = session.query(CellMeasure).all() self.assertEqual(len(measures), 3) cells = session.query(Cell).all() self.assertEqual(len(cells), 2)
def test_insert_invalid_lac(self): from ichnaea.service.submit.tasks import insert_cell_measures session = self.db_master_session time = datetime.utcnow().replace(microsecond=0) - timedelta(days=1) session.add( Cell(radio=RADIO_TYPE['gsm'], mcc=1, mnc=2, lac=3, cid=4, new_measures=2, total_measures=5)) session.add(Score(userid=1, key=SCORE_TYPE['new_cell'], value=7)) session.flush() measure = dict(id=0, created=encode_datetime(time), lat=10000000, lon=20000000, time=encode_datetime(time), accuracy=0, altitude=0, altitude_accuracy=0, radio=RADIO_TYPE['gsm']) entries = [ { "mcc": 1, "mnc": 2, "lac": 3147483647, "cid": 2147483647, "psc": 5, "asu": 8 }, { "mcc": 1, "mnc": 2, "lac": -1, "cid": -1, "psc": 5, "asu": 8 }, ] for e in entries: e.update(measure) result = insert_cell_measures.delay(entries, userid=1) self.assertEqual(result.get(), 2) measures = session.query(CellMeasure).all() self.assertEqual(len(measures), 2) self.assertEqual(set([m.lac for m in measures]), set([-1])) self.assertEqual(set([m.cid for m in measures]), set([-1])) # Nothing should change in the initially created Cell record cells = session.query(Cell).all() self.assertEqual(len(cells), 1) self.assertEqual(set([c.new_measures for c in cells]), set([2])) self.assertEqual(set([c.total_measures for c in cells]), set([5]))
def test_ignore_unhelpful_incomplete_cdma_cells(self): # CDMA cell records must have MNC, MCC, LAC and CID filled in from ichnaea.service.submit.tasks import insert_cell_measures session = self.db_master_session time = datetime.utcnow().replace(microsecond=0) - timedelta(days=1) measure = dict( id=0, created=encode_datetime(time), lat=10000000, lon=20000000, time=encode_datetime(time), accuracy=0, altitude=0, altitude_accuracy=0, radio=RADIO_TYPE['cdma'], ) entries = [ # This records is valid { "mcc": 1, "mnc": 2, "lac": 3, "cid": 4 }, # This record should fail as it's missing CID { "mcc": 1, "mnc": 2, "lac": 3 }, # This fails for missing lac { "mcc": 1, "mnc": 2, "cid": 4 }, # Adding a psc doesn't change things { "mcc": 1, "mnc": 2, "psc": 5 }, ] for e in entries: e.update(measure) result = insert_cell_measures.delay(entries, userid=1) self.assertEqual(result.get(), 1) measures = session.query(CellMeasure).all() self.assertEqual(len(measures), 1) cells = session.query(Cell).all() self.assertEqual(len(cells), 1)
def test_cell_out_of_range_values(self): from ichnaea.service.submit.tasks import insert_cell_measures session = self.db_master_session time = datetime.utcnow().replace(microsecond=0) - timedelta(days=1) measure = dict(id=0, created=encode_datetime(time), lat=10000000, lon=20000000, time=encode_datetime(time), accuracy=0, altitude=0, altitude_accuracy=0, radio=RADIO_TYPE['gsm'], mcc=1, mnc=2, lac=3, cid=4) entries = [ { "asu": 8, "signal": -70, "ta": 32 }, { "asu": -10, "signal": -300, "ta": -10 }, { "asu": 256, "signal": 16, "ta": 128 }, ] for e in entries: e.update(measure) result = insert_cell_measures.delay(entries) self.assertEqual(result.get(), 3) measures = session.query(CellMeasure).all() self.assertEqual(len(measures), 3) self.assertEqual(set([m.asu for m in measures]), set([-1, 8])) self.assertEqual(set([m.signal for m in measures]), set([0, -70])) self.assertEqual(set([m.ta for m in measures]), set([0, 32]))
def test_insert_invalid_lac(self): from ichnaea.service.submit.tasks import insert_cell_measures session = self.db_master_session time = datetime.utcnow().replace(microsecond=0) - timedelta(days=1) session.add(Cell(radio=RADIO_TYPE["gsm"], mcc=1, mnc=2, lac=3, cid=4, new_measures=2, total_measures=5)) session.add(Score(userid=1, key=SCORE_TYPE["new_cell"], value=7)) session.flush() measure = dict( id=0, created=encode_datetime(time), lat=10000000, lon=20000000, time=encode_datetime(time), accuracy=0, altitude=0, altitude_accuracy=0, radio=RADIO_TYPE["gsm"], ) entries = [ {"mcc": 1, "mnc": 2, "lac": 3147483647, "cid": 2147483647, "psc": 5, "asu": 8}, {"mcc": 1, "mnc": 2, "lac": -1, "cid": -1, "psc": 5, "asu": 8}, ] for e in entries: e.update(measure) result = insert_cell_measures.delay(entries, userid=1) self.assertEqual(result.get(), 2) measures = session.query(CellMeasure).all() self.assertEqual(len(measures), 2) self.assertEqual(set([m.lac for m in measures]), set([-1])) self.assertEqual(set([m.cid for m in measures]), set([-1])) # Nothing should change in the initially created Cell record cells = session.query(Cell).all() self.assertEqual(len(cells), 1) self.assertEqual(set([c.new_measures for c in cells]), set([2])) self.assertEqual(set([c.total_measures for c in cells]), set([5]))
def test_ignore_unhelpful_incomplete_cells(self): # Cell records must have MNC, MCC and at least one of (LAC, CID) or PSC # values filled in. from ichnaea.service.submit.tasks import insert_cell_measures session = self.db_master_session time = datetime.utcnow().replace(microsecond=0) - timedelta(days=1) measure = dict( id=0, created=encode_datetime(time), lat=10000000, lon=20000000, time=encode_datetime(time), accuracy=0, altitude=0, altitude_accuracy=0, radio=RADIO_TYPE['gsm'], ) entries = [ # These records are valid { "mcc": 1, "mnc": 2, "lac": 3, "cid": 4 }, { "mcc": 1, "mnc": 2, "lac": 3, "cid": 4, "psc": 5 }, # This record is missing everything {}, # These records fail the mcc check { "mnc": 2, "lac": 3, "cid": 4 }, { "mcc": 0, "mnc": 2, "lac": 3, "cid": 4 }, { "mcc": -1, "mnc": 2, "lac": 3, "cid": 4 }, { "mcc": -2, "mnc": 2, "lac": 3, "cid": 4 }, { "mcc": 2000, "mnc": 2, "lac": 3, "cid": 4 }, # These records fail the mnc check { "mcc": 1, "lac": 3, "cid": 4 }, { "mcc": 1, "mnc": -1, "lac": 3, "cid": 4 }, { "mcc": 1, "mnc": -2, "lac": 3, "cid": 4 }, { "mcc": 1, "mnc": 33000, "lac": 3, "cid": 4 }, # These records fail the lac check { "mcc": 1, "mnc": 2, "cid": 4 }, { "mcc": 1, "mnc": 2, "lac": -1, "cid": 4 }, { "mcc": 1, "mnc": 2, "lac": -2, "cid": 4 }, { "mcc": 1, "mnc": 2, "lac": 65536, "cid": 4 }, # These records fail the cid check { "mcc": 1, "mnc": 2, "lac": 3 }, { "mcc": 1, "mnc": 2, "lac": 3, "cid": -1 }, { "mcc": 1, "mnc": 2, "lac": 3, "cid": -2 }, { "mcc": 1, "mnc": 2, "lac": 3, "cid": 2**28 }, # These records fail the (lac or cid) and psc check { "mcc": 1, "mnc": 2 }, { "mcc": 1, "mnc": 2, "lac": 3 }, { "mcc": 1, "mnc": 2, "cid": 4 }, ] for e in entries: e.update(measure) result = insert_cell_measures.delay(entries, userid=1) self.assertEqual(result.get(), 2) measures = session.query(CellMeasure).all() self.assertEqual(len(measures), 2) cells = session.query(Cell).all() self.assertEqual(len(cells), 1) entries = [ # These records are valid { "mcc": 1, "mnc": 2, "psc": 5 }, { "mcc": 1, "mnc": 2, "lac": 3, "psc": 5 }, { "mcc": 1, "mnc": 2, "cid": 4, "psc": 5 }, ] for e in entries: e.update(measure) result = insert_cell_measures.delay(entries, userid=1) self.assertEqual(result.get(), 3) measures = session.query(CellMeasure).all() self.assertEqual(len(measures), 5) cells = session.query(Cell).all() self.assertEqual(len(cells), 1)
def test_cell(self): from ichnaea.service.submit.tasks import insert_cell_measures session = self.db_master_session time = datetime.utcnow().replace(microsecond=0) - timedelta(days=1) session.add( Cell(radio=RADIO_TYPE['gsm'], mcc=1, mnc=2, lac=3, cid=4, psc=5, new_measures=2, total_measures=5)) session.add(Score(userid=1, key=SCORE_TYPE['new_cell'], value=7)) session.flush() measure = dict( id=0, created=encode_datetime(time), lat=10000000, lon=20000000, time=encode_datetime(time), accuracy=0, altitude=0, altitude_accuracy=0, radio=RADIO_TYPE['gsm'], ) entries = [ # Note that this first entry will be skipped as it does # not include (lac, cid) or (psc) { "mcc": 1, "mnc": 2, "signal": -100 }, { "mcc": 1, "mnc": 2, "lac": 3, "cid": 4, "psc": 5, "asu": 8 }, { "mcc": 1, "mnc": 2, "lac": 3, "cid": 4, "psc": 5, "asu": 8 }, { "mcc": 1, "mnc": 2, "lac": 3, "cid": 4, "psc": 5, "asu": 15 }, { "mcc": 1, "mnc": 2, "lac": 3, "cid": 7, "psc": 5 }, ] for e in entries: e.update(measure) result = insert_cell_measures.delay(entries, userid=1) self.assertEqual(result.get(), 4) measures = session.query(CellMeasure).all() self.assertEqual(len(measures), 4) self.assertEqual(set([m.mcc for m in measures]), set([1])) self.assertEqual(set([m.mnc for m in measures]), set([2])) self.assertEqual(set([m.asu for m in measures]), set([-1, 8, 15])) self.assertEqual(set([m.psc for m in measures]), set([5])) self.assertEqual(set([m.signal for m in measures]), set([0])) cells = session.query(Cell).all() self.assertEqual(len(cells), 2) self.assertEqual(set([c.mcc for c in cells]), set([1])) self.assertEqual(set([c.mnc for c in cells]), set([2])) self.assertEqual(set([c.lac for c in cells]), set([3])) self.assertEqual(set([c.cid for c in cells]), set([4, 7])) self.assertEqual(set([c.psc for c in cells]), set([5])) self.assertEqual(set([c.new_measures for c in cells]), set([1, 5])) self.assertEqual(set([c.total_measures for c in cells]), set([1, 8])) scores = session.query(Score).all() self.assertEqual(len(scores), 1) self.assertEqual(scores[0].key, SCORE_TYPE['new_cell']) self.assertEqual(scores[0].value, 8) # test duplicate execution result = insert_cell_measures.delay(entries, userid=1) self.assertEqual(result.get(), 4) # TODO this task isn't idempotent yet measures = session.query(CellMeasure).all() self.assertEqual(len(measures), 8)
def test_ignore_unhelpful_incomplete_cells(self): # Cell records must have MNC, MCC and at least one of (LAC, CID) or PSC # values filled in. from ichnaea.service.submit.tasks import insert_cell_measures session = self.db_master_session time = datetime.utcnow().replace(microsecond=0) - timedelta(days=1) measure = dict( id=0, created=encode_datetime(time), lat=10000000, lon=20000000, time=encode_datetime(time), accuracy=0, altitude=0, altitude_accuracy=0, radio=RADIO_TYPE['gsm'], ) entries = [ # These records are valid {"mcc": 1, "mnc": 2, "lac": 3, "cid": 4}, {"mcc": 1, "mnc": 2, "lac": 3, "cid": 4, "psc": 5}, # This record is missing everything {}, # These records fail the mcc check {"mnc": 2, "lac": 3, "cid": 4}, {"mcc": 0, "mnc": 2, "lac": 3, "cid": 4}, {"mcc": -1, "mnc": 2, "lac": 3, "cid": 4}, {"mcc": -2, "mnc": 2, "lac": 3, "cid": 4}, {"mcc": 2000, "mnc": 2, "lac": 3, "cid": 4}, # These records fail the mnc check {"mcc": 1, "lac": 3, "cid": 4}, {"mcc": 1, "mnc": -1, "lac": 3, "cid": 4}, {"mcc": 1, "mnc": -2, "lac": 3, "cid": 4}, {"mcc": 1, "mnc": 33000, "lac": 3, "cid": 4}, # These records fail the lac check {"mcc": 1, "mnc": 2, "cid": 4}, {"mcc": 1, "mnc": 2, "lac": -1, "cid": 4}, {"mcc": 1, "mnc": 2, "lac": -2, "cid": 4}, {"mcc": 1, "mnc": 2, "lac": 65536, "cid": 4}, # These records fail the cid check {"mcc": 1, "mnc": 2, "lac": 3}, {"mcc": 1, "mnc": 2, "lac": 3, "cid": -1}, {"mcc": 1, "mnc": 2, "lac": 3, "cid": -2}, {"mcc": 1, "mnc": 2, "lac": 3, "cid": 2 ** 28}, # These records fail the (lac or cid) and psc check {"mcc": 1, "mnc": 2}, {"mcc": 1, "mnc": 2, "lac": 3}, {"mcc": 1, "mnc": 2, "cid": 4}, ] for e in entries: e.update(measure) result = insert_cell_measures.delay(entries, userid=1) self.assertEqual(result.get(), 2) measures = session.query(CellMeasure).all() self.assertEqual(len(measures), 2) cells = session.query(Cell).all() self.assertEqual(len(cells), 1) entries = [ # These records are valid {"mcc": 1, "mnc": 2, "psc": 5}, {"mcc": 1, "mnc": 2, "lac": 3, "psc": 5}, {"mcc": 1, "mnc": 2, "cid": 4, "psc": 5}, ] for e in entries: e.update(measure) result = insert_cell_measures.delay(entries, userid=1) self.assertEqual(result.get(), 3) measures = session.query(CellMeasure).all() self.assertEqual(len(measures), 5) cells = session.query(Cell).all() self.assertEqual(len(cells), 1)
def test_cell(self): from ichnaea.service.submit.tasks import insert_cell_measures session = self.db_master_session time = datetime.utcnow().replace(microsecond=0) - timedelta(days=1) session.add(Cell(radio=RADIO_TYPE['gsm'], mcc=1, mnc=2, lac=3, cid=4, psc=5, new_measures=2, total_measures=5)) session.add(Score(userid=1, key=SCORE_TYPE['new_cell'], value=7)) session.flush() measure = dict( id=0, created=encode_datetime(time), lat=10000000, lon=20000000, time=encode_datetime(time), accuracy=0, altitude=0, altitude_accuracy=0, radio=RADIO_TYPE['gsm'], ) entries = [ # Note that this first entry will be skipped as it does # not include (lac, cid) or (psc) {"mcc": 1, "mnc": 2, "signal": -100}, {"mcc": 1, "mnc": 2, "lac": 3, "cid": 4, "psc": 5, "asu": 8}, {"mcc": 1, "mnc": 2, "lac": 3, "cid": 4, "psc": 5, "asu": 8}, {"mcc": 1, "mnc": 2, "lac": 3, "cid": 4, "psc": 5, "asu": 15}, {"mcc": 1, "mnc": 2, "lac": 3, "cid": 7, "psc": 5}, ] for e in entries: e.update(measure) result = insert_cell_measures.delay(entries, userid=1) self.assertEqual(result.get(), 4) measures = session.query(CellMeasure).all() self.assertEqual(len(measures), 4) self.assertEqual(set([m.mcc for m in measures]), set([1])) self.assertEqual(set([m.mnc for m in measures]), set([2])) self.assertEqual(set([m.asu for m in measures]), set([-1, 8, 15])) self.assertEqual(set([m.psc for m in measures]), set([5])) self.assertEqual(set([m.signal for m in measures]), set([0])) cells = session.query(Cell).all() self.assertEqual(len(cells), 2) self.assertEqual(set([c.mcc for c in cells]), set([1])) self.assertEqual(set([c.mnc for c in cells]), set([2])) self.assertEqual(set([c.lac for c in cells]), set([3])) self.assertEqual(set([c.cid for c in cells]), set([4, 7])) self.assertEqual(set([c.psc for c in cells]), set([5])) self.assertEqual(set([c.new_measures for c in cells]), set([1, 5])) self.assertEqual(set([c.total_measures for c in cells]), set([1, 8])) scores = session.query(Score).all() self.assertEqual(len(scores), 1) self.assertEqual(scores[0].key, SCORE_TYPE['new_cell']) self.assertEqual(scores[0].value, 8) # test duplicate execution result = insert_cell_measures.delay(entries, userid=1) self.assertEqual(result.get(), 4) # TODO this task isn't idempotent yet measures = session.query(CellMeasure).all() self.assertEqual(len(measures), 8)