def test_blacklist_time_used_as_creation_time(self): now = util.utcnow() last_week = now - TEMPORARY_BLACKLIST_DURATION - timedelta(days=1) session = self.session cell_key = {'mcc': FRANCE_MCC, 'mnc': 2, 'lac': 3, 'cid': 1} session.add(CellBlacklist(time=last_week, count=1, radio=Radio.gsm, **cell_key)) session.flush() # add a new entry for the previously blacklisted cell obs = dict(lat=PARIS_LAT, lon=PARIS_LON, radio=int(Radio.gsm), **cell_key) insert_measures_cell.delay([obs]).get() self.assertEqual(self.data_queue.size(), 1) update_cell.delay().get() # the cell was inserted again cells = session.query(Cell).all() self.assertEqual(len(cells), 1) # and the creation date was set to the date of the blacklist entry self.assertEqual(cells[0].created, last_week) self.check_statcounter(StatKey.cell, 1) self.check_statcounter(StatKey.unique_cell, 0)
def test_blacklist_time_used_as_creation_time(self): now = util.utcnow() last_week = now - TEMPORARY_BLACKLIST_DURATION - timedelta(days=1) cell = CellFactory.build() self.session.add( CellBlacklist(time=last_week, count=1, radio=cell.radio, mcc=cell.mcc, mnc=cell.mnc, lac=cell.lac, cid=cell.cid)) self.session.flush() # add a new entry for the previously blacklisted cell obs = dict(lat=cell.lat, lon=cell.lon, radio=int(cell.radio), mcc=cell.mcc, mnc=cell.mnc, lac=cell.lac, cid=cell.cid) insert_measures_cell.delay([obs]).get() self.assertEqual(self.data_queue.size(), 1) update_cell.delay().get() # the cell was inserted again cells = self.session.query(Cell).all() self.assertEqual(len(cells), 1) # and the creation date was set to the date of the blacklist entry self.assertEqual(cells[0].created, last_week) self.check_statcounter(StatKey.cell, 1) self.check_statcounter(StatKey.unique_cell, 0)
def test_insert_measures_overflow(self): session = self.db_master_session measures = [ dict(mcc=FRANCE_MCC, mnc=2, lac=3, cid=4, psc=5, radio=RADIO_TYPE['gsm'], lat=PARIS_LAT + i * 0.0000001, lon=PARIS_LON + i * 0.0000001) for i in range(3) ] result = insert_measures_cell.delay(measures) self.assertEqual(result.get(), 3) result = insert_measures_cell.delay(measures, max_measures_per_cell=3) self.assertEqual(result.get(), 0) result = insert_measures_cell.delay(measures, max_measures_per_cell=10) self.assertEqual(result.get(), 3) result = insert_measures_cell.delay(measures, max_measures_per_cell=3) self.assertEqual(result.get(), 0) measures = session.query(CellMeasure).all() self.assertEqual(len(measures), 6) cells = session.query(Cell).all() self.assertEqual(len(cells), 1) self.assertEqual(cells[0].total_measures, 6)
def test_insert_measures_overflow(self): session = self.db_master_session measures = [dict(mcc=FRANCE_MCC, mnc=2, lac=3, cid=4, psc=5, radio=RADIO_TYPE['gsm'], lat=PARIS_LAT + i * 0.0000001, lon=PARIS_LON + i * 0.0000001) for i in range(3)] result = insert_measures_cell.delay(measures) self.assertEqual(result.get(), 3) result = insert_measures_cell.delay(measures, max_measures_per_cell=3) self.assertEqual(result.get(), 0) result = insert_measures_cell.delay(measures, max_measures_per_cell=10) self.assertEqual(result.get(), 3) result = insert_measures_cell.delay(measures, max_measures_per_cell=3) self.assertEqual(result.get(), 0) measures = session.query(CellMeasure).all() self.assertEqual(len(measures), 6) cells = session.query(Cell).all() self.assertEqual(len(cells), 1) self.assertEqual(cells[0].total_measures, 6)
def test_blacklist_time_used_as_creation_time(self): now = util.utcnow() last_week = now - TEMPORARY_BLACKLIST_DURATION - timedelta(days=1) session = self.db_master_session cell_key = { 'radio': RADIO_TYPE['gsm'], 'mcc': FRANCE_MCC, 'mnc': 2, 'lac': 3, 'cid': 1 } session.add(CellBlacklist(time=last_week, count=1, **cell_key)) session.flush() # add a new entry for the previously blacklisted cell obs = dict(lat=PARIS_LAT, lon=PARIS_LON, **cell_key) insert_measures_cell.delay([obs]).get() # the cell was inserted again cells = session.query(Cell).all() self.assertEqual(len(cells), 1) # and the creation date was set to the date of the blacklist entry self.assertEqual(cells[0].created, last_week)
def test_insert_measures_out_of_range(self): session = self.db_master_session time = util.utcnow() - timedelta(days=1) measure = dict( id=0, created=encode_datetime(time), lat=PARIS_LAT, lon=PARIS_LON, time=encode_datetime(time), accuracy=0, altitude=0, altitude_accuracy=0, radio=RADIO_TYPE['gsm'], mcc=FRANCE_MCC, mnc=2, lac=3, cid=4) entries = [ {"asu": 8, "signal": -70, "ta": 32}, {"asu": -10, "signal": -300, "ta": -10}, {"asu": 256, "signal": 16, "ta": 128}, ] for e in entries: e.update(measure) result = insert_measures_cell.delay(entries) self.assertEqual(result.get(), 3) measures = session.query(CellMeasure).all() self.assertEqual(len(measures), 3) self.assertEqual(set([m.asu for m in measures]), set([-1, 8])) self.assertEqual(set([m.signal for m in measures]), set([0, -70])) self.assertEqual(set([m.ta for m in measures]), set([0, 32]))
def test_blacklist(self): now = util.utcnow() session = self.session observations = [dict(mcc=FRANCE_MCC, mnc=2, lac=3, cid=i, psc=5, radio=int(Radio.gsm), lat=PARIS_LAT + i * 0.0000001, lon=PARIS_LON + i * 0.0000001) for i in range(1, 4)] black = CellBlacklist( mcc=FRANCE_MCC, mnc=2, lac=3, cid=1, radio=Radio.gsm, time=now, count=1, ) session.add(black) session.flush() result = insert_measures_cell.delay(observations) self.assertEqual(result.get(), 2) cell_observations = session.query(CellObservation).all() self.assertEqual(len(cell_observations), 2) cells = session.query(Cell).all() self.assertEqual(len(cells), 2)
def test_blacklist(self): session = self.db_master_session measures = [ dict(mcc=FRANCE_MCC, mnc=2, lac=3, cid=i, psc=5, radio=RADIO_TYPE['gsm'], lat=PARIS_LAT + i * 0.0000001, lon=PARIS_LON + i * 0.0000001) for i in range(1, 4) ] black = CellBlacklist( mcc=FRANCE_MCC, mnc=2, lac=3, cid=1, radio=RADIO_TYPE['gsm'], ) session.add(black) session.flush() result = insert_measures_cell.delay(measures) self.assertEqual(result.get(), 2) measures = session.query(CellMeasure).all() self.assertEqual(len(measures), 2) cells = session.query(Cell).all() self.assertEqual(len(cells), 2)
def test_insert_observations_invalid_lac(self): time = util.utcnow() - timedelta(days=1) today = util.utcnow().date() cell = CellFactory(total_measures=5) self.session.add(Score(key=ScoreKey.new_cell, userid=1, time=today, value=7)) self.session.flush() obs = dict( radio=int(cell.radio), mcc=cell.mcc, mnc=cell.mnc, psc=cell.psc, created=time, time=time, lat=cell.lat, lon=cell.lon, accuracy=0, altitude=0, altitude_accuracy=0) entries = [ {'lac': constants.MAX_LAC_ALL + 1, 'cid': constants.MAX_CID_ALL + 1, 'asu': 8}, {'lac': None, 'cid': None, 'asu': 8}, ] for entry in entries: entry.update(obs) result = insert_measures_cell.delay(entries, userid=1) self.assertEqual(result.get(), 0) # The incomplete observations never make it into the queue self.assertEqual(self.data_queue.size(), 0) update_cell.delay().get() # Nothing should change in the initially created Cell record self.session.refresh(cell) cells = self.session.query(Cell).all() self.assertEqual(len(cells), 1) self._compare_sets([c.total_measures for c in cells], [5])
def test_insert_observations_out_of_range(self): session = self.session time = util.utcnow() - timedelta(days=1) obs = dict( created=time, lat=PARIS_LAT, lon=PARIS_LON, time=time, accuracy=0, altitude=0, altitude_accuracy=0, radio=int(Radio.gsm), mcc=FRANCE_MCC, mnc=2, lac=3, cid=4) entries = [ {"asu": 8, "signal": -70, "ta": 32}, {"asu": -10, "signal": -300, "ta": -10}, {"asu": 256, "signal": 16, "ta": 128}, ] for e in entries: e.update(obs) result = insert_measures_cell.delay(entries) self.assertEqual(result.get(), 3) observations = session.query(CellObservation).all() self.assertEqual(len(observations), 3) self.assertEqual(set([o.asu for o in observations]), set([-1, 8])) self.assertEqual(set([o.signal for o in observations]), set([0, -70])) self.assertEqual(set([o.ta for o in observations]), set([0, 32]))
def test_insert_measures_invalid_lac(self): session = self.db_master_session schema = ValidCellBaseSchema() time = util.utcnow() - timedelta(days=1) session.add( Cell(radio=RADIO_TYPE['gsm'], mcc=FRANCE_MCC, mnc=2, lac=3, cid=4, new_measures=2, total_measures=5)) session.add(Score(userid=1, key=SCORE_TYPE['new_cell'], value=7)) session.flush() measure = dict(created=encode_datetime(time), lat=PARIS_LAT, lon=PARIS_LON, time=encode_datetime(time), accuracy=0, altitude=0, altitude_accuracy=0, radio=RADIO_TYPE['gsm']) entries = [ { "mcc": FRANCE_MCC, "mnc": 2, "lac": constants.MAX_LAC_ALL + 1, "cid": constants.MAX_CID_ALL + 1, "psc": 5, "asu": 8 }, { "mcc": FRANCE_MCC, "mnc": 2, "lac": schema.fields['lac'].missing, "cid": schema.fields['cid'].missing, "psc": 5, "asu": 8 }, ] for e in entries: e.update(measure) result = insert_measures_cell.delay(entries, userid=1) self.assertEqual(result.get(), 2) measures = session.query(CellMeasure).all() self.assertEqual(len(measures), 2) self.assertEqual(set([m.lac for m in measures]), set([schema.fields['lac'].missing])) self.assertEqual(set([m.cid for m in measures]), set([schema.fields['cid'].missing])) # Nothing should change in the initially created Cell record cells = session.query(Cell).all() self.assertEqual(len(cells), 1) self.assertEqual(set([c.new_measures for c in cells]), set([2])) self.assertEqual(set([c.total_measures for c in cells]), set([5]))
def test_blacklist(self): now = util.utcnow() cell = CellFactory.build() observations = [dict(radio=int(cell.radio), mcc=cell.mcc, mnc=cell.mnc, lac=cell.lac, cid=cell.cid + i, psc=cell.psc, lat=cell.lat + i * 0.0000001, lon=cell.lon + i * 0.0000001) for i in range(1, 4)] black = CellBlacklist( radio=cell.radio, mcc=cell.mcc, mnc=cell.mnc, lac=cell.lac, cid=cell.cid + 1, time=now, count=1, ) self.session.add(black) self.session.flush() result = insert_measures_cell.delay(observations) self.assertEqual(result.get(), 2) self.assertEqual(self.data_queue.size(), 2) update_cell.delay().get() cells = self.session.query(Cell).all() self.assertEqual(len(cells), 2) self.check_statcounter(StatKey.cell, 2) self.check_statcounter(StatKey.unique_cell, 2)
def test_blacklist(self): now = util.utcnow() session = self.db_master_session observations = [ dict(mcc=FRANCE_MCC, mnc=2, lac=3, cid=i, psc=5, radio=RADIO_TYPE['gsm'], lat=PARIS_LAT + i * 0.0000001, lon=PARIS_LON + i * 0.0000001) for i in range(1, 4) ] black = CellBlacklist( mcc=FRANCE_MCC, mnc=2, lac=3, cid=1, radio=RADIO_TYPE['gsm'], time=now, count=1, ) session.add(black) session.flush() result = insert_measures_cell.delay(observations) self.assertEqual(result.get(), 2) cell_observations = session.query(CellObservation).all() self.assertEqual(len(cell_observations), 2) cells = session.query(Cell).all() self.assertEqual(len(cells), 2)
def test_insert_observations(self): session = self.session time = util.utcnow() - timedelta(days=1) mcc = FRANCE_MCC session.add(Cell(radio=Radio.gsm, mcc=mcc, mnc=2, lac=3, cid=4, psc=5, total_measures=5)) user = User(nickname=u'test') session.add(user) session.flush() obs = dict( created=time, lat=PARIS_LAT, lon=PARIS_LON, time=time, accuracy=0, altitude=0, altitude_accuracy=0, radio=int(Radio.gsm), ) entries = [ # Note that this first entry will be skipped as it does # not include (lac, cid) or (psc) {'mcc': mcc, 'mnc': 2, 'signal': -100}, {'mcc': mcc, 'mnc': 2, 'lac': 3, 'cid': 4, 'psc': 5, 'asu': 8}, {'mcc': mcc, 'mnc': 2, 'lac': 3, 'cid': 4, 'psc': 5, 'asu': 8}, {'mcc': mcc, 'mnc': 2, 'lac': 3, 'cid': 4, 'psc': 5, 'asu': 15}, {'mcc': mcc, 'mnc': 2, 'lac': 3, 'cid': 7, 'psc': 5}, ] for entry in entries: entry.update(obs) result = insert_measures_cell.delay(entries, userid=user.id) self.assertEqual(result.get(), 4) self.assertEqual(self.data_queue.size(), 4) update_cell.delay().get() cells = session.query(Cell).all() self.assertEqual(len(cells), 2) self._compare_sets([c.mcc for c in cells], [mcc]) self._compare_sets([c.mnc for c in cells], [2]) self._compare_sets([c.lac for c in cells], [3]) self._compare_sets([c.cid for c in cells], [4, 7]) self._compare_sets([c.psc for c in cells], [5]) self._compare_sets([c.total_measures for c in cells], [1, 8]) score_queue = self.celery_app.data_queues['update_score'] scores = score_queue.dequeue() self.assertEqual(len(scores), 1) score = scores[0] self.assertEqual(score['hashkey'].userid, user.id) self.assertEqual(score['hashkey'].key, ScoreKey.new_cell) self.assertEqual(score['value'], 1) self.check_statcounter(StatKey.cell, 4) self.check_statcounter(StatKey.unique_cell, 1)
def test_insert_observations(self): time = util.utcnow() - timedelta(days=1) cell = CellFactory(radio=Radio.gsm, total_measures=5) user = User(nickname=u'test') self.session.add(user) self.session.flush() obs = dict( radio=int(cell.radio), mcc=cell.mcc, mnc=cell.mnc, created=time, time=time, lat=cell.lat, lon=cell.lon, accuracy=0, altitude=0, altitude_accuracy=0, ) entries = [ # Note that this first entry will be skipped as it does # not include (lac, cid) or (psc) {'signal': -100}, {'lac': cell.lac, 'cid': cell.cid, 'psc': cell.psc, 'asu': 8}, {'lac': cell.lac, 'cid': cell.cid, 'psc': cell.psc, 'asu': 8}, {'lac': cell.lac, 'cid': cell.cid, 'psc': cell.psc, 'asu': 15}, {'lac': cell.lac, 'cid': cell.cid + 1, 'psc': cell.psc}, ] for entry in entries: entry.update(obs) result = insert_measures_cell.delay(entries, userid=user.id) self.assertEqual(result.get(), 4) self.assertEqual(self.data_queue.size(), 4) update_cell.delay().get() self.session.refresh(cell) cells = self.session.query(Cell).all() self.assertEqual(len(cells), 2) self._compare_sets([c.mcc for c in cells], [cell.mcc]) self._compare_sets([c.mnc for c in cells], [cell.mnc]) self._compare_sets([c.lac for c in cells], [cell.lac]) self._compare_sets([c.cid for c in cells], [cell.cid, cell.cid + 1]) self._compare_sets([c.psc for c in cells], [cell.psc]) self._compare_sets([c.total_measures for c in cells], [1, 8]) score_queue = self.celery_app.data_queues['update_score'] scores = score_queue.dequeue() self.assertEqual(len(scores), 1) score = scores[0] self.assertEqual(score['hashkey'].userid, user.id) self.assertEqual(score['hashkey'].key, ScoreKey.new_cell) self.assertEqual(score['value'], 1) self.check_statcounter(StatKey.cell, 4) self.check_statcounter(StatKey.unique_cell, 1)
def test_insert_observations_invalid_lac(self): session = self.session schema = ValidCellKeySchema() time = util.utcnow() - timedelta(days=1) today = util.utcnow().date() session.add(Cell(radio=Radio.gsm, mcc=FRANCE_MCC, mnc=2, lac=3, cid=4, new_measures=2, total_measures=5)) session.add(Score(key=ScoreKey.new_cell, userid=1, time=today, value=7)) session.flush() obs = dict( created=time, lat=PARIS_LAT, lon=PARIS_LON, time=time, accuracy=0, altitude=0, altitude_accuracy=0, radio=int(Radio.gsm)) entries = [ {"mcc": FRANCE_MCC, "mnc": 2, "lac": constants.MAX_LAC_ALL + 1, "cid": constants.MAX_CID_ALL + 1, "psc": 5, "asu": 8}, {"mcc": FRANCE_MCC, "mnc": 2, "lac": schema.fields['lac'].missing, "cid": schema.fields['cid'].missing, "psc": 5, "asu": 8}, ] for e in entries: e.update(obs) result = insert_measures_cell.delay(entries, userid=1) self.assertEqual(result.get(), 2) observations = session.query(CellObservation).all() self.assertEqual(len(observations), 2) self.assertEqual( set([o.lac for o in observations]), set([schema.fields['lac'].missing])) self.assertEqual( set([o.cid for o in observations]), set([schema.fields['cid'].missing])) # Nothing should change in the initially created Cell record cells = session.query(Cell).all() self.assertEqual(len(cells), 1) self.assertEqual(set([c.new_measures for c in cells]), set([2])) self.assertEqual(set([c.total_measures for c in cells]), set([5]))
def test_insert_measures_out_of_range(self): session = self.db_master_session time = util.utcnow() - timedelta(days=1) measure = dict(created=encode_datetime(time), lat=PARIS_LAT, lon=PARIS_LON, time=encode_datetime(time), accuracy=0, altitude=0, altitude_accuracy=0, radio=RADIO_TYPE['gsm'], mcc=FRANCE_MCC, mnc=2, lac=3, cid=4) entries = [ { "asu": 8, "signal": -70, "ta": 32 }, { "asu": -10, "signal": -300, "ta": -10 }, { "asu": 256, "signal": 16, "ta": 128 }, ] for e in entries: e.update(measure) result = insert_measures_cell.delay(entries) self.assertEqual(result.get(), 3) measures = session.query(CellMeasure).all() self.assertEqual(len(measures), 3) self.assertEqual(set([m.asu for m in measures]), set([-1, 8])) self.assertEqual(set([m.signal for m in measures]), set([0, -70])) self.assertEqual(set([m.ta for m in measures]), set([0, 32]))
def test_insert_observations_out_of_range(self): session = self.session time = util.utcnow() - timedelta(days=1) obs = dict(created=time, lat=PARIS_LAT, lon=PARIS_LON, time=time, accuracy=0, altitude=0, altitude_accuracy=0, radio=int(Radio.gsm), mcc=FRANCE_MCC, mnc=2, lac=3, cid=4) entries = [ { "asu": 8, "signal": -70, "ta": 32 }, { "asu": -10, "signal": -300, "ta": -10 }, { "asu": 256, "signal": 16, "ta": 128 }, ] for e in entries: e.update(obs) result = insert_measures_cell.delay(entries) self.assertEqual(result.get(), 3) observations = session.query(CellObservation).all() self.assertEqual(len(observations), 3) self.assertEqual(set([o.asu for o in observations]), set([-1, 8])) self.assertEqual(set([o.signal for o in observations]), set([0, -70])) self.assertEqual(set([o.ta for o in observations]), set([0, 32]))
def test_insert_observations_out_of_range(self): obs = dict( lat=PARIS_LAT, lon=PARIS_LON, radio=int(Radio.gsm), mcc=FRANCE_MCC, mnc=2, lac=3, cid=4) entries = [ {'asu': 8, 'signal': -70, 'ta': 32}, {'asu': -10, 'signal': -300, 'ta': -10}, {'asu': 256, 'signal': 16, 'ta': 128}, ] for entry in entries: entry.update(obs) result = insert_measures_cell.delay(entries) self.assertEqual(result.get(), 3) observations = self.data_queue.dequeue() self.assertEqual(len(observations), 3) self._compare_sets([o.asu for o in observations], [None, 8]) self._compare_sets([o.signal for o in observations], [None, -70]) self._compare_sets([o.ta for o in observations], [None, 32])
def test_blacklist(self): session = self.db_master_session measures = [dict(mcc=FRANCE_MCC, mnc=2, lac=3, cid=i, psc=5, radio=RADIO_TYPE['gsm'], lat=PARIS_LAT + i * 0.0000001, lon=PARIS_LON + i * 0.0000001) for i in range(1, 4)] black = CellBlacklist( mcc=FRANCE_MCC, mnc=2, lac=3, cid=1, radio=RADIO_TYPE['gsm'], ) session.add(black) session.flush() result = insert_measures_cell.delay(measures) self.assertEqual(result.get(), 2) measures = session.query(CellMeasure).all() self.assertEqual(len(measures), 2) cells = session.query(Cell).all() self.assertEqual(len(cells), 2)
def test_insert_observations_out_of_range(self): cell = CellFactory.build() obs = dict( lat=cell.lat, lon=cell.lon, radio=int(cell.radio), mcc=cell.mcc, mnc=cell.mnc, lac=cell.lac, cid=cell.cid) entries = [ {'asu': 8, 'signal': -70, 'ta': 32}, {'asu': -10, 'signal': -300, 'ta': -10}, {'asu': 256, 'signal': 16, 'ta': 128}, ] for entry in entries: entry.update(obs) result = insert_measures_cell.delay(entries) self.assertEqual(result.get(), 3) observations = self.data_queue.dequeue() self.assertEqual(len(observations), 3) self._compare_sets([o.asu for o in observations], [None, 8]) self._compare_sets([o.signal for o in observations], [None, -70]) self._compare_sets([o.ta for o in observations], [None, 32])
def test_insert_observations_invalid_lac(self): session = self.session time = util.utcnow() - timedelta(days=1) today = util.utcnow().date() session.add(Cell(radio=Radio.gsm, mcc=FRANCE_MCC, mnc=2, lac=3, cid=4, total_measures=5)) session.add(Score(key=ScoreKey.new_cell, userid=1, time=today, value=7)) session.flush() obs = dict( created=time, lat=PARIS_LAT, lon=PARIS_LON, time=time, accuracy=0, altitude=0, altitude_accuracy=0, radio=int(Radio.gsm)) entries = [ {'mcc': FRANCE_MCC, 'mnc': 2, 'lac': constants.MAX_LAC_ALL + 1, 'cid': constants.MAX_CID_ALL + 1, 'psc': 5, 'asu': 8}, {'mcc': FRANCE_MCC, 'mnc': 2, 'lac': None, 'cid': None, 'psc': 5, 'asu': 8}, ] for entry in entries: entry.update(obs) result = insert_measures_cell.delay(entries, userid=1) self.assertEqual(result.get(), 0) # The incomplete observations never make it into the queue self.assertEqual(self.data_queue.size(), 0) update_cell.delay().get() # Nothing should change in the initially created Cell record cells = session.query(Cell).all() self.assertEqual(len(cells), 1) self._compare_sets([c.total_measures for c in cells], [5])
def test_insert_measures_invalid_lac(self): session = self.db_master_session time = util.utcnow() - timedelta(days=1) session.add(Cell(radio=RADIO_TYPE['gsm'], mcc=FRANCE_MCC, mnc=2, lac=3, cid=4, new_measures=2, total_measures=5)) session.add(Score(userid=1, key=SCORE_TYPE['new_cell'], value=7)) session.flush() measure = dict( id=0, created=encode_datetime(time), lat=PARIS_LAT, lon=PARIS_LON, time=encode_datetime(time), accuracy=0, altitude=0, altitude_accuracy=0, radio=RADIO_TYPE['gsm']) entries = [ {"mcc": FRANCE_MCC, "mnc": 2, "lac": 3147483647, "cid": 2147483647, "psc": 5, "asu": 8}, {"mcc": FRANCE_MCC, "mnc": 2, "lac": -1, "cid": -1, "psc": 5, "asu": 8}, ] for e in entries: e.update(measure) result = insert_measures_cell.delay(entries, userid=1) self.assertEqual(result.get(), 2) measures = session.query(CellMeasure).all() self.assertEqual(len(measures), 2) self.assertEqual(set([m.lac for m in measures]), set([-1])) self.assertEqual(set([m.cid for m in measures]), set([-1])) # Nothing should change in the initially created Cell record cells = session.query(Cell).all() self.assertEqual(len(cells), 1) self.assertEqual(set([c.new_measures for c in cells]), set([2])) self.assertEqual(set([c.total_measures for c in cells]), set([5]))
def test_blacklist_temporary_and_permanent(self): # This test simulates a cell that moves once a month, for 2 years. # The first 2 * PERMANENT_BLACKLIST_THRESHOLD (12) moves should be # temporary, forgotten after a week; after that it should be # permanently blacklisted. now = util.utcnow() # Station moves between these 4 points, all in the USA: points = [ (40.0, -74.0), # NYC (37.0, -122.0), # SF (47.0, -122.0), # Seattle (25.0, -80.0), # Miami ] N = 4 * PERMANENT_BLACKLIST_THRESHOLD for month in range(0, N): days_ago = (N - (month + 1)) * 30 time = now - timedelta(days=days_ago) obs = dict(radio=int(Radio.gsm), mcc=310, mnc=150, lac=456, cid=123, time=time, lat=points[month % 4][0], lon=points[month % 4][1]) # insert_result is num-accepted-observations, override # utcnow to set creation date insert_result = insert_measures_cell.delay( [obs], utcnow=time) # update_result is (num-stations, num-moving-stations) update_result = update_cell.delay() # Assuming PERMANENT_BLACKLIST_THRESHOLD == 6: # # 0th insert will create the station # 1st insert will create first blacklist entry, delete station # 2nd insert will recreate the station at new position # 3rd insert will update blacklist, re-delete station # 4th insert will recreate the station at new position # 5th insert will update blacklist, re-delete station # 6th insert will recreate the station at new position # ... # 11th insert will make blacklisting permanent, re-delete station # 12th insert will not recreate station # 13th insert will not recreate station # ... # 23rd insert will not recreate station bl = self.session.query(CellBlacklist).all() if month == 0: self.assertEqual(len(bl), 0) else: self.assertEqual(len(bl), 1) # force the blacklist back in time to whenever the # observation was supposedly inserted. bl = bl[0] bl.time = time self.session.add(bl) self.session.commit() if month < N / 2: # We still haven't exceeded the threshold, so the # observation was admitted. self.assertEqual(insert_result.get(), 1) if month % 2 == 0: # The station was (re)created. self.assertEqual(update_result.get(), (1, 0)) # Rescan lacs to update entries self.assertEqual( scan_areas.delay().get(), 1) # One cell + one cell-LAC record should exist. self.assertEqual(self.session.query(Cell).count(), 1) self.assertEqual(self.session.query(CellArea).count(), 1) else: # The station existed and was seen moving, # thereby activating the blacklist and deleting the cell. self.assertEqual(update_result.get(), (1, 1)) # Rescan lacs to delete orphaned lac entry self.assertEqual( scan_areas.delay().get(), 1) self.assertEqual(bl.count, ((month + 1) / 2)) self.assertEqual( self.session.query(CellBlacklist).count(), 1) self.assertEqual(self.session.query(Cell).count(), 0) # Try adding one more observation 1 day later # to be sure it is dropped by the now-active blacklist. next_day = time + timedelta(days=1) obs['time'] = next_day self.assertEqual( 0, insert_measures_cell.delay([obs], utcnow=next_day).get()) else: # Blacklist has exceeded threshold, gone to permanent mode, # so no observation accepted, no stations seen. self.assertEqual(insert_result.get(), 0) self.assertEqual(update_result.get(), (0, 0))
def test_insert_observations_invalid_lac(self): session = self.session schema = ValidCellKeySchema() time = util.utcnow() - timedelta(days=1) today = util.utcnow().date() session.add( Cell(radio=Radio.gsm, mcc=FRANCE_MCC, mnc=2, lac=3, cid=4, new_measures=2, total_measures=5)) session.add(Score(key=ScoreKey.new_cell, userid=1, time=today, value=7)) session.flush() obs = dict(created=time, lat=PARIS_LAT, lon=PARIS_LON, time=time, accuracy=0, altitude=0, altitude_accuracy=0, radio=int(Radio.gsm)) entries = [ { "mcc": FRANCE_MCC, "mnc": 2, "lac": constants.MAX_LAC_ALL + 1, "cid": constants.MAX_CID_ALL + 1, "psc": 5, "asu": 8 }, { "mcc": FRANCE_MCC, "mnc": 2, "lac": schema.fields['lac'].missing, "cid": schema.fields['cid'].missing, "psc": 5, "asu": 8 }, ] for e in entries: e.update(obs) result = insert_measures_cell.delay(entries, userid=1) self.assertEqual(result.get(), 2) observations = session.query(CellObservation).all() self.assertEqual(len(observations), 2) self.assertEqual(set([o.lac for o in observations]), set([schema.fields['lac'].missing])) self.assertEqual(set([o.cid for o in observations]), set([schema.fields['cid'].missing])) # Nothing should change in the initially created Cell record cells = session.query(Cell).all() self.assertEqual(len(cells), 1) self.assertEqual(set([c.new_measures for c in cells]), set([2])) self.assertEqual(set([c.total_measures for c in cells]), set([5]))
def test_insert_measures(self): session = self.db_master_session time = util.utcnow() - timedelta(days=1) mcc = FRANCE_MCC session.add(Cell(radio=RADIO_TYPE['gsm'], mcc=mcc, mnc=2, lac=3, cid=4, psc=5, new_measures=2, total_measures=5)) session.add(Score(userid=1, key=SCORE_TYPE['new_cell'], value=7)) session.flush() measure = dict( id=0, created=encode_datetime(time), lat=PARIS_LAT, lon=PARIS_LON, time=encode_datetime(time), accuracy=0, altitude=0, altitude_accuracy=0, radio=RADIO_TYPE['gsm'], ) entries = [ # Note that this first entry will be skipped as it does # not include (lac, cid) or (psc) {"mcc": mcc, "mnc": 2, "signal": -100}, {"mcc": mcc, "mnc": 2, "lac": 3, "cid": 4, "psc": 5, "asu": 8}, {"mcc": mcc, "mnc": 2, "lac": 3, "cid": 4, "psc": 5, "asu": 8}, {"mcc": mcc, "mnc": 2, "lac": 3, "cid": 4, "psc": 5, "asu": 15}, {"mcc": mcc, "mnc": 2, "lac": 3, "cid": 7, "psc": 5}, ] for e in entries: e.update(measure) result = insert_measures_cell.delay(entries, userid=1) self.assertEqual(result.get(), 4) measures = session.query(CellMeasure).all() self.assertEqual(len(measures), 4) self.assertEqual(set([m.mcc for m in measures]), set([mcc])) self.assertEqual(set([m.mnc for m in measures]), set([2])) self.assertEqual(set([m.asu for m in measures]), set([-1, 8, 15])) self.assertEqual(set([m.psc for m in measures]), set([5])) self.assertEqual(set([m.signal for m in measures]), set([0])) cells = session.query(Cell).all() self.assertEqual(len(cells), 2) self.assertEqual(set([c.mcc for c in cells]), set([mcc])) self.assertEqual(set([c.mnc for c in cells]), set([2])) self.assertEqual(set([c.lac for c in cells]), set([3])) self.assertEqual(set([c.cid for c in cells]), set([4, 7])) self.assertEqual(set([c.psc for c in cells]), set([5])) self.assertEqual(set([c.new_measures for c in cells]), set([1, 5])) self.assertEqual(set([c.total_measures for c in cells]), set([1, 8])) scores = session.query(Score).all() self.assertEqual(len(scores), 1) self.assertEqual(scores[0].key, SCORE_TYPE['new_cell']) self.assertEqual(scores[0].value, 8) # test duplicate execution result = insert_measures_cell.delay(entries, userid=1) self.assertEqual(result.get(), 4) # TODO this task isn't idempotent yet measures = session.query(CellMeasure).all() self.assertEqual(len(measures), 8)
def test_blacklist_temporary_and_permanent(self): session = self.db_master_session # This test simulates a cell that moves once a month, for 2 years. # The first 2 * PERMANENT_BLACKLIST_THRESHOLD (12) moves should be # temporary, forgotten after a week; after that it should be # permanently blacklisted. now = util.utcnow() # Station moves between these 4 points, all in the USA: points = [ # NYC (40.0, -74.0), # SF (37.0, -122.0), # Seattle (47.0, -122.0), # Miami (25.0, -80.0), ] N = 4 * PERMANENT_BLACKLIST_THRESHOLD for month in range(0, N): days_ago = (N - (month + 1)) * 30 time = now - timedelta(days=days_ago) time_enc = encode_datetime(time) measure = dict(id=month, radio=RADIO_TYPE['gsm'], mcc=USA_MCC, mnc=ATT_MNC, lac=456, cid=123, time=time_enc, lat=points[month % 4][0], lon=points[month % 4][1]) # insert_result is num-accepted-measures, override # utcnow to set creation date insert_result = insert_measures_cell.delay( [measure], utcnow=time_enc) # update_result is (num-stations, num-moving-stations) update_result = location_update_cell.delay(min_new=1) # Assuming PERMANENT_BLACKLIST_THRESHOLD == 6: # # 0th insert will create the station # 1st insert will create first blacklist entry, delete station # 2nd insert will recreate the station at new position # 3rd insert will update blacklist, re-delete station # 4th insert will recreate the station at new position # 5th insert will update blacklist, re-delete station # 6th insert will recreate the station at new position # ... # 11th insert will make blacklisting permanent, re-delete station # 12th insert will not recreate station # 13th insert will not recreate station # ... # 23rd insert will not recreate station bl = session.query(CellBlacklist).all() if month == 0: self.assertEqual(len(bl), 0) else: self.assertEqual(len(bl), 1) # force the blacklist back in time to whenever the # measure was supposedly inserted. bl = bl[0] bl.time = time session.add(bl) session.commit() if month < N / 2: # We still haven't exceeded the threshold, so the # measurement was admitted. self.assertEqual(insert_result.get(), 1) self.assertEqual(session.query(CellMeasure).count(), month + 1) if month % 2 == 0: # The station was (re)created. self.assertEqual(update_result.get(), (1, 0)) # One cell + one cell-LAC record should exist. self.assertEqual(session.query(Cell).count(), 2) else: # The station existed and was seen moving, # thereby activating the blacklist and deleting the cell. self.assertEqual(update_result.get(), (1, 1)) self.assertEqual(bl.count, ((month + 1) / 2)) self.assertEqual(session.query(CellBlacklist).count(), 1) self.assertEqual(session.query(Cell).count(), 0) # Try adding one more measurement 1 day later # to be sure it is dropped by the now-active blacklist. next_day = encode_datetime(time + timedelta(days=1)) measure['time'] = next_day self.assertEqual( 0, insert_measures_cell.delay([measure], utcnow=next_day).get()) else: # Blacklist has exceeded threshold, gone to "permanent" mode, # so no measures accepted, no stations seen. self.assertEqual(insert_result.get(), 0) self.assertEqual(update_result.get(), 0)
def test_insert_observations(self): session = self.session time = util.utcnow() - timedelta(days=1) today = util.utcnow().date() mcc = FRANCE_MCC session.add( Cell(radio=Radio.gsm, mcc=mcc, mnc=2, lac=3, cid=4, psc=5, new_measures=2, total_measures=5)) session.add(Score(key=ScoreKey.new_cell, userid=1, time=today, value=7)) session.flush() obs = dict( created=time, lat=PARIS_LAT, lon=PARIS_LON, time=time, accuracy=0, altitude=0, altitude_accuracy=0, radio=int(Radio.gsm), ) entries = [ # Note that this first entry will be skipped as it does # not include (lac, cid) or (psc) { "mcc": mcc, "mnc": 2, "signal": -100 }, { "mcc": mcc, "mnc": 2, "lac": 3, "cid": 4, "psc": 5, "asu": 8 }, { "mcc": mcc, "mnc": 2, "lac": 3, "cid": 4, "psc": 5, "asu": 8 }, { "mcc": mcc, "mnc": 2, "lac": 3, "cid": 4, "psc": 5, "asu": 15 }, { "mcc": mcc, "mnc": 2, "lac": 3, "cid": 7, "psc": 5 }, ] for e in entries: e.update(obs) result = insert_measures_cell.delay(entries, userid=1) self.assertEqual(result.get(), 4) observations = session.query(CellObservation).all() self.assertEqual(len(observations), 4) self.assertEqual(set([o.mcc for o in observations]), set([mcc])) self.assertEqual(set([o.mnc for o in observations]), set([2])) self.assertEqual(set([o.asu for o in observations]), set([-1, 8, 15])) self.assertEqual(set([o.psc for o in observations]), set([5])) self.assertEqual(set([o.signal for o in observations]), set([0])) cells = session.query(Cell).all() self.assertEqual(len(cells), 2) self.assertEqual(set([c.mcc for c in cells]), set([mcc])) self.assertEqual(set([c.mnc for c in cells]), set([2])) self.assertEqual(set([c.lac for c in cells]), set([3])) self.assertEqual(set([c.cid for c in cells]), set([4, 7])) self.assertEqual(set([c.psc for c in cells]), set([5])) self.assertEqual(set([c.new_measures for c in cells]), set([1, 5])) self.assertEqual(set([c.total_measures for c in cells]), set([1, 8])) scores = session.query(Score).all() self.assertEqual(len(scores), 1) self.assertEqual(scores[0].key, ScoreKey.new_cell) self.assertEqual(scores[0].value, 8) # test duplicate execution result = insert_measures_cell.delay(entries, userid=1) self.assertEqual(result.get(), 4) # TODO this task isn't idempotent yet observations = session.query(CellObservation).all() self.assertEqual(len(observations), 8)
def test_blacklist_temporary_and_permanent(self): session = self.db_master_session # This test simulates a cell that moves once a month, for 2 years. # The first 2 * PERMANENT_BLACKLIST_THRESHOLD (12) moves should be # temporary, forgotten after a week; after that it should be # permanently blacklisted. now = util.utcnow() # Station moves between these 4 points, all in the USA: points = [ # NYC (40.0, -74.0), # SF (37.0, -122.0), # Seattle (47.0, -122.0), # Miami (25.0, -80.0), ] N = 4 * PERMANENT_BLACKLIST_THRESHOLD for month in range(0, N): days_ago = (N - (month + 1)) * 30 time = now - timedelta(days=days_ago) time_enc = encode_datetime(time) measure = dict(radio=RADIO_TYPE['gsm'], mcc=USA_MCC, mnc=ATT_MNC, lac=456, cid=123, time=time_enc, lat=points[month % 4][0], lon=points[month % 4][1]) # insert_result is num-accepted-measures, override # utcnow to set creation date insert_result = insert_measures_cell.delay([measure], utcnow=time_enc) # update_result is (num-stations, num-moving-stations) update_result = location_update_cell.delay(min_new=1) # Assuming PERMANENT_BLACKLIST_THRESHOLD == 6: # # 0th insert will create the station # 1st insert will create first blacklist entry, delete station # 2nd insert will recreate the station at new position # 3rd insert will update blacklist, re-delete station # 4th insert will recreate the station at new position # 5th insert will update blacklist, re-delete station # 6th insert will recreate the station at new position # ... # 11th insert will make blacklisting permanent, re-delete station # 12th insert will not recreate station # 13th insert will not recreate station # ... # 23rd insert will not recreate station bl = session.query(CellBlacklist).all() if month == 0: self.assertEqual(len(bl), 0) else: self.assertEqual(len(bl), 1) # force the blacklist back in time to whenever the # measure was supposedly inserted. bl = bl[0] bl.time = time session.add(bl) session.commit() if month < N / 2: # We still haven't exceeded the threshold, so the # measurement was admitted. self.assertEqual(insert_result.get(), 1) self.assertEqual(session.query(CellMeasure).count(), month + 1) if month % 2 == 0: # The station was (re)created. self.assertEqual(update_result.get(), (1, 0)) # Rescan lacs to update entries self.assertEqual(scan_lacs.delay().get(), 1) # One cell + one cell-LAC record should exist. self.assertEqual(session.query(Cell).count(), 2) else: # The station existed and was seen moving, # thereby activating the blacklist and deleting the cell. self.assertEqual(update_result.get(), (1, 1)) # Rescan lacs to delete orphaned lac entry self.assertEqual(scan_lacs.delay().get(), 1) self.assertEqual(bl.count, ((month + 1) / 2)) self.assertEqual(session.query(CellBlacklist).count(), 1) self.assertEqual(session.query(Cell).count(), 0) # Try adding one more measurement 1 day later # to be sure it is dropped by the now-active blacklist. next_day = encode_datetime(time + timedelta(days=1)) measure['time'] = next_day self.assertEqual( 0, insert_measures_cell.delay([measure], utcnow=next_day).get()) else: # Blacklist has exceeded threshold, gone to "permanent" mode, # so no measures accepted, no stations seen. self.assertEqual(insert_result.get(), 0) self.assertEqual(update_result.get(), 0)
def test_insert_measures(self): session = self.db_master_session time = util.utcnow() - timedelta(days=1) mcc = FRANCE_MCC session.add( Cell(radio=RADIO_TYPE['gsm'], mcc=mcc, mnc=2, lac=3, cid=4, psc=5, new_measures=2, total_measures=5)) session.add(Score(userid=1, key=SCORE_TYPE['new_cell'], value=7)) session.flush() measure = dict( created=encode_datetime(time), lat=PARIS_LAT, lon=PARIS_LON, time=encode_datetime(time), accuracy=0, altitude=0, altitude_accuracy=0, radio=RADIO_TYPE['gsm'], ) entries = [ # Note that this first entry will be skipped as it does # not include (lac, cid) or (psc) { "mcc": mcc, "mnc": 2, "signal": -100 }, { "mcc": mcc, "mnc": 2, "lac": 3, "cid": 4, "psc": 5, "asu": 8 }, { "mcc": mcc, "mnc": 2, "lac": 3, "cid": 4, "psc": 5, "asu": 8 }, { "mcc": mcc, "mnc": 2, "lac": 3, "cid": 4, "psc": 5, "asu": 15 }, { "mcc": mcc, "mnc": 2, "lac": 3, "cid": 7, "psc": 5 }, ] for e in entries: e.update(measure) result = insert_measures_cell.delay(entries, userid=1) self.assertEqual(result.get(), 4) measures = session.query(CellMeasure).all() self.assertEqual(len(measures), 4) self.assertEqual(set([m.mcc for m in measures]), set([mcc])) self.assertEqual(set([m.mnc for m in measures]), set([2])) self.assertEqual(set([m.asu for m in measures]), set([-1, 8, 15])) self.assertEqual(set([m.psc for m in measures]), set([5])) self.assertEqual(set([m.signal for m in measures]), set([0])) cells = session.query(Cell).all() self.assertEqual(len(cells), 2) self.assertEqual(set([c.mcc for c in cells]), set([mcc])) self.assertEqual(set([c.mnc for c in cells]), set([2])) self.assertEqual(set([c.lac for c in cells]), set([3])) self.assertEqual(set([c.cid for c in cells]), set([4, 7])) self.assertEqual(set([c.psc for c in cells]), set([5])) self.assertEqual(set([c.new_measures for c in cells]), set([1, 5])) self.assertEqual(set([c.total_measures for c in cells]), set([1, 8])) scores = session.query(Score).all() self.assertEqual(len(scores), 1) self.assertEqual(scores[0].key, SCORE_TYPE['new_cell']) self.assertEqual(scores[0].value, 8) # test duplicate execution result = insert_measures_cell.delay(entries, userid=1) self.assertEqual(result.get(), 4) # TODO this task isn't idempotent yet measures = session.query(CellMeasure).all() self.assertEqual(len(measures), 8)
def test_insert_observations(self): session = self.session time = util.utcnow() - timedelta(days=1) today = util.utcnow().date() mcc = FRANCE_MCC session.add(Cell(radio=Radio.gsm, mcc=mcc, mnc=2, lac=3, cid=4, psc=5, new_measures=2, total_measures=5)) session.add(Score(key=ScoreKey.new_cell, userid=1, time=today, value=7)) session.flush() obs = dict( created=time, lat=PARIS_LAT, lon=PARIS_LON, time=time, accuracy=0, altitude=0, altitude_accuracy=0, radio=int(Radio.gsm), ) entries = [ # Note that this first entry will be skipped as it does # not include (lac, cid) or (psc) {"mcc": mcc, "mnc": 2, "signal": -100}, {"mcc": mcc, "mnc": 2, "lac": 3, "cid": 4, "psc": 5, "asu": 8}, {"mcc": mcc, "mnc": 2, "lac": 3, "cid": 4, "psc": 5, "asu": 8}, {"mcc": mcc, "mnc": 2, "lac": 3, "cid": 4, "psc": 5, "asu": 15}, {"mcc": mcc, "mnc": 2, "lac": 3, "cid": 7, "psc": 5}, ] for e in entries: e.update(obs) result = insert_measures_cell.delay(entries, userid=1) self.assertEqual(result.get(), 4) observations = session.query(CellObservation).all() self.assertEqual(len(observations), 4) self.assertEqual(set([o.mcc for o in observations]), set([mcc])) self.assertEqual(set([o.mnc for o in observations]), set([2])) self.assertEqual(set([o.asu for o in observations]), set([-1, 8, 15])) self.assertEqual(set([o.psc for o in observations]), set([5])) self.assertEqual(set([o.signal for o in observations]), set([0])) cells = session.query(Cell).all() self.assertEqual(len(cells), 2) self.assertEqual(set([c.mcc for c in cells]), set([mcc])) self.assertEqual(set([c.mnc for c in cells]), set([2])) self.assertEqual(set([c.lac for c in cells]), set([3])) self.assertEqual(set([c.cid for c in cells]), set([4, 7])) self.assertEqual(set([c.psc for c in cells]), set([5])) self.assertEqual(set([c.new_measures for c in cells]), set([1, 5])) self.assertEqual(set([c.total_measures for c in cells]), set([1, 8])) scores = session.query(Score).all() self.assertEqual(len(scores), 1) self.assertEqual(scores[0].key, ScoreKey.new_cell) self.assertEqual(scores[0].value, 8) # test duplicate execution result = insert_measures_cell.delay(entries, userid=1) self.assertEqual(result.get(), 4) # TODO this task isn't idempotent yet observations = session.query(CellObservation).all() self.assertEqual(len(observations), 8)