def test_insert_measures_overflow(self): session = self.db_master_session key = "001234567890" measures = [dict(id=0, key=key, lat=1.0 + i * 0.0000001, lon=2.0 + i * 0.0000001) for i in range(3)] result = insert_measures_wifi.delay(measures) self.assertEqual(result.get(), 3) result = insert_measures_wifi.delay(measures, max_measures_per_wifi=3) self.assertEqual(result.get(), 0) result = insert_measures_wifi.delay(measures, max_measures_per_wifi=10) self.assertEqual(result.get(), 3) result = insert_measures_wifi.delay(measures, max_measures_per_wifi=3) self.assertEqual(result.get(), 0) measures = session.query(WifiMeasure).all() self.assertEqual(len(measures), 6) wifis = session.query(Wifi).all() self.assertEqual(len(wifis), 1) self.assertEqual(wifis[0].total_measures, 6)
def test_database_error(self): session = self.session stmt = text('drop table wifi;') session.execute(stmt) entries = [ {'lat': 1.0, 'lon': 2.0, 'key': 'ab:12:34:56:78:90', 'channel': 11}, {'lat': 1.0, 'lon': 2.0, 'key': 'ab:12:34:56:78:90', 'channel': 3}, {'lat': 1.0, 'lon': 2.0, 'key': 'ab:12:34:56:78:90', 'channel': 3}, {'lat': 1.0, 'lon': 2.0, 'key': 'cd:12:34:56:78:90', 'channel': 3}, ] try: insert_measures_wifi.delay(entries) except ProgrammingError: pass except Exception as exc: self.fail('Unexpected exception caught: %s' % repr(exc)) self.check_raven([('ProgrammingError', 1)]) self.check_statcounter(StatKey.wifi, 0) self.check_statcounter(StatKey.unique_wifi, 0)
def test_database_error(self): session = self.db_master_session stmt = text("drop table wifi;") session.execute(stmt) entries = [ {"lat": 1.0, "lon": 2.0, "key": "ab:12:34:56:78:90", "channel": 11}, {"lat": 1.0, "lon": 2.0, "key": "ab:12:34:56:78:90", "channel": 3}, {"lat": 1.0, "lon": 2.0, "key": "ab:12:34:56:78:90", "channel": 3}, {"lat": 1.0, "lon": 2.0, "key": "cd:12:34:56:78:90", "channel": 3}, ] try: insert_measures_wifi.delay(entries) except ProgrammingError: pass except Exception as exc: self.fail("Unexpected exception caught: %s" % repr(exc)) find_msg = self.find_heka_messages messages = find_msg('sentry', RAVEN_ERROR, field_name='msg') self.assertEquals(len(messages), 1) payload = messages[0].payload # duplicate raven.base.RavenClient.decode data = json.loads(zlib.decompress(base64.b64decode(payload))) sentry_exc = data['sentry.interfaces.Exception'] self.assertEqual(sentry_exc['module'], ProgrammingError.__module__) self.assertEqual(sentry_exc['type'], 'ProgrammingError')
def test_insert_measures_overflow(self): session = self.db_master_session key = "001234567890" measures = [ dict(key=key, lat=1.0 + i * 0.0000001, lon=2.0 + i * 0.0000001) for i in range(3) ] result = insert_measures_wifi.delay(measures) self.assertEqual(result.get(), 3) result = insert_measures_wifi.delay(measures, max_measures_per_wifi=3) self.assertEqual(result.get(), 0) result = insert_measures_wifi.delay(measures, max_measures_per_wifi=10) self.assertEqual(result.get(), 3) result = insert_measures_wifi.delay(measures, max_measures_per_wifi=3) self.assertEqual(result.get(), 0) measures = session.query(WifiMeasure).all() self.assertEqual(len(measures), 6) wifis = session.query(Wifi).all() self.assertEqual(len(wifis), 1) self.assertEqual(wifis[0].total_measures, 6)
def test_database_error(self): session = self.session stmt = text("drop table wifi;") session.execute(stmt) entries = [ {"lat": 1.0, "lon": 2.0, "key": "ab:12:34:56:78:90", "channel": 11}, {"lat": 1.0, "lon": 2.0, "key": "ab:12:34:56:78:90", "channel": 3}, {"lat": 1.0, "lon": 2.0, "key": "ab:12:34:56:78:90", "channel": 3}, {"lat": 1.0, "lon": 2.0, "key": "cd:12:34:56:78:90", "channel": 3}, ] try: insert_measures_wifi.delay(entries) except ProgrammingError: pass except Exception as exc: self.fail("Unexpected exception caught: %s" % repr(exc)) self.check_raven([('ProgrammingError', 4)])
def test_insert_observations(self): session = self.session time = util.utcnow() - timedelta(days=1) today = util.utcnow().date() session.add(Wifi(key="ab1234567890", new_measures=0, total_measures=0)) session.add(Score(key=ScoreKey.new_wifi, userid=1, time=today, value=7)) session.flush() obs = dict( created=time, lat=1.0, lon=2.0, time=time, accuracy=0, altitude=0, altitude_accuracy=0, radio=-1, heading=52.9, speed=158.5, ) entries = [ {"key": "ab1234567890", "channel": 11, "signal": -80}, {"key": "ab1234567890", "channel": 3, "signal": -90}, {"key": "ab1234567890", "channel": 3, "signal": -80}, {"key": "cd3456789012", "channel": 3, "signal": -90}, ] for e in entries: e.update(obs) result = insert_measures_wifi.delay(entries, userid=1) self.assertEqual(result.get(), 4) observations = session.query(WifiObservation).all() self.assertEqual(len(observations), 4) self.assertEqual(set([o.key for o in observations]), set(["ab1234567890", "cd3456789012"])) self.assertEqual(set([o.channel for o in observations]), set([3, 11])) self.assertEqual(set([o.signal for o in observations]), set([-80, -90])) self.assertEqual(set([o.heading or o in observations]), set([52.9])) self.assertEqual(set([o.speed or o in observations]), set([158.5])) wifis = session.query(Wifi).all() self.assertEqual(len(wifis), 2) self.assertEqual(set([w.key for w in wifis]), set(["ab1234567890", "cd3456789012"])) self.assertEqual(set([w.new_measures for w in wifis]), set([1, 3])) self.assertEqual(set([w.total_measures for w in wifis]), set([1, 3])) scores = session.query(Score).all() self.assertEqual(len(scores), 1) self.assertEqual(scores[0].key, ScoreKey.new_wifi) self.assertEqual(scores[0].value, 8) # test duplicate execution result = insert_measures_wifi.delay(entries, userid=1) self.assertEqual(result.get(), 4) # TODO this task isn't idempotent yet observations = session.query(WifiObservation).all() self.assertEqual(len(observations), 8)
def test_insert_measures(self): session = self.db_master_session time = util.utcnow() - timedelta(days=1) session.add(Wifi(key="ab1234567890")) session.add(Score(userid=1, key=SCORE_TYPE['new_wifi'], value=7)) session.flush() measure = dict( id=0, created=encode_datetime(time), lat=1.0, lon=2.0, time=encode_datetime(time), accuracy=0, altitude=0, altitude_accuracy=0, radio=-1, heading=52.9, speed=158.5, ) entries = [ {"key": "ab1234567890", "channel": 11, "signal": -80}, {"key": "ab1234567890", "channel": 3, "signal": -90}, {"key": "ab1234567890", "channel": 3, "signal": -80}, {"key": "cd3456789012", "channel": 3, "signal": -90}, ] for e in entries: e.update(measure) result = insert_measures_wifi.delay(entries, userid=1) self.assertEqual(result.get(), 4) measures = session.query(WifiMeasure).all() self.assertEqual(len(measures), 4) self.assertEqual(set([m.key for m in measures]), set(["ab1234567890", "cd3456789012"])) self.assertEqual(set([m.channel for m in measures]), set([3, 11])) self.assertEqual(set([m.signal for m in measures]), set([-80, -90])) self.assertEqual(set([m.heading or m in measures]), set([52.9])) self.assertEqual(set([m.speed or m in measures]), set([158.5])) wifis = session.query(Wifi).all() self.assertEqual(len(wifis), 2) self.assertEqual(set([w.key for w in wifis]), set(["ab1234567890", "cd3456789012"])) self.assertEqual(set([w.new_measures for w in wifis]), set([1, 3])) self.assertEqual(set([w.total_measures for w in wifis]), set([1, 3])) scores = session.query(Score).all() self.assertEqual(len(scores), 1) self.assertEqual(scores[0].key, SCORE_TYPE['new_wifi']) self.assertEqual(scores[0].value, 8) # test duplicate execution result = insert_measures_wifi.delay(entries, userid=1) self.assertEqual(result.get(), 4) # TODO this task isn't idempotent yet measures = session.query(WifiMeasure).all() self.assertEqual(len(measures), 8)
def test_database_error(self): session = self.db_master_session stmt = text("drop table wifi;") session.execute(stmt) entries = [ { "lat": 1.0, "lon": 2.0, "key": "ab:12:34:56:78:90", "channel": 11 }, { "lat": 1.0, "lon": 2.0, "key": "ab:12:34:56:78:90", "channel": 3 }, { "lat": 1.0, "lon": 2.0, "key": "ab:12:34:56:78:90", "channel": 3 }, { "lat": 1.0, "lon": 2.0, "key": "cd:12:34:56:78:90", "channel": 3 }, ] try: insert_measures_wifi.delay(entries) except ProgrammingError: pass except Exception as exc: self.fail("Unexpected exception caught: %s" % repr(exc)) find_msg = self.find_heka_messages messages = find_msg('sentry', RAVEN_ERROR, field_name='msg') self.assertEquals(len(messages), 1) payload = messages[0].payload # duplicate raven.base.RavenClient.decode data = json.loads(zlib.decompress(base64.b64decode(payload))) sentry_exc = data['sentry.interfaces.Exception'] self.assertEqual(sentry_exc['module'], ProgrammingError.__module__) self.assertEqual(sentry_exc['type'], 'ProgrammingError')
def test_blacklist(self): utcnow = util.utcnow() bad_wifi = WifiFactory.build() good_wifi = WifiFactory.build() black = WifiBlacklist(time=utcnow, count=1, key=bad_wifi.key) self.session.add(black) self.session.flush() obs = dict(lat=good_wifi.lat, lon=good_wifi.lon) entries = [ {'key': good_wifi.key}, {'key': good_wifi.key}, {'key': bad_wifi.key}, ] for entry in entries: entry.update(obs) result = insert_measures_wifi.delay(entries) self.assertEqual(result.get(), 2) self.assertEqual(self.data_queue.size(), 2) update_wifi.delay().get() wifis = self.session.query(Wifi).all() self.assertEqual(len(wifis), 1) self._compare_sets([w.key for w in wifis], [good_wifi.key]) self.check_statcounter(StatKey.wifi, 2) self.check_statcounter(StatKey.unique_wifi, 1)
def test_blacklist_time_used_as_creation_time(self): now = util.utcnow() last_week = now - TEMPORARY_BLACKLIST_DURATION - timedelta(days=1) session = self.session wifi_key = "ab1234567890" session.add(WifiBlacklist(time=last_week, count=1, key=wifi_key)) session.flush() # add a new entry for the previously blacklisted wifi obs = dict(lat=PARIS_LAT, lon=PARIS_LON, key=wifi_key) insert_measures_wifi.delay([obs]).get() # the wifi was inserted again wifis = session.query(Wifi).all() self.assertEqual(len(wifis), 1) # and the creation date was set to the date of the blacklist entry self.assertEqual(wifis[0].created, last_week)
def test_blacklist_time_used_as_creation_time(self): now = util.utcnow() last_week = now - TEMPORARY_BLACKLIST_DURATION - timedelta(days=1) session = self.db_master_session wifi_key = "ab1234567890" session.add(WifiBlacklist(time=last_week, count=1, key=wifi_key)) session.flush() # add a new entry for the previously blacklisted wifi obs = dict(lat=PARIS_LAT, lon=PARIS_LON, key=wifi_key) insert_measures_wifi.delay([obs]).get() # the wifi was inserted again wifis = session.query(Wifi).all() self.assertEqual(len(wifis), 1) # and the creation date was set to the date of the blacklist entry self.assertEqual(wifis[0].created, last_week)
def test_database_error(self): session = self.session stmt = text("drop table wifi;") session.execute(stmt) entries = [ { "lat": 1.0, "lon": 2.0, "key": "ab:12:34:56:78:90", "channel": 11 }, { "lat": 1.0, "lon": 2.0, "key": "ab:12:34:56:78:90", "channel": 3 }, { "lat": 1.0, "lon": 2.0, "key": "ab:12:34:56:78:90", "channel": 3 }, { "lat": 1.0, "lon": 2.0, "key": "cd:12:34:56:78:90", "channel": 3 }, ] try: insert_measures_wifi.delay(entries) except ProgrammingError: pass except Exception as exc: self.fail("Unexpected exception caught: %s" % repr(exc)) self.check_raven([('ProgrammingError', 4)])
def test_blacklist_time_used_as_creation_time(self): now = util.utcnow() last_week = now - TEMPORARY_BLACKLIST_DURATION - timedelta(days=1) wifi = WifiFactory.build() self.session.add(WifiBlacklist(time=last_week, count=1, key=wifi.key)) self.session.flush() # add a new entry for the previously blacklisted wifi obs = dict(lat=wifi.lat, lon=wifi.lon, key=wifi.key) insert_measures_wifi.delay([obs]).get() self.assertEqual(self.data_queue.size(), 1) update_wifi.delay().get() # the wifi was inserted again wifis = self.session.query(Wifi).all() self.assertEqual(len(wifis), 1) # and the creation date was set to the date of the blacklist entry self.assertEqual(wifis[0].created, last_week) self.check_statcounter(StatKey.unique_wifi, 0)
def test_database_error(self): self.session.execute(text('drop table wifi;')) wifi = WifiFactory.build() wifi2 = WifiFactory.build() entries = [ {'lat': wifi.lat, 'lon': wifi.lon, 'key': wifi.key, 'channel': 7}, {'lat': wifi.lat, 'lon': wifi.lon, 'key': wifi.key, 'channel': 3}, {'lat': wifi.lat, 'lon': wifi.lon, 'key': wifi.key, 'channel': 3}, {'lat': wifi2.lat, 'lon': wifi2.lon, 'key': wifi2.key}, ] try: insert_measures_wifi.delay(entries) except ProgrammingError: pass except Exception as exc: self.fail('Unexpected exception caught: %s' % repr(exc)) self.check_raven([('ProgrammingError', 1)]) self.check_statcounter(StatKey.wifi, 0) self.check_statcounter(StatKey.unique_wifi, 0)
def test_insert_observations(self): session = self.session time = util.utcnow() - timedelta(days=1) wifi = WifiFactory(total_measures=0) wifi2 = WifiFactory.build(total_measures=0) user = User(nickname=u'test') session.add(user) session.flush() obs = dict( created=time, time=time, lat=wifi.lat, lon=wifi.lon, accuracy=None, altitude=None, altitude_accuracy=None, heading=52.9, speed=158.5, ) entries = [ {'key': wifi.key, 'channel': 11, 'signal': -80}, {'key': wifi.key, 'channel': 3, 'signal': -90}, {'key': wifi.key, 'channel': 3, 'signal': -80}, {'key': wifi2.key, 'channel': 3, 'signal': -90}, ] for entry in entries: entry.update(obs) result = insert_measures_wifi.delay(entries, userid=user.id) self.assertEqual(result.get(), 4) self.assertEqual(self.data_queue.size(), 4) update_wifi.delay().get() self.session.refresh(wifi) wifis = session.query(Wifi).all() self.assertEqual(len(wifis), 2) self._compare_sets([w.key for w in wifis], [wifi.key, wifi2.key]) self._compare_sets([w.total_measures for w in wifis], [1, 3]) score_queue = self.celery_app.data_queues['update_score'] scores = score_queue.dequeue() self.assertEqual(len(scores), 1) score = scores[0] self.assertEqual(score['hashkey'].userid, user.id) self.assertEqual(score['hashkey'].key, ScoreKey.new_wifi) self.assertEqual(score['value'], 1) self.check_statcounter(StatKey.wifi, 4) self.check_statcounter(StatKey.unique_wifi, 1)
def test_blacklist(self): session = self.db_master_session bad_key = "ab1234567890" good_key = "cd1234567890" black = WifiBlacklist(key=bad_key) session.add(black) session.flush() measure = dict(lat=1, lon=2) entries = [{"key": good_key}, {"key": good_key}, {"key": bad_key}] for e in entries: e.update(measure) result = insert_measures_wifi.delay(entries) self.assertEqual(result.get(), 2) measures = session.query(WifiMeasure).all() self.assertEqual(len(measures), 2) self.assertEqual(set([m.key for m in measures]), set([good_key])) wifis = session.query(Wifi).all() self.assertEqual(len(wifis), 1) self.assertEqual(set([w.key for w in wifis]), set([good_key]))
def test_blacklist(self): utcnow = util.utcnow() session = self.db_master_session bad_key = "ab1234567890" good_key = "cd1234567890" black = WifiBlacklist(time=utcnow, count=1, key=bad_key) session.add(black) session.flush() obs = dict(lat=1, lon=2) entries = [{"key": good_key}, {"key": good_key}, {"key": bad_key}] for e in entries: e.update(obs) result = insert_measures_wifi.delay(entries) self.assertEqual(result.get(), 2) observations = session.query(WifiObservation).all() self.assertEqual(len(observations), 2) self.assertEqual(set([o.key for o in observations]), set([good_key])) wifis = session.query(Wifi).all() self.assertEqual(len(wifis), 1) self.assertEqual(set([w.key for w in wifis]), set([good_key]))
def test_blacklist(self): session = self.db_master_session bad_key = "ab1234567890" good_key = "cd1234567890" black = WifiBlacklist(key=bad_key) session.add(black) session.flush() measure = dict(id=0, lat=1, lon=2) entries = [{"key": good_key}, {"key": good_key}, {"key": bad_key}] for e in entries: e.update(measure) result = insert_measures_wifi.delay(entries) self.assertEqual(result.get(), 2) measures = session.query(WifiMeasure).all() self.assertEqual(len(measures), 2) self.assertEqual( set([m.key for m in measures]), set([good_key])) wifis = session.query(Wifi).all() self.assertEqual(len(wifis), 1) self.assertEqual(set([w.key for w in wifis]), set([good_key]))
def test_blacklist(self): utcnow = util.utcnow() session = self.session bad_key = "ab1234567890" good_key = "cd1234567890" black = WifiBlacklist(time=utcnow, count=1, key=bad_key) session.add(black) session.flush() obs = dict(lat=1, lon=2) entries = [{"key": good_key}, {"key": good_key}, {"key": bad_key}] for e in entries: e.update(obs) result = insert_measures_wifi.delay(entries) self.assertEqual(result.get(), 2) observations = session.query(WifiObservation).all() self.assertEqual(len(observations), 2) self.assertEqual( set([o.key for o in observations]), set([good_key])) wifis = session.query(Wifi).all() self.assertEqual(len(wifis), 1) self.assertEqual(set([w.key for w in wifis]), set([good_key]))
def test_blacklist_temporary_and_permanent(self): session = self.db_master_session # This test simulates a wifi that moves once a month, for 2 years. # The first 2 * PERMANENT_BLACKLIST_THRESHOLD (12) moves should be # temporary, forgotten after a week; after that it should be # permanently blacklisted. now = util.utcnow() # Station moves between these 4 points, all in the USA: points = [ # NYC (40.0, -74.0), # SF (37.0, -122.0), # Seattle (47.0, -122.0), # Miami (25.0, -80.0), ] N = 4 * PERMANENT_BLACKLIST_THRESHOLD for month in range(0, N): days_ago = (N - (month + 1)) * 30 time = now - timedelta(days=days_ago) time_enc = encode_datetime(time) measure = dict(id=month, key="ab1234567890", time=time_enc, lat=points[month % 4][0], lon=points[month % 4][1]) # insert_result is num-accepted-measures, override # utcnow to set creation date insert_result = insert_measures_wifi.delay( [measure], utcnow=time_enc) # update_result is (num-stations, num-moving-stations) update_result = location_update_wifi.delay(min_new=1) # Assuming PERMANENT_BLACKLIST_THRESHOLD == 6: # # 0th insert will create the station # 1st insert will create first blacklist entry, delete station # 2nd insert will recreate the station at new position # 3rd insert will update blacklist, re-delete station # 4th insert will recreate the station at new position # 5th insert will update blacklist, re-delete station # 6th insert will recreate the station at new position # ... # 11th insert will make blacklisting permanent, re-delete station # 12th insert will not recreate station # 13th insert will not recreate station # ... # 23rd insert will not recreate station bl = session.query(WifiBlacklist).all() if month == 0: self.assertEqual(len(bl), 0) else: self.assertEqual(len(bl), 1) # force the blacklist back in time to whenever the # measure was supposedly inserted. bl = bl[0] bl.time = time session.add(bl) session.commit() if month < N / 2: # We still haven't exceeded the threshold, so the # measurement was admitted. self.assertEqual(insert_result.get(), 1) self.assertEqual(session.query(WifiMeasure).count(), month + 1) if month % 2 == 0: # The station was (re)created. self.assertEqual(update_result.get(), (1, 0)) # One wifi record should exist. self.assertEqual(session.query(Wifi).count(), 1) else: # The station existed and was seen moving, # thereby activating the blacklist. self.assertEqual(update_result.get(), (1, 1)) self.assertEqual(bl.count, ((month + 1) / 2)) self.assertEqual(session.query(WifiBlacklist).count(), 1) self.assertEqual(session.query(Wifi).count(), 0) # Try adding one more measurement 1 day later # to be sure it is dropped by the now-active blacklist. next_day = encode_datetime(time + timedelta(days=1)) measure['time'] = next_day self.assertEqual( 0, insert_measures_wifi.delay([measure], utcnow=next_day).get()) else: # Blacklist has exceeded threshold, gone to "permanent" mode, # so no measures accepted, no stations seen. self.assertEqual(insert_result.get(), 0) self.assertEqual(update_result.get(), 0)
def test_blacklist_temporary_and_permanent(self): session = self.db_master_session # This test simulates a wifi that moves once a month, for 2 years. # The first 2 * PERMANENT_BLACKLIST_THRESHOLD (12) moves should be # temporary, forgotten after a week; after that it should be # permanently blacklisted. now = util.utcnow() # Station moves between these 4 points, all in the USA: points = [ # NYC (40.0, -74.0), # SF (37.0, -122.0), # Seattle (47.0, -122.0), # Miami (25.0, -80.0), ] N = 4 * PERMANENT_BLACKLIST_THRESHOLD for month in range(0, N): days_ago = (N - (month + 1)) * 30 time = now - timedelta(days=days_ago) time_enc = encode_datetime(time) measure = dict(key="ab1234567890", time=time_enc, lat=points[month % 4][0], lon=points[month % 4][1]) # insert_result is num-accepted-measures, override # utcnow to set creation date insert_result = insert_measures_wifi.delay([measure], utcnow=time_enc) # update_result is (num-stations, num-moving-stations) update_result = location_update_wifi.delay(min_new=1) # Assuming PERMANENT_BLACKLIST_THRESHOLD == 6: # # 0th insert will create the station # 1st insert will create first blacklist entry, delete station # 2nd insert will recreate the station at new position # 3rd insert will update blacklist, re-delete station # 4th insert will recreate the station at new position # 5th insert will update blacklist, re-delete station # 6th insert will recreate the station at new position # ... # 11th insert will make blacklisting permanent, re-delete station # 12th insert will not recreate station # 13th insert will not recreate station # ... # 23rd insert will not recreate station bl = session.query(WifiBlacklist).all() if month == 0: self.assertEqual(len(bl), 0) else: self.assertEqual(len(bl), 1) # force the blacklist back in time to whenever the # measure was supposedly inserted. bl = bl[0] bl.time = time session.add(bl) session.commit() if month < N / 2: # We still haven't exceeded the threshold, so the # measurement was admitted. self.assertEqual(insert_result.get(), 1) self.assertEqual(session.query(WifiMeasure).count(), month + 1) if month % 2 == 0: # The station was (re)created. self.assertEqual(update_result.get(), (1, 0)) # One wifi record should exist. self.assertEqual(session.query(Wifi).count(), 1) else: # The station existed and was seen moving, # thereby activating the blacklist. self.assertEqual(update_result.get(), (1, 1)) self.assertEqual(bl.count, ((month + 1) / 2)) self.assertEqual(session.query(WifiBlacklist).count(), 1) self.assertEqual(session.query(Wifi).count(), 0) # Try adding one more measurement 1 day later # to be sure it is dropped by the now-active blacklist. next_day = encode_datetime(time + timedelta(days=1)) measure['time'] = next_day self.assertEqual( 0, insert_measures_wifi.delay([measure], utcnow=next_day).get()) else: # Blacklist has exceeded threshold, gone to "permanent" mode, # so no measures accepted, no stations seen. self.assertEqual(insert_result.get(), 0) self.assertEqual(update_result.get(), 0)
def test_insert_measures(self): session = self.db_master_session time = util.utcnow() - timedelta(days=1) session.add(Wifi(key="ab1234567890")) session.add(Score(userid=1, key=SCORE_TYPE['new_wifi'], value=7)) session.flush() measure = dict( created=encode_datetime(time), lat=1.0, lon=2.0, time=encode_datetime(time), accuracy=0, altitude=0, altitude_accuracy=0, radio=-1, heading=52.9, speed=158.5, ) entries = [ { "key": "ab1234567890", "channel": 11, "signal": -80 }, { "key": "ab1234567890", "channel": 3, "signal": -90 }, { "key": "ab1234567890", "channel": 3, "signal": -80 }, { "key": "cd3456789012", "channel": 3, "signal": -90 }, ] for e in entries: e.update(measure) result = insert_measures_wifi.delay(entries, userid=1) self.assertEqual(result.get(), 4) measures = session.query(WifiMeasure).all() self.assertEqual(len(measures), 4) self.assertEqual(set([m.key for m in measures]), set(["ab1234567890", "cd3456789012"])) self.assertEqual(set([m.channel for m in measures]), set([3, 11])) self.assertEqual(set([m.signal for m in measures]), set([-80, -90])) self.assertEqual(set([m.heading or m in measures]), set([52.9])) self.assertEqual(set([m.speed or m in measures]), set([158.5])) wifis = session.query(Wifi).all() self.assertEqual(len(wifis), 2) self.assertEqual(set([w.key for w in wifis]), set(["ab1234567890", "cd3456789012"])) self.assertEqual(set([w.new_measures for w in wifis]), set([1, 3])) self.assertEqual(set([w.total_measures for w in wifis]), set([1, 3])) scores = session.query(Score).all() self.assertEqual(len(scores), 1) self.assertEqual(scores[0].key, SCORE_TYPE['new_wifi']) self.assertEqual(scores[0].value, 8) # test duplicate execution result = insert_measures_wifi.delay(entries, userid=1) self.assertEqual(result.get(), 4) # TODO this task isn't idempotent yet measures = session.query(WifiMeasure).all() self.assertEqual(len(measures), 8)
def test_insert_observations(self): session = self.db_master_session time = util.utcnow() - timedelta(days=1) today = util.utcnow().date() session.add(Wifi(key="ab1234567890", new_measures=0, total_measures=0)) session.add(Score(key=ScoreKey.new_wifi, userid=1, time=today, value=7)) session.flush() obs = dict( created=time, lat=1.0, lon=2.0, time=time, accuracy=0, altitude=0, altitude_accuracy=0, radio=-1, heading=52.9, speed=158.5, ) entries = [ { "key": "ab1234567890", "channel": 11, "signal": -80 }, { "key": "ab1234567890", "channel": 3, "signal": -90 }, { "key": "ab1234567890", "channel": 3, "signal": -80 }, { "key": "cd3456789012", "channel": 3, "signal": -90 }, ] for e in entries: e.update(obs) result = insert_measures_wifi.delay(entries, userid=1) self.assertEqual(result.get(), 4) observations = session.query(WifiObservation).all() self.assertEqual(len(observations), 4) self.assertEqual(set([o.key for o in observations]), set(["ab1234567890", "cd3456789012"])) self.assertEqual(set([o.channel for o in observations]), set([3, 11])) self.assertEqual(set([o.signal for o in observations]), set([-80, -90])) self.assertEqual(set([o.heading or o in observations]), set([52.9])) self.assertEqual(set([o.speed or o in observations]), set([158.5])) wifis = session.query(Wifi).all() self.assertEqual(len(wifis), 2) self.assertEqual(set([w.key for w in wifis]), set(["ab1234567890", "cd3456789012"])) self.assertEqual(set([w.new_measures for w in wifis]), set([1, 3])) self.assertEqual(set([w.total_measures for w in wifis]), set([1, 3])) scores = session.query(Score).all() self.assertEqual(len(scores), 1) self.assertEqual(scores[0].key, ScoreKey.new_wifi) self.assertEqual(scores[0].value, 8) # test duplicate execution result = insert_measures_wifi.delay(entries, userid=1) self.assertEqual(result.get(), 4) # TODO this task isn't idempotent yet observations = session.query(WifiObservation).all() self.assertEqual(len(observations), 8)