def test_update_does_not_overwrite(self): self.add_counter(StatKey.cell, self.yesterday, 5) self.add_counter(StatKey.cell, self.today, 7) self.session.add(Stat(key=StatKey.cell, time=self.two_days, value=1)) self.session.add(Stat(key=StatKey.cell, time=self.yesterday, value=3)) self.session.flush() update_statcounter.delay(ago=1).get() self.check_stat(StatKey.cell, self.yesterday, 8)
def test_update_two_days(self, celery, redis, session): self.add_counter(redis, StatKey.cell, self.yesterday, 5) self.add_counter(redis, StatKey.cell, self.today, 7) session.add(Stat(key=StatKey.cell, time=self.two_days, value=1)) session.add(Stat(key=StatKey.cell, time=self.yesterday, value=3)) session.flush() update_statcounter.delay().get() self.check_stat(session, StatKey.cell, self.yesterday, 8) self.check_stat(session, StatKey.cell, self.today, 15)
def test_local_import(self): self.import_csv() cells = self.session.query(OCIDCell).all() self.assertEqual(len(cells), 9) lacs = set([ (cell.radio, cell.mcc, cell.mnc, cell.lac) for cell in cells]) self.assertEqual( self.session.query(OCIDCellArea).count(), len(lacs)) update_statcounter.delay(ago=0).get() today = util.utcnow().date() stat_key = Stat.to_hashkey(key=StatKey.unique_ocid_cell, time=today) self.assertEqual(Stat.getkey(self.session, stat_key).value, 9)
def test_update_all_keys(self): self.add_counter(StatKey.cell, self.yesterday, 2) self.add_counter(StatKey.wifi, self.yesterday, 3) self.add_counter(StatKey.unique_cell, self.yesterday, 4) self.add_counter(StatKey.unique_wifi, self.yesterday, 5) self.add_counter(StatKey.unique_cell_ocid, self.yesterday, 6) self.session.add(Stat(key=StatKey.cell, time=self.two_days, value=7)) self.session.add(Stat(key=StatKey.wifi, time=self.two_days, value=8)) self.session.flush() update_statcounter.delay(ago=1).get() self.check_stat(StatKey.cell, self.yesterday, 9) self.check_stat(StatKey.wifi, self.yesterday, 11) self.check_stat(StatKey.unique_cell, self.yesterday, 4) self.check_stat(StatKey.unique_wifi, self.yesterday, 5) self.check_stat(StatKey.unique_cell_ocid, self.yesterday, 6)
def test_update_from_yesterday(self, celery, redis, session): self.add_counter(redis, StatKey.cell, self.today, 4) session.add(Stat(key=StatKey.cell, time=self.yesterday, value=2)) session.flush() update_statcounter.delay().get() self.check_stat(session, StatKey.cell, self.today, 6)
def test_update_from_yesterday(self): self.add_counter(StatKey.cell, self.yesterday, 3) self.add_counter(StatKey.cell, self.today, 4) self.session.add(Stat(key=StatKey.cell, time=self.two_days, value=2)) self.session.flush() update_statcounter.delay(ago=1).get() self.check_stat(StatKey.cell, self.yesterday, 5)
def test_update_with_gap(self, celery, redis, session): a_week_ago = self.today - timedelta(days=7) self.add_counter(redis, StatKey.cell, self.today, 3) session.add(Stat(key=StatKey.cell, time=a_week_ago, value=7)) session.flush() update_statcounter.delay().get() self.check_stat(session, StatKey.cell, self.today, 10)
def test_update_with_gap(self): a_week_ago = self.today - timedelta(days=7) self.add_counter(StatKey.cell, self.yesterday, 3) self.add_counter(StatKey.cell, self.today, 4) self.session.add(Stat(key=StatKey.cell, time=a_week_ago, value=7)) self.session.flush() update_statcounter.delay(ago=1).get() self.check_stat(StatKey.cell, self.yesterday, 10)
def test_update_all_keys(self, celery, redis, session): self.add_counter(redis, StatKey.blue, self.today, 1) self.add_counter(redis, StatKey.cell, self.today, 2) self.add_counter(redis, StatKey.wifi, self.today, 3) self.add_counter(redis, StatKey.unique_blue, self.today, 4) self.add_counter(redis, StatKey.unique_cell, self.today, 5) self.add_counter(redis, StatKey.unique_wifi, self.today, 6) session.add(Stat(key=StatKey.blue, time=self.yesterday, value=8)) session.add(Stat(key=StatKey.cell, time=self.yesterday, value=9)) session.add(Stat(key=StatKey.wifi, time=self.yesterday, value=10)) session.flush() update_statcounter.delay().get() self.check_stat(session, StatKey.blue, self.today, 9) self.check_stat(session, StatKey.cell, self.today, 11) self.check_stat(session, StatKey.wifi, self.today, 13) self.check_stat(session, StatKey.unique_blue, self.today, 4) self.check_stat(session, StatKey.unique_cell, self.today, 5) self.check_stat(session, StatKey.unique_wifi, self.today, 6)
def test_multiple_updates_for_today(self): self.add_counter(StatKey.cell, self.today, 4) self.session.add(Stat(key=StatKey.cell, time=self.yesterday, value=5)) self.session.flush() update_statcounter.delay(ago=0).get() self.check_stat(StatKey.cell, self.today, 9) self.add_counter(StatKey.cell, self.today, 3) update_statcounter.delay(ago=0).get() self.check_stat(StatKey.cell, self.today, 12)
def test_local_import_delta(self): old_time = 1407000000 new_time = 1408000000 old_date = datetime.fromtimestamp(old_time).replace(tzinfo=UTC) new_date = datetime.fromtimestamp(new_time).replace(tzinfo=UTC) today = util.utcnow().date() self.import_csv(time=old_time) cells = self.session.query(OCIDCell).all() self.assertEqual(len(cells), 9) update_statcounter.delay(ago=0).get() stat_key = Stat.to_hashkey(key=StatKey.unique_ocid_cell, time=today) self.assertEqual(Stat.getkey(self.session, stat_key).value, 9) lacs = set([ (cell.radio, cell.mcc, cell.mnc, cell.lac) for cell in cells]) self.assertEqual( self.session.query(OCIDCellArea).count(), len(lacs)) # update some entries self.import_csv(lo=5, hi=13, time=new_time) self.session.commit() cells = (self.session.query(OCIDCell) .order_by(OCIDCell.modified).all()) self.assertEqual(len(cells), 12) for i in range(0, 4): self.assertEqual(cells[i].modified, old_date) for i in range(4, 12): self.assertEqual(cells[i].modified, new_date) lacs = set([ (cell.radio, cell.mcc, cell.mnc, cell.lac) for cell in cells]) self.assertEqual( self.session.query(OCIDCellArea).count(), len(lacs)) update_statcounter.delay(ago=0).get() stat_key = Stat.to_hashkey(key=StatKey.unique_ocid_cell, time=today) self.assertEqual(Stat.getkey(self.session, stat_key).value, 12)
def update_key(self, stat_key, day): # determine the value from the day before query = (self.session.query(Stat) .filter(Stat.key == stat_key) .filter(Stat.time < day) .order_by(Stat.time.desc())) before = query.first() old_value = 0 if before: old_value = before.value # get the value from redis for the day in question stat_counter = StatCounter(stat_key, day) value = stat_counter.get(self.redis_client) # insert or update a new stat value hashkey = Stat.to_hashkey(key=stat_key, time=day) Stat.incr(self.session, hashkey, value, old=old_value) # queue the redis value to be decreased stat_counter.decr(self.pipe, value)
def _one(self, key, time): return Stat(key=key, time=time, value=1)
def check_stat(self, stat_key, time, value): hashkey = Stat.to_hashkey(key=stat_key, time=time) stat = Stat.getkey(self.session, hashkey) self.assertEqual(stat.value, value)