def trim_and_check(keep): from ichnaea.content.tasks import get_curr_stat # trim model to 'keep' measures per key result = trim_func.delay(keep) result.get() # check that exactly the rows expected were kept measures = session.query(measure_model).all() self.assertEqual(len(measures), len(keys) * keep) for k in keys: for i in range(measures_per_key - keep / 2, measures_per_key): self.assertTrue( any(m.lat == m1 + i and m.lat == m1 + i and kcmp(m, k) for m in measures)) self.assertTrue( any(m.lat == m2 + i and m.lat == m2 + i and kcmp(m, k) for m in measures)) stations = session.query(unique_model).all() for s in stations: self.assertTrue(s.new_measures <= s.total_measures) # check that the deletion stat was updated self.assertEqual(get_curr_stat(session, delstat), len(keys) * (2 * measures_per_key - keep))
def trim_and_check(keep): from ichnaea.content.tasks import get_curr_stat # trim model to 'keep' measures per key result = trim_func.delay(keep) result.get() # check that exactly the rows expected were kept measures = session.query(measure_model).all() self.assertEqual(len(measures), len(keys) * keep) for k in keys: for i in range(measures_per_key - keep / 2, measures_per_key): self.assertTrue(any(m.lat == m1 + i and m.lat == m1 + i and kcmp(m, k) for m in measures)) self.assertTrue(any(m.lat == m2 + i and m.lat == m2 + i and kcmp(m, k) for m in measures)) stations = session.query(unique_model).all() for s in stations: self.assertTrue(s.new_measures <= s.total_measures) # check that the deletion stat was updated self.assertEqual(get_curr_stat(session, delstat), len(keys) * (2 * measures_per_key - keep))
def check_no_trim_young_data(self, unique_model, measure_model, trim_func, kinit, delstat): """ Check that a trim function run against young data leaves it alone. """ from ichnaea.content.tasks import get_curr_stat session = self.db_master_session measures = [] keys = range(3) measures_per_key = 4 m1 = 10000000 m2 = 20000000 session.query(unique_model).delete() session.query(measure_model).delete() session.flush() for k in keys: kargs = kinit(k) measures.append( unique_model(lat=(m1 + m2) / 2, lon=(m1 + m2) / 2, total_measures=measures_per_key * 2, **kargs)) for i in range(measures_per_key): measures.append(measure_model(lat=m1 + i, lon=m1 + i, **kargs)) measures.append(measure_model(lat=m2 + i, lon=m2 + i, **kargs)) session.add_all(measures) session.flush() measures = session.query(measure_model).count() self.assertEqual(measures, measures_per_key * 2 * len(keys)) dels = get_curr_stat(session, delstat) result = trim_func.delay(2) result.get() # check that all data was preserved measures = session.query(measure_model).count() self.assertEqual(measures, measures_per_key * 2 * len(keys)) # check that the deletion stat is unchanged self.assertEqual(get_curr_stat(session, delstat), dels)
def check_no_trim_young_data(self, unique_model, measure_model, trim_func, kinit, delstat): """ Check that a trim function run against young data leaves it alone. """ from ichnaea.content.tasks import get_curr_stat session = self.db_master_session measures = [] keys = range(3) measures_per_key = 4 m1 = 10000000 m2 = 20000000 session.query(unique_model).delete() session.query(measure_model).delete() session.flush() for k in keys: kargs = kinit(k) measures.append(unique_model(lat=(m1 + m2) / 2, lon=(m1 + m2) / 2, total_measures=measures_per_key * 2, **kargs)) for i in range(measures_per_key): measures.append(measure_model(lat=m1 + i, lon=m1 + i, **kargs)) measures.append(measure_model(lat=m2 + i, lon=m2 + i, **kargs)) session.add_all(measures) session.flush() measures = session.query(measure_model).count() self.assertEqual(measures, measures_per_key * 2 * len(keys)) dels = get_curr_stat(session, delstat) result = trim_func.delay(2) result.get() # check that all data was preserved measures = session.query(measure_model).count() self.assertEqual(measures, measures_per_key * 2 * len(keys)) # check that the deletion stat is unchanged self.assertEqual(get_curr_stat(session, delstat), dels)
def test_incr_and_curr_stats(self): """ Check that ichnaea.content.tasks.{incr_stat,get_curr_stat} work on the most-recent-stat associated with a given day, and copy forward each day's stat to the next day, as they go. """ from ichnaea.content.tasks import incr_stat, get_curr_stat session = self.db_master_session today = datetime.utcnow().date() yesterday = (today - timedelta(1)) two_days = (today - timedelta(2)) for stat in ['deleted_cell', 'deleted_wifi']: p = 0 for i in range(1, 10): p += i incr_stat(session, stat, i, two_days) self.assertEqual(get_curr_stat(session, stat, two_days), p) self.assertEqual(get_curr_stat(session, stat, yesterday), p) self.assertEqual(get_curr_stat(session, stat, today), p) self.assertEqual(get_curr_stat(session, stat), p) q = p for i in range(1, 10): q += i incr_stat(session, stat, i, yesterday) self.assertEqual(get_curr_stat(session, stat, two_days), p) self.assertEqual(get_curr_stat(session, stat, yesterday), q) self.assertEqual(get_curr_stat(session, stat, today), q) self.assertEqual(get_curr_stat(session, stat), q) r = q for i in range(1, 10): r += i incr_stat(session, stat, i, today) self.assertEqual(get_curr_stat(session, stat, two_days), p) self.assertEqual(get_curr_stat(session, stat, yesterday), q) self.assertEqual(get_curr_stat(session, stat, today), r) self.assertEqual(get_curr_stat(session, stat), r)