def test_unthrottle_wifi_measures(self): session = self.db_master_session block = MeasureBlock() block.measure_type = MEASURE_TYPE_CODE['wifi'] block.start_id = 120 block.end_id = 140 block.s3_key = 'fake_key' block.archive_sha = 'fake_sha' block.archive_date = None session.add(block) k = dict(lat=1.0, lon=1.0) for i in range(100, 150): session.add(WifiMeasure(id=i, key=str(i), created=self.really_old)) session.add(Wifi(total_measures=11000, key=str(i), **k)) session.commit() with patch.object(S3Backend, 'check_archive', lambda x, y, z: True): delete_wifimeasure_records.delay(batch=7).get() wifi_unthrottle_measures.delay(10000, 1000).get() wifis = session.query(Wifi).all() self.assertEquals(len(wifis), 50) for wifi in wifis: if 120 <= int(wifi.key) and int(wifi.key) < 140: self.assertEquals(wifi.total_measures, 0) else: self.assertEquals(wifi.total_measures, 1) self.check_stats(counter=['items.wifi_unthrottled'])
def test_unthrottle_cell_measures(self): session = self.db_master_session block = MeasureBlock() block.measure_type = MEASURE_TYPE_CODE['cell'] block.start_id = 120 block.end_id = 140 block.s3_key = 'fake_key' block.archive_sha = 'fake_sha' block.archive_date = None session.add(block) k = dict(mcc=1, mnc=2, lac=4, lat=1.0, lon=1.0) for i in range(100, 150): session.add(CellMeasure(id=i, cid=i, created=self.really_old, **k)) session.add(Cell(total_measures=11000, cid=i, **k)) session.commit() with patch.object(S3Backend, 'check_archive', lambda x, y, z: True): delete_cellmeasure_records.delay(batch=3).get() cell_unthrottle_measures.delay(10000, 1000).get() cells = session.query(Cell).all() self.assertEquals(len(cells), 50) for cell in cells: if 120 <= cell.cid and cell.cid < 140: self.assertEquals(cell.total_measures, 0) else: self.assertEquals(cell.total_measures, 1) self.check_stats(counter=['items.cell_unthrottled'])
def schedule_measure_archival(self, measure_type, limit=100, batch=1000000): blocks = [] measure_meta = MEASURE_TYPE_META[measure_type] measure_cls = measure_meta['class'] with self.db_session() as session: table_min_id = 0 table_max_id = 0 query = session.query(measure_cls.id).order_by( measure_cls.id.asc()) record = query.first() if record is not None: table_min_id = record[0] query = session.query(measure_cls.id).order_by( measure_cls.id.desc()) record = query.first() if record is not None: table_max_id = record[0] if not table_max_id: # no data in the table return blocks query = session.query(MeasureBlock.end_id).filter( MeasureBlock.measure_type == measure_type).order_by( MeasureBlock.end_id.desc()) record = query.first() if record is not None: min_id = record[0] else: min_id = table_min_id max_id = table_max_id if max_id - min_id < batch - 1: # Not enough to fill a block return blocks # We're using half-open ranges, so we need to bump the max_id max_id += 1 this_max_id = min_id + batch i = 0 while i < limit and (this_max_id - min_id) == batch: cm_blk = MeasureBlock(start_id=min_id, end_id=this_max_id, measure_type=measure_type) blocks.append((cm_blk.start_id, cm_blk.end_id)) session.add(cm_blk) min_id = this_max_id this_max_id = min(batch + this_max_id, max_id) i += 1 session.commit() return blocks
def test_delete_wifi_measures(self): session = self.db_master_session block = MeasureBlock() block.measure_type = MEASURE_TYPE_CODE['wifi'] block.start_id = 120 block.end_id = 140 block.s3_key = 'fake_key' block.archive_sha = 'fake_sha' block.archive_date = None session.add(block) for i in range(100, 150): session.add(WifiMeasure(id=i, created=self.really_old)) session.commit() with patch.object(S3Backend, 'check_archive', lambda x, y, z: True): delete_wifimeasure_records.delay(batch=7).get() self.assertEquals(session.query(WifiMeasure).count(), 30) self.assertTrue(block.archive_date is not None)
def test_skip_delete_new_blocks(self): now = util.utcnow() session = self.db_master_session block = MeasureBlock() block.measure_type = MEASURE_TYPE_CODE['cell'] block.start_id = 120 block.end_id = 140 block.s3_key = 'fake_key' block.archive_sha = 'fake_sha' block.archive_date = None session.add(block) measures = [] for i in range(100, 150): measures.append(CellMeasure(id=i, created=now)) session.add_all(measures) session.commit() with patch.object(S3Backend, 'check_archive', lambda x, y, z: True): delete_cellmeasure_records.delay(batch=3).get() self.assertEquals(session.query(CellMeasure).count(), 50) # The archive_date should *not* be set as we haven't deleted # the actual records yet self.assertTrue(block.archive_date is None) # Update all the create dates to be far in the past for m in measures: m.created = self.really_old session.add_all(measures) session.commit() with patch.object(S3Backend, 'check_archive', lambda x, y, z: True): delete_cellmeasure_records.delay(batch=3).get() self.assertEquals(session.query(CellMeasure).count(), 30) # The archive_date should now be set as the measure records # have been deleted. self.assertTrue(block.archive_date is not None)
def test_delete_wifi_measures(self): session = self.db_master_session block = MeasureBlock() block.measure_type = MEASURE_TYPE['wifi'] block.start_id = 120 block.end_id = 140 block.s3_key = 'fake_key' block.archive_sha = 'fake_sha' block.archive_date = None session.add(block) for i in range(100, 150): session.add(WifiMeasure(id=i)) session.commit() with patch.object(S3Backend, 'check_archive', lambda x, y, z: True): delete_wifimeasure_records() self.assertEquals(session.query(WifiMeasure).count(), 29) self.assertTrue(block.archive_date is not None)
def test_delete_cell_measures(self): session = self.db_master_session block = MeasureBlock() block.measure_type = MEASURE_TYPE_CODE['cell'] block.start_id = 120 block.end_id = 140 block.s3_key = 'fake_key' block.archive_sha = 'fake_sha' block.archive_date = None session.add(block) for i in range(100, 150): session.add(CellMeasure(id=i, created=self.really_old)) session.commit() with patch.object(S3Backend, 'check_archive', lambda x, y, z: True): delete_cellmeasure_records.delay(batch=3).get() self.assertEquals(session.query(CellMeasure).count(), 30) self.assertTrue(block.archive_date is not None)