def test_created_from_blocklist_time(self): now = util.utcnow() last_week = now - TEMPORARY_BLOCKLIST_DURATION - timedelta(days=1) obs = CellObservationFactory.build() self.session.add( CellBlocklist(time=last_week, count=1, radio=obs.radio, mcc=obs.mcc, mnc=obs.mnc, lac=obs.lac, cid=obs.cid)) self.session.flush() # add a new entry for the previously blocklisted cell self.data_queue.enqueue([obs]) self.assertEqual(self.data_queue.size(), 1) update_cell.delay().get() # the cell was inserted again cells = self.session.query(Cell).all() self.assertEqual(len(cells), 1) # and the creation date was set to the date of the blocklist entry self.assertEqual(cells[0].created, last_week) self.check_statcounter(StatKey.cell, 1) self.check_statcounter(StatKey.unique_cell, 0)
def test_json(self): obs = CellObservationFactory.build(accuracy=None) result = CellObservation.from_json(simplejson.loads( simplejson.dumps(obs.to_json()))) self.assertTrue(type(result), CellObservation) self.assertTrue(result.accuracy is None) self.assertEqual(type(result.radio), Radio) self.assertEqual(result.radio, obs.radio) self.assertEqual(result.mcc, obs.mcc) self.assertEqual(result.mnc, obs.mnc) self.assertEqual(result.lac, obs.lac) self.assertEqual(result.cid, obs.cid) self.assertEqual(result.lat, obs.lat) self.assertEqual(result.lon, obs.lon)
def test_lock_timeout(self, celery, db_rw_drop_table, redis, ro_session, session, stats): obs = CellObservationFactory.build() cell = CellShardFactory.build( radio=obs.radio, mcc=obs.mcc, mnc=obs.mnc, lac=obs.lac, cid=obs.cid, samples=10, ) ro_session.add(cell) ro_session.flush() orig_add_area = CellUpdater.add_area_update orig_wait = CellUpdater._retry_wait num = [0] def mock_area(self, updated_areas, key, num=num, ro_session=ro_session): orig_add_area(self, updated_areas, key) num[0] += 1 if num[0] == 2: ro_session.rollback() try: CellUpdater._retry_wait = 0.0001 session.execute('set session innodb_lock_wait_timeout = 1') with mock.patch.object(CellUpdater, 'add_area_update', mock_area): self.queue_and_update(celery, [obs]) # the inner task logic was called exactly twice assert num[0] == 2 shard = CellShard.shard_model(obs.cellid) cells = session.query(shard).all() assert len(cells) == 1 assert cells[0].samples == 1 self.check_statcounter(redis, StatKey.cell, 1) self.check_statcounter(redis, StatKey.unique_cell, 1) stats.check( counter=[('data.observation.insert', 1, ['type:cell'])], timer=[('task', 1, ['task:data.update_cell'])], ) finally: CellUpdater._retry_wait = orig_wait for model in CellShard.shards().values(): session.execute(text( 'drop table %s;' % model.__tablename__))
def test_lock_timeout(self, celery, redis, session, session2, stats, restore_db): obs = CellObservationFactory.build() cell = CellShardFactory.build( radio=obs.radio, mcc=obs.mcc, mnc=obs.mnc, lac=obs.lac, cid=obs.cid, samples=10, ) session2.add(cell) session2.flush() orig_add_area = CellUpdater.add_area_update orig_wait = CellUpdater._retry_wait num = [0] def mock_area(self, updated_areas, key, num=num, session2=session2): orig_add_area(self, updated_areas, key) num[0] += 1 if num[0] == 2: session2.rollback() try: CellUpdater._retry_wait = 0.0001 session.execute('set session innodb_lock_wait_timeout = 1') with mock.patch.object(CellUpdater, 'add_area_update', mock_area): self.queue_and_update(celery, [obs]) # the inner task logic was called exactly twice assert num[0] == 2 shard = CellShard.shard_model(obs.cellid) cells = session.query(shard).all() assert len(cells) == 1 assert cells[0].samples == 1 self.check_statcounter(redis, StatKey.cell, 1) self.check_statcounter(redis, StatKey.unique_cell, 1) stats.check( counter=[('data.observation.insert', 1, ['type:cell'])], timer=[('task', 1, ['task:data.update_cell'])], ) finally: CellUpdater._retry_wait = orig_wait session.execute(text('drop table %s;' % cell.__tablename__))
def test_json(self): obs = CellObservationFactory.build(accuracy=None, source='fixed') result = CellObservation.from_json( simplejson.loads(simplejson.dumps(obs.to_json()))) assert type(result) is CellObservation assert result.accuracy is None assert type(result.radio), Radio assert result.radio == obs.radio assert result.mcc == obs.mcc assert result.mnc == obs.mnc assert result.lac == obs.lac assert result.cid == obs.cid assert result.lat == obs.lat assert result.lon == obs.lon assert result.source is ReportSource.fixed assert type(result.source) is ReportSource
def test_json(self): obs = CellObservationFactory.build( accuracy=None, source='fixed') result = CellObservation.from_json(simplejson.loads( simplejson.dumps(obs.to_json()))) assert type(result) is CellObservation assert result.accuracy is None assert type(result.radio), Radio assert result.radio == obs.radio assert result.mcc == obs.mcc assert result.mnc == obs.mnc assert result.lac == obs.lac assert result.cid == obs.cid assert result.lat == obs.lat assert result.lon == obs.lon assert result.source is ReportSource.fixed assert type(result.source) is ReportSource
def test_lock_timeout(self): obs = CellObservationFactory.build() cell = CellShardFactory.build( radio=obs.radio, mcc=obs.mcc, mnc=obs.mnc, lac=obs.lac, cid=obs.cid, samples=10, ) self.db_ro_session.add(cell) self.db_ro_session.flush() orig_add_area = CellUpdater.add_area_update orig_wait = CellUpdater._retry_wait num = [0] def mock_area(self, updated_areas, key, num=num, ro_session=self.db_ro_session): orig_add_area(self, updated_areas, key) num[0] += 1 if num[0] == 2: ro_session.rollback() try: CellUpdater._retry_wait = 0.001 self.session.execute('set session innodb_lock_wait_timeout = 1') with mock.patch.object(CellUpdater, 'add_area_update', mock_area): self._queue_and_update_cell([obs]) finally: CellUpdater._retry_wait = orig_wait # the inner task logic was called exactly twice self.assertEqual(num[0], 2) shard = CellShard.shard_model(obs.cellid) cells = self.session.query(shard).all() self.assertEqual(len(cells), 1) self.assertEqual(cells[0].samples, 1) self.check_statcounter(StatKey.cell, 1) self.check_statcounter(StatKey.unique_cell, 1) self.check_stats( counter=[('data.observation.insert', 1, ['type:cell'])], timer=[('task', 1, ['task:data.update_cell'])], )
def obs_factory(**kw): obs = CellObservationFactory.build(**kw) if obs is not None: observations.append(obs)
def test_retriable_exceptions( self, celery, redis, session, db_shared_session, metricsmock, errclass, errno, errmsg, backoff_sleep_mock, ): """Test database exceptions where the task should wait and try again.""" obs = CellObservationFactory.build(radio=Radio.lte) shard = CellShard.shard_model(obs.cellid) cell = CellShardFactory.build( radio=obs.radio, mcc=obs.mcc, mnc=obs.mnc, lac=obs.lac, cid=obs.cid, samples=10, created=datetime(2019, 12, 5, tzinfo=UTC), ) session.add(cell) session.commit() session.begin_nested() # Protect test cell from task rollback error = errclass(errno, errmsg) wrapped = InterfaceError.instance( statement="SELECT COUNT(*) FROM cell_area", params={}, orig=error, dbapi_base_err=MySQLError, ) with mock.patch.object(CellUpdater, "add_area_update", side_effect=[wrapped, None]): self._queue_and_update(celery, [obs], update_cell) assert CellUpdater.add_area_update.call_count == 2 backoff_sleep_mock.assert_called_once() cells = session.query(shard).all() assert len(cells) == 1 self.check_statcounter(redis, StatKey.cell, 1) # The existing cell record was updated cell = cells[0] assert cell.samples == 11 assert cell.created == datetime(2019, 12, 5, tzinfo=UTC) self.check_statcounter(redis, StatKey.unique_cell, 0) # Assert generated metrics are correct metricsmock.assert_incr_once("data.observation.insert", value=1, tags=["type:cell"]) metricsmock.assert_incr_once("data.station.confirm", value=1, tags=["type:cell"]) metricsmock.assert_timing_once("task", tags=["task:data.update_cell"]) metricsmock.assert_incr_once("data.station.dberror", tags=["type:cell", "errno:%s" % errno])
def obs_factory(**kw): obs = CellObservationFactory.build(**kw) observations.append(obs)
def test_retriable_exceptions(self, celery, redis, session, db, metricsmock, errclass, errno, errmsg): """Test database exceptions where the task should wait and try again.""" obs = CellObservationFactory.build(radio=Radio.lte) shard = CellShard.shard_model(obs.cellid) cell = CellShardFactory.build( radio=obs.radio, mcc=obs.mcc, mnc=obs.mnc, lac=obs.lac, cid=obs.cid, samples=10, created=datetime(2019, 12, 5, tzinfo=UTC), ) session.add(cell) session.commit() # TODO: Find a more elegant way to do this db.tests_task_use_savepoint = True error = errclass(errno, errmsg) wrapped = InterfaceError.instance( statement="SELECT COUNT(*) FROM cell_area", params={}, orig=error, dbapi_base_err=MySQLError, ) with mock.patch.object( CellUpdater, "add_area_update", side_effect=[ wrapped, None ]), mock.patch("ichnaea.data.station.time.sleep") as sleepy: self._queue_and_update(celery, [obs], update_cell) assert CellUpdater.add_area_update.call_count == 2 sleepy.assert_called_once_with(1) del db.tests_task_use_savepoint cells = session.query(shard).all() assert len(cells) == 1 self.check_statcounter(redis, StatKey.cell, 1) # The existing cell record was updated cell = cells[0] assert cell.samples == 11 assert cell.created == datetime(2019, 12, 5, tzinfo=UTC) self.check_statcounter(redis, StatKey.unique_cell, 0) # Assert generated metrics are correct assert metricsmock.has_record("incr", "data.observation.insert", value=1, tags=["type:cell"]) assert metricsmock.has_record("incr", "data.station.confirm", value=1, tags=["type:cell"]) assert metricsmock.has_record("timing", "task", tags=["task:data.update_cell"]) assert metricsmock.has_record( "incr", "data.station.dberror", value=1, tags=["type:cell", "errno:%s" % errno], )