def configure_data(redis_client): """ Configure fixed set of data queues. """ data_queues = { # *_incoming need to be the exact same as in webapp.config 'update_incoming': DataQueue('update_incoming', redis_client, batch=100, compress=True), 'transfer_incoming': DataQueue('transfer_incoming', redis_client, batch=100, compress=True), } for key in ('update_cellarea', ): data_queues[key] = DataQueue(key, redis_client, batch=100, json=False) for shard_id in BlueShard.shards().keys(): key = 'update_blue_' + shard_id data_queues[key] = DataQueue(key, redis_client, batch=500) for shard_id in DataMap.shards().keys(): key = 'update_datamap_' + shard_id data_queues[key] = DataQueue(key, redis_client, batch=500, json=False) for shard_id in CellShard.shards().keys(): key = 'update_cell_' + shard_id data_queues[key] = DataQueue(key, redis_client, batch=500) for shard_id in WifiShard.shards().keys(): key = 'update_wifi_' + shard_id data_queues[key] = DataQueue(key, redis_client, batch=500) return data_queues
def _queue_and_update_blue(self, obs): sharded_obs = defaultdict(list) for ob in obs: sharded_obs[BlueShard.shard_id(ob.mac)].append(ob) for shard_id, values in sharded_obs.items(): queue = self.celery_app.data_queues['update_blue_' + shard_id] queue.enqueue([value.to_json() for value in values]) update_blue.delay(shard_id=shard_id).get()
def _update_all(self): update_incoming.delay().get() for shard_id in BlueShard.shards().keys(): update_blue.delay(shard_id=shard_id).get() for shard_id in CellShard.shards().keys(): update_cell.delay(shard_id=shard_id).get() for shard_id in WifiShard.shards().keys(): update_wifi.delay(shard_id=shard_id).get()
def _update_all(self): schedule_export_reports.delay().get() for shard_id in BlueShard.shards().keys(): update_blue.delay(shard_id=shard_id).get() for shard_id in CellShard.shards().keys(): update_cell.delay(shard_id=shard_id).get() for shard_id in WifiShard.shards().keys(): update_wifi.delay(shard_id=shard_id).get()
def test_blue(self, celery, session): reports = self.add_reports(celery, blue_factor=1, cell_factor=0, wifi_factor=0) self._update_all(session) position = reports[0]["position"] blue_data = reports[0]["bluetoothBeacons"][0] shard = BlueShard.shard_model(blue_data["macAddress"]) blues = session.query(shard).all() assert len(blues) == 1 blue = blues[0] assert blue.lat == pytest.approx(position["latitude"]) assert blue.lon == pytest.approx(position["longitude"]) assert blue.mac == blue_data["macAddress"] assert blue.samples == 1
def test_blue(self): reports = self.add_reports(blue_factor=1, cell_factor=0, wifi_factor=0) self._update_all() position = reports[0]['position'] wifi_data = reports[0]['bluetoothBeacons'][0] mac = wifi_data['macAddress'] shard = BlueShard.shard_model(mac) blues = self.session.query(shard).all() self.assertEqual(len(blues), 1) blue = blues[0] self.assertEqual(blue.lat, position['latitude']) self.assertEqual(blue.lon, position['longitude']) self.assertEqual(blue.mac, wifi_data['macAddress']) self.assertEqual(blue.samples, 1)
def test_blue(self, celery, session): reports = self.add_reports( celery, blue_factor=1, cell_factor=0, wifi_factor=0) self._update_all(session) position = reports[0]['position'] blue_data = reports[0]['bluetoothBeacons'][0] shard = BlueShard.shard_model(blue_data['macAddress']) blues = session.query(shard).all() assert len(blues) == 1 blue = blues[0] assert blue.lat == position['latitude'] assert blue.lon == position['longitude'] assert blue.mac == blue_data['macAddress'] assert blue.samples == 1
def _update_all(self, session, datamap_only=False): ExportConfigFactory(name="internal", batch=0, schema="internal") session.flush() update_incoming.delay().get() if datamap_only: return for shard_id in BlueShard.shards().keys(): update_blue.delay(shard_id=shard_id).get() for shard_id in CellShard.shards().keys(): update_cell.delay(shard_id=shard_id).get() for shard_id in WifiShard.shards().keys(): update_wifi.delay(shard_id=shard_id).get()
def _update_all(self, session, datamap_only=False): ExportConfigFactory(name='internal', batch=0, schema='internal') session.flush() update_incoming.delay().get() if datamap_only: return for shard_id in BlueShard.shards().keys(): update_blue.delay(shard_id=shard_id).get() for shard_id in CellShard.shards().keys(): update_cell.delay(shard_id=shard_id).get() for shard_id in WifiShard.shards().keys(): update_wifi.delay(shard_id=shard_id).get()
def test_blue(self, celery, session): reports = self.add_reports(celery, blue_factor=1, cell_factor=0, wifi_factor=0) self._update_all(session) position = reports[0]['position'] blue_data = reports[0]['bluetoothBeacons'][0] shard = BlueShard.shard_model(blue_data['macAddress']) blues = session.query(shard).all() assert len(blues) == 1 blue = blues[0] assert blue.lat == position['latitude'] assert blue.lon == position['longitude'] assert blue.mac == blue_data['macAddress'] assert blue.samples == 1
def test_blue_duplicated(self, celery, session): self.add_reports(celery, blue_factor=1, cell_factor=0, wifi_factor=0) # duplicate the Bluetooth entry inside the report item = self._pop_item(celery) report = item["report"] blue = report["bluetoothBeacons"][0] mac = blue["macAddress"] report["bluetoothBeacons"].append(blue.copy()) report["bluetoothBeacons"].append(blue.copy()) report["bluetoothBeacons"][1]["signalStrength"] += 2 report["bluetoothBeacons"][2]["signalStrength"] -= 2 self._push_item(celery, item) self._update_all(session) shard = BlueShard.shard_model(mac) blues = session.query(shard).all() assert len(blues) == 1 assert blues[0].samples == 1
def test_blue_duplicated(self): self.add_reports(blue_factor=1, cell_factor=0, wifi_factor=0) # duplicate the Bluetooth entry inside the report item = self._pop_item() report = item['report'] blue = report['bluetoothBeacons'][0] mac = blue['macAddress'] report['bluetoothBeacons'].append(blue.copy()) report['bluetoothBeacons'].append(blue.copy()) report['bluetoothBeacons'][1]['signalStrength'] += 2 report['bluetoothBeacons'][2]['signalStrength'] -= 2 self._push_item(item) self._update_all() shard = BlueShard.shard_model(mac) blues = self.session.query(shard).all() self.assertEqual(len(blues), 1) self.assertEqual(blues[0].samples, 1)
def test_blue_duplicated(self, celery, session): self.add_reports( celery, blue_factor=1, cell_factor=0, wifi_factor=0) # duplicate the Bluetooth entry inside the report item = self._pop_item(celery) report = item['report'] blue = report['bluetoothBeacons'][0] mac = blue['macAddress'] report['bluetoothBeacons'].append(blue.copy()) report['bluetoothBeacons'].append(blue.copy()) report['bluetoothBeacons'][1]['signalStrength'] += 2 report['bluetoothBeacons'][2]['signalStrength'] -= 2 self._push_item(celery, item) self._update_all(session) shard = BlueShard.shard_model(mac) blues = session.query(shard).all() assert len(blues) == 1 assert blues[0].samples == 1
def test_blue_duplicated(self): self.add_reports(blue_factor=1, cell_factor=0, wifi_factor=0) # duplicate the Bluetooth entry inside the report queue = self.celery_app.export_queues['queue_export_internal'] items = queue.dequeue(queue.queue_key()) report = items[0]['report'] blue = report['bluetoothBeacons'][0] mac = blue['macAddress'] report['bluetoothBeacons'].append(blue.copy()) report['bluetoothBeacons'].append(blue.copy()) report['bluetoothBeacons'][1]['signalStrength'] += 2 report['bluetoothBeacons'][2]['signalStrength'] -= 2 queue.enqueue(items, queue.queue_key()) self._update_all() shard = BlueShard.shard_model(mac) blues = self.session.query(shard).all() self.assertEqual(len(blues), 1) self.assertEqual(blues[0].samples, 1)
def test_new(self): utcnow = util.utcnow() obs = BlueObservationFactory.build() self._queue_and_update_blue([obs]) shard = BlueShard.shard_model(obs.mac) blues = self.session.query(shard).all() self.assertEqual(len(blues), 1) blue = blues[0] self.assertAlmostEqual(blue.lat, obs.lat) self.assertAlmostEqual(blue.max_lat, obs.lat) self.assertAlmostEqual(blue.min_lat, obs.lat) self.assertAlmostEqual(blue.lon, obs.lon) self.assertAlmostEqual(blue.max_lon, obs.lon) self.assertAlmostEqual(blue.min_lon, obs.lon) self.assertEqual(blue.radius, 0) self.assertEqual(blue.region, 'GB') self.assertEqual(blue.samples, 1) self.assertAlmostEqual(blue.weight, 1.0, 2) self.assertEqual(blue.created.date(), utcnow.date()) self.assertEqual(blue.modified.date(), utcnow.date()) self.assertEqual(blue.block_first, None) self.assertEqual(blue.block_last, None) self.assertEqual(blue.block_count, None)
def configure_data(redis_client): """ Configure fixed set of data queues. """ data_queues = { # update_incoming needs to be the exact same as in webapp.config 'update_incoming': DataQueue('update_incoming', redis_client, batch=100, compress=True), } for key in ('update_cellarea', 'update_cellarea_ocid'): data_queues[key] = DataQueue(key, redis_client, batch=100, json=False) for shard_id in BlueShard.shards().keys(): key = 'update_blue_' + shard_id data_queues[key] = DataQueue(key, redis_client, batch=500) for shard_id in DataMap.shards().keys(): key = 'update_datamap_' + shard_id data_queues[key] = DataQueue(key, redis_client, batch=500, json=False) for shard_id in CellShard.shards().keys(): key = 'update_cell_' + shard_id data_queues[key] = DataQueue(key, redis_client, batch=500) for shard_id in WifiShard.shards().keys(): key = 'update_wifi_' + shard_id data_queues[key] = DataQueue(key, redis_client, batch=500) return data_queues