def add_reports(self, num=1, blue_factor=0, cell_factor=1, wifi_factor=2, api_key='test', nickname=None, blue_key=None, cell_mcc=None, wifi_key=None, lat=None, lon=None): reports = [] for i in range(num): pos = CellShardFactory.build() report = { 'timestamp': time.time() * 1000.0, 'position': {}, 'bluetoothBeacons': [], 'cellTowers': [], 'wifiAccessPoints': [], } report['position']['latitude'] = lat or pos.lat report['position']['longitude'] = lon or pos.lon report['position']['accuracy'] = 17.0 + i blues = BlueShardFactory.build_batch(blue_factor, lat=pos.lat, lon=pos.lon) for blue in blues: blue_data = { 'macAddress': blue_key or blue.mac, 'signalStrength': -100 + i, } report['bluetoothBeacons'].append(blue_data) cells = CellShardFactory.build_batch(cell_factor, lat=pos.lat, lon=pos.lon) for cell in cells: cell_data = { 'radioType': cell.radio.name, 'mobileCountryCode': cell_mcc or cell.mcc, 'mobileNetworkCode': cell.mnc, 'locationAreaCode': cell.lac, 'cellId': cell.cid, 'primaryScramblingCode': cell.psc, 'signalStrength': -110 + i, } report['cellTowers'].append(cell_data) wifis = WifiShardFactory.build_batch(wifi_factor, lat=pos.lat, lon=pos.lon) for wifi in wifis: wifi_data = { 'macAddress': wifi_key or wifi.mac, 'signalStrength': -90 + i, 'ssid': 'my-wifi', } report['wifiAccessPoints'].append(wifi_data) reports.append(report) items = [{'api_key': api_key, 'nickname': nickname, 'report': rep} for rep in reports] self.incoming_queue.enqueue(items) update_incoming.delay().get() return reports
def test_upload(self, celery, session, stats): ExportConfigFactory( name='backup', batch=3, schema='s3', url='s3://bucket/backups/{source}/{api_key}/{year}/{month}/{day}') ApiKeyFactory(valid_key='e5444-794') session.flush() reports = self.add_reports(celery, 3) self.add_reports(celery, 3, api_key='e5444-794', source='gnss') self.add_reports(celery, 3, api_key='e5444-794', source='fused') self.add_reports(celery, 3, api_key=None) mock_conn = mock.MagicMock() mock_bucket = mock.MagicMock() mock_obj = mock.MagicMock() mock_conn.return_value.Bucket.return_value = mock_bucket mock_bucket.Object.return_value = mock_obj with mock.patch.object(boto3, 'resource', mock_conn): update_incoming.delay().get() obj_calls = mock_bucket.Object.call_args_list put_calls = mock_obj.put.call_args_list assert len(obj_calls) == 4 assert len(put_calls) == 4 keys = [] test_export = None for obj_call, put_call in zip(obj_calls, put_calls): s3_key = obj_call[0][0] assert s3_key.startswith('backups/') assert s3_key.endswith('.json.gz') assert put_call[1]['Body'] assert put_call[1]['ContentType'] == 'application/json' assert put_call[1]['ContentEncoding'] == 'gzip' keys.append(s3_key) if 'test' in s3_key: test_export = put_call[1]['Body'] # extract second and third path segment from key names groups = [tuple(key.split('/')[1:3]) for key in keys] assert (set(groups) == set([('gnss', 'test'), ('gnss', 'no_key'), ('gnss', 'e5444-794'), ('fused', 'e5444-794')])) # check uploaded content uploaded_text = util.decode_gzip(test_export) send_reports = simplejson.loads(uploaded_text)['items'] assert len(send_reports) == 3 expect = [report['position']['accuracy'] for report in reports] gotten = [report['position']['accuracy'] for report in send_reports] assert set(expect) == set(gotten) stats.check(counter=[ ('data.export.batch', 4, 1, ['key:backup']), ('data.export.upload', 4, ['key:backup', 'status:success']), ], timer=[ ('data.export.upload', 4, ['key:backup']), ])
def test_queues(self, celery, redis, session): ApiKeyFactory(valid_key="test2") ExportConfigFactory(name="test", batch=3, skip_keys=frozenset(["export_source"])) ExportConfigFactory(name="everything", batch=5) ExportConfigFactory(name="no_test", batch=2, skip_keys=frozenset(["test", "test_1"])) ExportConfigFactory(name="query", batch=2, skip_sources=frozenset(["gnss"])) session.flush() self.add_reports(celery, 4) self.add_reports(celery, 1, api_key="test2") self.add_reports(celery, 2, api_key=None, source="gnss") self.add_reports(celery, 1, api_key="test", source="query") update_incoming.delay().get() for queue_key, num in [ ("queue_export_test", 2), ("queue_export_everything", 3), ("queue_export_no_test", 1), ("queue_export_query", 1), ]: assert self.queue_length(redis, queue_key) == num
def test_queues(self, celery, redis, session): ApiKeyFactory(valid_key='test2') ExportConfigFactory(name='test', batch=3, skip_keys=frozenset(['export_source'])) ExportConfigFactory(name='everything', batch=5) ExportConfigFactory(name='no_test', batch=2, skip_keys=frozenset(['test', 'test_1'])) ExportConfigFactory(name='query', batch=2, skip_sources=frozenset(['gnss'])) session.flush() self.add_reports(celery, 4) self.add_reports(celery, 1, api_key='test2') self.add_reports(celery, 2, api_key=None, source='gnss') self.add_reports(celery, 1, api_key='test', source='query') update_incoming.delay().get() for queue_key, num in [('queue_export_test', 2), ('queue_export_everything', 3), ('queue_export_no_test', 1), ('queue_export_query', 1)]: assert self.queue_length(redis, queue_key) == num
def test_datamap(self): self.add_reports(1, cell_factor=0, wifi_factor=2, lat=50.0, lon=10.0) self.add_reports(2, cell_factor=0, wifi_factor=2, lat=20.0, lon=-10.0) update_incoming.delay().get() self.assertEqual( self.celery_app.data_queues['update_datamap_ne'].size(), 1) self.assertEqual( self.celery_app.data_queues['update_datamap_sw'].size(), 1)
def test_null_position(self, celery, redis, session): """Reports with null position are queued.""" ApiKeyFactory(valid_key="no-position") ExportConfigFactory(name="everything", batch=5) session.flush() self.add_reports(celery, 1, api_key="no-position", set_position=False) update_incoming.delay().get() assert self.queue_length(redis, "queue_export_everything") == 1
def _update_all(self): update_incoming.delay().get() for shard_id in BlueShard.shards().keys(): update_blue.delay(shard_id=shard_id).get() for shard_id in CellShard.shards().keys(): update_cell.delay(shard_id=shard_id).get() for shard_id in WifiShard.shards().keys(): update_wifi.delay(shard_id=shard_id).get()
def test_upload(self, celery, session, metricsmock): ApiKeyFactory(valid_key="e5444-7946") ExportConfigFactory( name="test", batch=4, schema="geosubmit", url="http://127.0.0.1:9/v2/geosubmit?key=external", ) session.flush() reports = [] reports.extend(self.add_reports(celery, 1, source="gnss")) reports.extend(self.add_reports(celery, 1, api_key="e5444e9f-7946")) reports.extend( self.add_reports(celery, 1, api_key=None, source="fused")) reports.extend(self.add_reports(celery, 1, set_position=False)) with requests_mock.Mocker() as mock: mock.register_uri("POST", requests_mock.ANY, text="{}") update_incoming.delay().get() assert mock.call_count == 1 req = mock.request_history[0] # check headers assert req.headers["Content-Type"] == "application/json" assert req.headers["Content-Encoding"] == "gzip" assert req.headers["User-Agent"] == "ichnaea" body = util.decode_gzip(req.body) send_reports = json.loads(body)["items"] assert len(send_reports) == 4 for field in ("accuracy", "source", "timestamp"): expect = [(report["position"] or {}).get(field) for report in reports] gotten = [(report["position"] or {}).get(field) for report in send_reports] assert set(expect) == set(gotten) assert set([w["ssid"] for w in send_reports[0]["wifiAccessPoints"] ]) == set(["my-wifi"]) assert metricsmock.has_record("incr", "data.export.batch", value=1, tags=["key:test"]) assert metricsmock.has_record("incr", "data.export.upload", value=1, tags=["key:test", "status:200"]) assert metricsmock.has_record("timing", "data.export.upload.timing", tags=["key:test"])
def test_queues(self): self.add_reports(4) self.add_reports(1, api_key='test2') self.add_reports(2, api_key=None) update_incoming.delay().get() for queue_key, num in [ ('queue_export_test', 1), ('queue_export_everything', 2), ('queue_export_no_test', 1)]: self.assertEqual(self.queue_length(queue_key), num)
def _post(self, items, api_key=None, status=status, **kw): url = self.url if api_key: url += '?key=%s' % api_key extra = {'HTTP_X_FORWARDED_FOR': self.geoip_data['London']['ip']} result = self.app.post_json( url, {'items': items}, status=status, extra_environ=extra, **kw) while self.data_queue.size() > 0: update_incoming.delay().get() return result
def test_upload(self, celery, session, stats): ExportConfigFactory( name='backup', batch=3, schema='s3', url='s3://bucket/backups/{source}/{api_key}/{year}/{month}/{day}') ApiKeyFactory(valid_key='e5444-794') session.flush() reports = self.add_reports(celery, 3) self.add_reports(celery, 3, api_key='e5444-794', source='gnss') self.add_reports(celery, 3, api_key='e5444-794', source='fused') self.add_reports(celery, 3, api_key=None) mock_keys = [] with mock_s3(mock_keys): update_incoming.delay().get() assert len(mock_keys) == 4 keys = [] test_export = None for mock_key in mock_keys: assert mock_key.set_contents_from_string.called assert mock_key.content_encoding == 'gzip' assert mock_key.content_type == 'application/json' assert mock_key.key.startswith('backups/') assert mock_key.key.endswith('.json.gz') assert mock_key.close.called keys.append(mock_key.key) if 'test' in mock_key.key: test_export = mock_key # extract second and third path segment from key names groups = [tuple(key.split('/')[1:3]) for key in keys] assert (set(groups) == set([('gnss', 'test'), ('gnss', 'no_key'), ('gnss', 'e5444-794'), ('fused', 'e5444-794')])) # check uploaded content args, kw = test_export.set_contents_from_string.call_args uploaded_data = args[0] uploaded_text = util.decode_gzip(uploaded_data) send_reports = simplejson.loads(uploaded_text)['items'] assert len(send_reports) == 3 expect = [report['position']['accuracy'] for report in reports] gotten = [report['position']['accuracy'] for report in send_reports] assert set(expect) == set(gotten) stats.check(counter=[ ('data.export.batch', 4, 1, ['key:backup']), ('data.export.upload', 4, ['key:backup', 'status:success']), ], timer=[ ('data.export.upload', 4, ['key:backup']), ])
def test_gzip(self): cell, query = self._one_cell_query() data = {'items': [query]} body = util.encode_gzip(dumps(data)) headers = {'Content-Encoding': 'gzip'} res = self.app.post( self.url, body, headers=headers, content_type='application/json', status=self.status) self.assertEqual(res.headers['Access-Control-Allow-Origin'], '*') self.assertEqual(res.headers['Access-Control-Max-Age'], '2592000') update_incoming.delay().get() self._assert_queue_size(1)
def test_upload(self): ApiKeyFactory(valid_key='e5444-794') self.session.flush() reports = self.add_reports(3) self.add_reports(6, api_key='e5444-794') self.add_reports(3, api_key=None) mock_keys = [] with mock_s3(mock_keys): update_incoming.delay().get() self.assertEqual(len(mock_keys), 4) keys = [] test_export = None for mock_key in mock_keys: self.assertTrue(mock_key.set_contents_from_string.called) self.assertEqual(mock_key.content_encoding, 'gzip') self.assertEqual(mock_key.content_type, 'application/json') self.assertTrue(mock_key.key.startswith('backups/')) self.assertTrue(mock_key.key.endswith('.json.gz')) self.assertTrue(mock_key.close.called) keys.append(mock_key.key) if 'test' in mock_key.key: test_export = mock_key # extract second path segment from key names queue_keys = [key.split('/')[1] for key in keys] self.assertEqual(set(queue_keys), set(['test', 'no_key', 'e5444-794'])) # check uploaded content args, kw = test_export.set_contents_from_string.call_args uploaded_data = args[0] uploaded_text = util.decode_gzip(uploaded_data) send_reports = simplejson.loads(uploaded_text)['items'] self.assertEqual(len(send_reports), 3) expect = [report['position']['accuracy'] for report in reports] gotten = [report['position']['accuracy'] for report in send_reports] self.assertEqual(set(expect), set(gotten)) self.check_stats(counter=[ ('data.export.batch', 4, 1, ['key:backup']), ('data.export.upload', 4, ['key:backup', 'status:success']), ], timer=[ ('data.export.upload', 4, ['key:backup']), ])
def _update_all(self, session, datamap_only=False): ExportConfigFactory(name='internal', batch=0, schema='internal') session.flush() update_incoming.delay().get() if datamap_only: return for shard_id in BlueShard.shards().keys(): update_blue.delay(shard_id=shard_id).get() for shard_id in CellShard.shards().keys(): update_cell.delay(shard_id=shard_id).get() for shard_id in WifiShard.shards().keys(): update_wifi.delay(shard_id=shard_id).get()
def _update_all(self, session, datamap_only=False): ExportConfigFactory(name="internal", batch=0, schema="internal") session.flush() update_incoming.delay().get() if datamap_only: return for shard_id in BlueShard.shards().keys(): update_blue.delay(shard_id=shard_id).get() for shard_id in CellShard.shards().keys(): update_cell.delay(shard_id=shard_id).get() for shard_id in WifiShard.shards().keys(): update_wifi.delay(shard_id=shard_id).get()
def test_upload(self, celery, session, stats): ApiKeyFactory(valid_key='e5444-794') ExportConfigFactory( name='test', batch=3, schema='geosubmit', url='http://127.0.0.1:9/v2/geosubmit?key=external') session.flush() reports = [] reports.extend(self.add_reports( celery, 1, source='gnss')) reports.extend(self.add_reports( celery, 1, api_key='e5444e9f-7946')) reports.extend(self.add_reports( celery, 1, api_key=None, source='fused')) with requests_mock.Mocker() as mock: mock.register_uri('POST', requests_mock.ANY, text='{}') update_incoming.delay().get() assert mock.call_count == 1 req = mock.request_history[0] # check headers assert req.headers['Content-Type'] == 'application/json' assert req.headers['Content-Encoding'] == 'gzip' assert req.headers['User-Agent'] == 'ichnaea' body = util.decode_gzip(req.body) send_reports = simplejson.loads(body)['items'] assert len(send_reports) == 3 for field in ('accuracy', 'source', 'timestamp'): expect = [report['position'].get(field) for report in reports] gotten = [report['position'].get(field) for report in send_reports] assert set(expect) == set(gotten) assert ( set([w['ssid'] for w in send_reports[0]['wifiAccessPoints']]) == set(['my-wifi'])) stats.check(counter=[ ('data.export.batch', 1, 1, ['key:test']), ('data.export.upload', 1, ['key:test', 'status:200']), ], timer=[ ('data.export.upload', ['key:test']), ])
def test_upload(self, celery, session, stats): ApiKeyFactory(valid_key='e5444-794') ExportConfigFactory(name='test', batch=3, schema='geosubmit', url='http://127.0.0.1:9/v2/geosubmit?key=external') session.flush() reports = [] reports.extend(self.add_reports(celery, 1, source='gnss')) reports.extend(self.add_reports(celery, 1, api_key='e5444e9f-7946')) reports.extend( self.add_reports(celery, 1, api_key=None, source='fused')) with requests_mock.Mocker() as mock: mock.register_uri('POST', requests_mock.ANY, text='{}') update_incoming.delay().get() assert mock.call_count == 1 req = mock.request_history[0] # check headers assert req.headers['Content-Type'] == 'application/json' assert req.headers['Content-Encoding'] == 'gzip' assert req.headers['User-Agent'] == 'ichnaea' body = util.decode_gzip(req.body) send_reports = simplejson.loads(body)['items'] assert len(send_reports) == 3 for field in ('accuracy', 'source', 'timestamp'): expect = [report['position'].get(field) for report in reports] gotten = [report['position'].get(field) for report in send_reports] assert set(expect) == set(gotten) assert (set([w['ssid'] for w in send_reports[0]['wifiAccessPoints'] ]) == set(['my-wifi'])) stats.check(counter=[ ('data.export.batch', 1, 1, ['key:test']), ('data.export.upload', 1, ['key:test', 'status:200']), ], timer=[ ('data.export.upload', ['key:test']), ])
def test_retry(self): self.add_reports(3) num = [0] orig_wait = DummyExporter._retry_wait def mock_send(self, data, num=num): num[0] += 1 if num[0] == 1: raise IOError() with mock.patch('ichnaea.data.export.DummyExporter.send', mock_send): try: DummyExporter._retry_wait = 0.001 update_incoming.delay().get() finally: DummyExporter._retry_wait = orig_wait self.assertEqual(self.queue_length('queue_export_test'), 0)
def test_retry(self, celery, redis, session): ExportConfigFactory(name="test", batch=1) session.flush() self.add_reports(celery, 1) num = [0] orig_wait = DummyExporter._retry_wait def mock_send(self, data, num=num): num[0] += 1 if num[0] == 1: raise IOError() with mock.patch("ichnaea.data.export.DummyExporter.send", mock_send): try: DummyExporter._retry_wait = 0.001 update_incoming.delay().get() finally: DummyExporter._retry_wait = orig_wait assert self.queue_length(redis, "queue_export_test") == 0
def test_retry(self, celery, redis, session): ExportConfigFactory(name='test', batch=1) session.flush() self.add_reports(celery, 1) num = [0] orig_wait = DummyExporter._retry_wait def mock_send(self, data, num=num): num[0] += 1 if num[0] == 1: raise IOError() with mock.patch('ichnaea.data.export.DummyExporter.send', mock_send): try: DummyExporter._retry_wait = 0.001 update_incoming.delay().get() finally: DummyExporter._retry_wait = orig_wait assert self.queue_length(redis, 'queue_export_test') == 0
def test_upload(self): ApiKeyFactory(valid_key='e5444-794') self.session.flush() reports = [] reports.extend(self.add_reports(1)) reports.extend(self.add_reports(1, api_key='e5444e9f-7946')) reports.extend(self.add_reports(1, api_key=None)) with requests_mock.Mocker() as mock: mock.register_uri('POST', requests_mock.ANY, text='{}') update_incoming.delay().get() self.assertEqual(mock.call_count, 1) req = mock.request_history[0] # check headers self.assertEqual(req.headers['Content-Type'], 'application/json') self.assertEqual(req.headers['Content-Encoding'], 'gzip') self.assertEqual(req.headers['User-Agent'], 'ichnaea') body = util.decode_gzip(req.body) send_reports = simplejson.loads(body)['items'] self.assertEqual(len(send_reports), 3) expect = [report['position']['accuracy'] for report in reports] gotten = [report['position']['accuracy'] for report in send_reports] self.assertEqual(set(expect), set(gotten)) self.assertEqual( set([w['ssid'] for w in send_reports[0]['wifiAccessPoints']]), set(['my-wifi'])) self.check_stats(counter=[ ('data.export.batch', 1, 1, ['key:test']), ('data.export.upload', 1, ['key:test', 'status:200']), ], timer=[ ('data.export.upload', ['key:test']), ])
def test_queues(self, celery, redis, session): ApiKeyFactory(valid_key='test2') ExportConfigFactory(name='test', batch=3, skip_keys=frozenset(['export_source'])) ExportConfigFactory(name='everything', batch=5) ExportConfigFactory(name='no_test', batch=2, skip_keys=frozenset(['test', 'test_1'])) ExportConfigFactory(name='query', batch=2, skip_sources=frozenset(['gnss'])) session.flush() self.add_reports(celery, 4) self.add_reports(celery, 1, api_key='test2') self.add_reports(celery, 2, api_key=None, source='gnss') self.add_reports(celery, 1, api_key='test', source='query') update_incoming.delay().get() for queue_key, num in [ ('queue_export_test', 2), ('queue_export_everything', 3), ('queue_export_no_test', 1), ('queue_export_query', 1)]: assert self.queue_length(redis, queue_key) == num
def test_upload(self, celery, session, metricsmock): ExportConfigFactory( name="backup", batch=3, schema="s3", url="s3://bucket/backups/{source}/{api_key}/{year}/{month}/{day}", ) ApiKeyFactory(valid_key="e5444-794") session.flush() reports = self.add_reports(celery, 3) self.add_reports(celery, 3, api_key="e5444-794", source="gnss") self.add_reports(celery, 3, api_key="e5444-794", source="fused") self.add_reports(celery, 3, api_key=None) mock_conn = mock.MagicMock() mock_bucket = mock.MagicMock() mock_obj = mock.MagicMock() mock_conn.return_value.Bucket.return_value = mock_bucket mock_bucket.Object.return_value = mock_obj with mock.patch.object(boto3, "resource", mock_conn): update_incoming.delay().get() obj_calls = mock_bucket.Object.call_args_list put_calls = mock_obj.put.call_args_list assert len(obj_calls) == 4 assert len(put_calls) == 4 keys = [] test_export = None for obj_call, put_call in zip(obj_calls, put_calls): s3_key = obj_call[0][0] assert s3_key.startswith("backups/") assert s3_key.endswith(".json.gz") assert put_call[1]["Body"] assert put_call[1]["ContentType"] == "application/json" assert put_call[1]["ContentEncoding"] == "gzip" keys.append(s3_key) if "test" in s3_key: test_export = put_call[1]["Body"] # extract second and third path segment from key names groups = [tuple(key.split("/")[1:3]) for key in keys] assert set(groups) == set([ ("gnss", "test"), ("gnss", "no_key"), ("gnss", "e5444-794"), ("fused", "e5444-794"), ]) # check uploaded content uploaded_text = util.decode_gzip(test_export) send_reports = json.loads(uploaded_text)["items"] assert len(send_reports) == 3 expect = [report["position"]["accuracy"] for report in reports] gotten = [report["position"]["accuracy"] for report in send_reports] assert set(expect) == set(gotten) assert (len( metricsmock.filter_records("incr", "data.export.batch", value=1, tags=["key:backup"])) == 4) assert (len( metricsmock.filter_records( "incr", "data.export.upload", value=1, tags=["key:backup", "status:success"], )) == 4) assert (len( metricsmock.filter_records("timing", "data.export.upload.timing", tags=["key:backup"])) == 4)
def test_upload(self, celery, session, stats): ExportConfigFactory( name='backup', batch=3, schema='s3', url='s3://bucket/backups/{source}/{api_key}/{year}/{month}/{day}') ApiKeyFactory(valid_key='e5444-794') session.flush() reports = self.add_reports(celery, 3) self.add_reports(celery, 3, api_key='e5444-794', source='gnss') self.add_reports(celery, 3, api_key='e5444-794', source='fused') self.add_reports(celery, 3, api_key=None) mock_conn = mock.MagicMock() mock_bucket = mock.MagicMock() mock_obj = mock.MagicMock() mock_conn.return_value.Bucket.return_value = mock_bucket mock_bucket.Object.return_value = mock_obj with mock.patch.object(boto3, 'resource', mock_conn): update_incoming.delay().get() obj_calls = mock_bucket.Object.call_args_list put_calls = mock_obj.put.call_args_list assert len(obj_calls) == 4 assert len(put_calls) == 4 keys = [] test_export = None for obj_call, put_call in zip(obj_calls, put_calls): s3_key = obj_call[0][0] assert s3_key.startswith('backups/') assert s3_key.endswith('.json.gz') assert put_call[1]['Body'] assert put_call[1]['ContentType'] == 'application/json' assert put_call[1]['ContentEncoding'] == 'gzip' keys.append(s3_key) if 'test' in s3_key: test_export = put_call[1]['Body'] # extract second and third path segment from key names groups = [tuple(key.split('/')[1:3]) for key in keys] assert (set(groups) == set([('gnss', 'test'), ('gnss', 'no_key'), ('gnss', 'e5444-794'), ('fused', 'e5444-794')])) # check uploaded content uploaded_text = util.decode_gzip(test_export) send_reports = simplejson.loads(uploaded_text)['items'] assert len(send_reports) == 3 expect = [report['position']['accuracy'] for report in reports] gotten = [report['position']['accuracy'] for report in send_reports] assert set(expect) == set(gotten) stats.check(counter=[ ('data.export.batch', 4, 1, ['key:backup']), ('data.export.upload', 4, ['key:backup', 'status:success']), ], timer=[ ('data.export.upload', 4, ['key:backup']), ])