def test_queues(self, celery, redis, session):
ApiKeyFactory(valid_key="test2")
ExportConfigFactory(name="test",
batch=3,
skip_keys=frozenset(["export_source"]))
ExportConfigFactory(name="everything", batch=5)
ExportConfigFactory(name="no_test",
batch=2,
skip_keys=frozenset(["test", "test_1"]))
ExportConfigFactory(name="query",
batch=2,
skip_sources=frozenset(["gnss"]))
session.flush()
self.add_reports(celery, 4)
self.add_reports(celery, 1, api_key="test2")
self.add_reports(celery, 2, api_key=None, source="gnss")
self.add_reports(celery, 1, api_key="test", source="query")
update_incoming.delay().get()
for queue_key, num in [
("queue_export_test", 2),
("queue_export_everything", 3),
("queue_export_no_test", 1),
("queue_export_query", 1),
]:
assert self.queue_length(redis, queue_key) == num
def test_queues(self, celery, redis, session):
ApiKeyFactory(valid_key='test2')
ExportConfigFactory(name='test',
batch=3,
skip_keys=frozenset(['export_source']))
ExportConfigFactory(name='everything', batch=5)
ExportConfigFactory(name='no_test',
batch=2,
skip_keys=frozenset(['test', 'test_1']))
ExportConfigFactory(name='query',
batch=2,
skip_sources=frozenset(['gnss']))
session.flush()
self.add_reports(celery, 4)
self.add_reports(celery, 1, api_key='test2')
self.add_reports(celery, 2, api_key=None, source='gnss')
self.add_reports(celery, 1, api_key='test', source='query')
update_incoming.delay().get()
for queue_key, num in [('queue_export_test', 2),
('queue_export_everything', 3),
('queue_export_no_test', 1),
('queue_export_query', 1)]:
assert self.queue_length(redis, queue_key) == num
def test_null_position(self, celery, redis, session):
"""Reports with null position are queued."""
ApiKeyFactory(valid_key="no-position")
ExportConfigFactory(name="everything", batch=5)
session.flush()
self.add_reports(celery, 1, api_key="no-position", set_position=False)
update_incoming.delay().get()
assert self.queue_length(redis, "queue_export_everything") == 1
def test_upload(self, celery, session, metricsmock):
ApiKeyFactory(valid_key="e5444-7946")
ExportConfigFactory(
name="test",
batch=4,
schema="geosubmit",
url="http://127.0.0.1:9/v2/geosubmit?key=external",
)
session.flush()
reports = []
reports.extend(self.add_reports(celery, 1, source="gnss"))
reports.extend(self.add_reports(celery, 1, api_key="e5444e9f-7946"))
reports.extend(
self.add_reports(celery, 1, api_key=None, source="fused"))
reports.extend(self.add_reports(celery, 1, set_position=False))
with requests_mock.Mocker() as mock:
mock.register_uri("POST", requests_mock.ANY, text="{}")
update_incoming.delay().get()
assert mock.call_count == 1
req = mock.request_history[0]
# check headers
assert req.headers["Content-Type"] == "application/json"
assert req.headers["Content-Encoding"] == "gzip"
assert req.headers["User-Agent"] == "ichnaea"
body = util.decode_gzip(req.body)
send_reports = json.loads(body)["items"]
assert len(send_reports) == 4
for field in ("accuracy", "source", "timestamp"):
expect = [(report["position"] or {}).get(field)
for report in reports]
gotten = [(report["position"] or {}).get(field)
for report in send_reports]
assert set(expect) == set(gotten)
assert set([w["ssid"] for w in send_reports[0]["wifiAccessPoints"]
]) == set(["my-wifi"])
assert metricsmock.has_record("incr",
"data.export.batch",
value=1,
tags=["key:test"])
assert metricsmock.has_record("incr",
"data.export.upload",
value=1,
tags=["key:test", "status:200"])
assert metricsmock.has_record("timing",
"data.export.upload.timing",
tags=["key:test"])
def _update_all(self, session, datamap_only=False):
ExportConfigFactory(name="internal", batch=0, schema="internal")
session.flush()
update_incoming.delay().get()
if datamap_only:
return
for shard_id in BlueShard.shards().keys():
update_blue.delay(shard_id=shard_id).get()
for shard_id in CellShard.shards().keys():
update_cell.delay(shard_id=shard_id).get()
for shard_id in WifiShard.shards().keys():
update_wifi.delay(shard_id=shard_id).get()
def test_upload(self, celery, session, stats):
ApiKeyFactory(valid_key='e5444-794')
ExportConfigFactory(name='test',
batch=3,
schema='geosubmit',
url='http://127.0.0.1:9/v2/geosubmit?key=external')
session.flush()
reports = []
reports.extend(self.add_reports(celery, 1, source='gnss'))
reports.extend(self.add_reports(celery, 1, api_key='e5444e9f-7946'))
reports.extend(
self.add_reports(celery, 1, api_key=None, source='fused'))
with requests_mock.Mocker() as mock:
mock.register_uri('POST', requests_mock.ANY, text='{}')
update_incoming.delay().get()
assert mock.call_count == 1
req = mock.request_history[0]
# check headers
assert req.headers['Content-Type'] == 'application/json'
assert req.headers['Content-Encoding'] == 'gzip'
assert req.headers['User-Agent'] == 'ichnaea'
body = util.decode_gzip(req.body)
send_reports = simplejson.loads(body)['items']
assert len(send_reports) == 3
for field in ('accuracy', 'source', 'timestamp'):
expect = [report['position'].get(field) for report in reports]
gotten = [report['position'].get(field) for report in send_reports]
assert set(expect) == set(gotten)
assert (set([w['ssid'] for w in send_reports[0]['wifiAccessPoints']
]) == set(['my-wifi']))
stats.check(counter=[
('data.export.batch', 1, 1, ['key:test']),
('data.export.upload', 1, ['key:test', 'status:200']),
],
timer=[
('data.export.upload', ['key:test']),
])
def test_retry(self, celery, redis, session):
ExportConfigFactory(name="test", batch=1)
session.flush()
self.add_reports(celery, 1)
num = [0]
orig_wait = DummyExporter._retry_wait
def mock_send(self, data, num=num):
num[0] += 1
if num[0] == 1:
raise IOError()
with mock.patch("ichnaea.data.export.DummyExporter.send", mock_send):
try:
DummyExporter._retry_wait = 0.001
update_incoming.delay().get()
finally:
DummyExporter._retry_wait = orig_wait
assert self.queue_length(redis, "queue_export_test") == 0
def test_upload(self, celery, session, metricsmock):
ExportConfigFactory(
name="backup",
batch=3,
schema="s3",
url="s3://bucket/backups/{source}/{api_key}/{year}/{month}/{day}",
)
ApiKeyFactory(valid_key="e5444-794")
session.flush()
reports = self.add_reports(celery, 3)
self.add_reports(celery, 3, api_key="e5444-794", source="gnss")
self.add_reports(celery, 3, api_key="e5444-794", source="fused")
self.add_reports(celery, 3, api_key=None)
mock_conn = mock.MagicMock()
mock_bucket = mock.MagicMock()
mock_obj = mock.MagicMock()
mock_conn.return_value.Bucket.return_value = mock_bucket
mock_bucket.Object.return_value = mock_obj
with mock.patch.object(boto3, "resource", mock_conn):
update_incoming.delay().get()
obj_calls = mock_bucket.Object.call_args_list
put_calls = mock_obj.put.call_args_list
assert len(obj_calls) == 4
assert len(put_calls) == 4
keys = []
test_export = None
for obj_call, put_call in zip(obj_calls, put_calls):
s3_key = obj_call[0][0]
assert s3_key.startswith("backups/")
assert s3_key.endswith(".json.gz")
assert put_call[1]["Body"]
assert put_call[1]["ContentType"] == "application/json"
assert put_call[1]["ContentEncoding"] == "gzip"
keys.append(s3_key)
if "test" in s3_key:
test_export = put_call[1]["Body"]
# extract second and third path segment from key names
groups = [tuple(key.split("/")[1:3]) for key in keys]
assert set(groups) == set([
("gnss", "test"),
("gnss", "no_key"),
("gnss", "e5444-794"),
("fused", "e5444-794"),
])
# check uploaded content
uploaded_text = util.decode_gzip(test_export)
send_reports = json.loads(uploaded_text)["items"]
assert len(send_reports) == 3
expect = [report["position"]["accuracy"] for report in reports]
gotten = [report["position"]["accuracy"] for report in send_reports]
assert set(expect) == set(gotten)
assert (len(
metricsmock.filter_records("incr",
"data.export.batch",
value=1,
tags=["key:backup"])) == 4)
assert (len(
metricsmock.filter_records(
"incr",
"data.export.upload",
value=1,
tags=["key:backup", "status:success"],
)) == 4)
assert (len(
metricsmock.filter_records("timing",
"data.export.upload.timing",
tags=["key:backup"])) == 4)
def test_upload(self, celery, session, stats):
ExportConfigFactory(
name='backup',
batch=3,
schema='s3',
url='s3://bucket/backups/{source}/{api_key}/{year}/{month}/{day}')
ApiKeyFactory(valid_key='e5444-794')
session.flush()
reports = self.add_reports(celery, 3)
self.add_reports(celery, 3, api_key='e5444-794', source='gnss')
self.add_reports(celery, 3, api_key='e5444-794', source='fused')
self.add_reports(celery, 3, api_key=None)
mock_conn = mock.MagicMock()
mock_bucket = mock.MagicMock()
mock_obj = mock.MagicMock()
mock_conn.return_value.Bucket.return_value = mock_bucket
mock_bucket.Object.return_value = mock_obj
with mock.patch.object(boto3, 'resource', mock_conn):
update_incoming.delay().get()
obj_calls = mock_bucket.Object.call_args_list
put_calls = mock_obj.put.call_args_list
assert len(obj_calls) == 4
assert len(put_calls) == 4
keys = []
test_export = None
for obj_call, put_call in zip(obj_calls, put_calls):
s3_key = obj_call[0][0]
assert s3_key.startswith('backups/')
assert s3_key.endswith('.json.gz')
assert put_call[1]['Body']
assert put_call[1]['ContentType'] == 'application/json'
assert put_call[1]['ContentEncoding'] == 'gzip'
keys.append(s3_key)
if 'test' in s3_key:
test_export = put_call[1]['Body']
# extract second and third path segment from key names
groups = [tuple(key.split('/')[1:3]) for key in keys]
assert (set(groups) == set([('gnss', 'test'), ('gnss', 'no_key'),
('gnss', 'e5444-794'),
('fused', 'e5444-794')]))
# check uploaded content
uploaded_text = util.decode_gzip(test_export)
send_reports = simplejson.loads(uploaded_text)['items']
assert len(send_reports) == 3
expect = [report['position']['accuracy'] for report in reports]
gotten = [report['position']['accuracy'] for report in send_reports]
assert set(expect) == set(gotten)
stats.check(counter=[
('data.export.batch', 4, 1, ['key:backup']),
('data.export.upload', 4, ['key:backup', 'status:success']),
],
timer=[
('data.export.upload', 4, ['key:backup']),
])