def add_reports(self, num=1, blue_factor=0, cell_factor=1, wifi_factor=2,
api_key='test', nickname=None,
blue_key=None, cell_mcc=None, wifi_key=None,
lat=None, lon=None):
reports = []
for i in range(num):
pos = CellShardFactory.build()
report = {
'timestamp': time.time() * 1000.0,
'position': {},
'bluetoothBeacons': [],
'cellTowers': [],
'wifiAccessPoints': [],
}
report['position']['latitude'] = lat or pos.lat
report['position']['longitude'] = lon or pos.lon
report['position']['accuracy'] = 17.0 + i
blues = BlueShardFactory.build_batch(blue_factor,
lat=pos.lat, lon=pos.lon)
for blue in blues:
blue_data = {
'macAddress': blue_key or blue.mac,
'signalStrength': -100 + i,
}
report['bluetoothBeacons'].append(blue_data)
cells = CellShardFactory.build_batch(cell_factor,
lat=pos.lat, lon=pos.lon)
for cell in cells:
cell_data = {
'radioType': cell.radio.name,
'mobileCountryCode': cell_mcc or cell.mcc,
'mobileNetworkCode': cell.mnc,
'locationAreaCode': cell.lac,
'cellId': cell.cid,
'primaryScramblingCode': cell.psc,
'signalStrength': -110 + i,
}
report['cellTowers'].append(cell_data)
wifis = WifiShardFactory.build_batch(wifi_factor,
lat=pos.lat, lon=pos.lon)
for wifi in wifis:
wifi_data = {
'macAddress': wifi_key or wifi.mac,
'signalStrength': -90 + i,
'ssid': 'my-wifi',
}
report['wifiAccessPoints'].append(wifi_data)
reports.append(report)
items = [{'api_key': api_key,
'nickname': nickname,
'report': rep} for rep in reports]
self.incoming_queue.enqueue(items)
update_incoming.delay().get()
return reports
def test_upload(self, celery, session, stats):
ExportConfigFactory(
name='backup', batch=3, schema='s3',
url='s3://bucket/backups/{source}/{api_key}/{year}/{month}/{day}')
ApiKeyFactory(valid_key='e5444-794')
session.flush()
reports = self.add_reports(celery, 3)
self.add_reports(celery, 3, api_key='e5444-794', source='gnss')
self.add_reports(celery, 3, api_key='e5444-794', source='fused')
self.add_reports(celery, 3, api_key=None)
mock_conn = mock.MagicMock()
mock_bucket = mock.MagicMock()
mock_obj = mock.MagicMock()
mock_conn.return_value.Bucket.return_value = mock_bucket
mock_bucket.Object.return_value = mock_obj
with mock.patch.object(boto3, 'resource', mock_conn):
update_incoming.delay().get()
obj_calls = mock_bucket.Object.call_args_list
put_calls = mock_obj.put.call_args_list
assert len(obj_calls) == 4
assert len(put_calls) == 4
keys = []
test_export = None
for obj_call, put_call in zip(obj_calls, put_calls):
s3_key = obj_call[0][0]
assert s3_key.startswith('backups/')
assert s3_key.endswith('.json.gz')
assert put_call[1]['Body']
assert put_call[1]['ContentType'] == 'application/json'
assert put_call[1]['ContentEncoding'] == 'gzip'
keys.append(s3_key)
if 'test' in s3_key:
test_export = put_call[1]['Body']
# extract second and third path segment from key names
groups = [tuple(key.split('/')[1:3]) for key in keys]
assert (set(groups) ==
set([('gnss', 'test'), ('gnss', 'no_key'),
('gnss', 'e5444-794'), ('fused', 'e5444-794')]))
# check uploaded content
uploaded_text = util.decode_gzip(test_export)
send_reports = simplejson.loads(uploaded_text)['items']
assert len(send_reports) == 3
expect = [report['position']['accuracy'] for report in reports]
gotten = [report['position']['accuracy'] for report in send_reports]
assert set(expect) == set(gotten)
stats.check(counter=[
('data.export.batch', 4, 1, ['key:backup']),
('data.export.upload', 4, ['key:backup', 'status:success']),
], timer=[
('data.export.upload', 4, ['key:backup']),
])
def test_queues(self, celery, redis, session):
ApiKeyFactory(valid_key="test2")
ExportConfigFactory(name="test",
batch=3,
skip_keys=frozenset(["export_source"]))
ExportConfigFactory(name="everything", batch=5)
ExportConfigFactory(name="no_test",
batch=2,
skip_keys=frozenset(["test", "test_1"]))
ExportConfigFactory(name="query",
batch=2,
skip_sources=frozenset(["gnss"]))
session.flush()
self.add_reports(celery, 4)
self.add_reports(celery, 1, api_key="test2")
self.add_reports(celery, 2, api_key=None, source="gnss")
self.add_reports(celery, 1, api_key="test", source="query")
update_incoming.delay().get()
for queue_key, num in [
("queue_export_test", 2),
("queue_export_everything", 3),
("queue_export_no_test", 1),
("queue_export_query", 1),
]:
assert self.queue_length(redis, queue_key) == num
def test_queues(self, celery, redis, session):
ApiKeyFactory(valid_key='test2')
ExportConfigFactory(name='test',
batch=3,
skip_keys=frozenset(['export_source']))
ExportConfigFactory(name='everything', batch=5)
ExportConfigFactory(name='no_test',
batch=2,
skip_keys=frozenset(['test', 'test_1']))
ExportConfigFactory(name='query',
batch=2,
skip_sources=frozenset(['gnss']))
session.flush()
self.add_reports(celery, 4)
self.add_reports(celery, 1, api_key='test2')
self.add_reports(celery, 2, api_key=None, source='gnss')
self.add_reports(celery, 1, api_key='test', source='query')
update_incoming.delay().get()
for queue_key, num in [('queue_export_test', 2),
('queue_export_everything', 3),
('queue_export_no_test', 1),
('queue_export_query', 1)]:
assert self.queue_length(redis, queue_key) == num
def test_datamap(self):
self.add_reports(1, cell_factor=0, wifi_factor=2, lat=50.0, lon=10.0)
self.add_reports(2, cell_factor=0, wifi_factor=2, lat=20.0, lon=-10.0)
update_incoming.delay().get()
self.assertEqual(
self.celery_app.data_queues['update_datamap_ne'].size(), 1)
self.assertEqual(
self.celery_app.data_queues['update_datamap_sw'].size(), 1)
def test_null_position(self, celery, redis, session):
"""Reports with null position are queued."""
ApiKeyFactory(valid_key="no-position")
ExportConfigFactory(name="everything", batch=5)
session.flush()
self.add_reports(celery, 1, api_key="no-position", set_position=False)
update_incoming.delay().get()
assert self.queue_length(redis, "queue_export_everything") == 1
def _update_all(self):
update_incoming.delay().get()
for shard_id in BlueShard.shards().keys():
update_blue.delay(shard_id=shard_id).get()
for shard_id in CellShard.shards().keys():
update_cell.delay(shard_id=shard_id).get()
for shard_id in WifiShard.shards().keys():
update_wifi.delay(shard_id=shard_id).get()
def test_upload(self, celery, session, metricsmock):
ApiKeyFactory(valid_key="e5444-7946")
ExportConfigFactory(
name="test",
batch=4,
schema="geosubmit",
url="http://127.0.0.1:9/v2/geosubmit?key=external",
)
session.flush()
reports = []
reports.extend(self.add_reports(celery, 1, source="gnss"))
reports.extend(self.add_reports(celery, 1, api_key="e5444e9f-7946"))
reports.extend(
self.add_reports(celery, 1, api_key=None, source="fused"))
reports.extend(self.add_reports(celery, 1, set_position=False))
with requests_mock.Mocker() as mock:
mock.register_uri("POST", requests_mock.ANY, text="{}")
update_incoming.delay().get()
assert mock.call_count == 1
req = mock.request_history[0]
# check headers
assert req.headers["Content-Type"] == "application/json"
assert req.headers["Content-Encoding"] == "gzip"
assert req.headers["User-Agent"] == "ichnaea"
body = util.decode_gzip(req.body)
send_reports = json.loads(body)["items"]
assert len(send_reports) == 4
for field in ("accuracy", "source", "timestamp"):
expect = [(report["position"] or {}).get(field)
for report in reports]
gotten = [(report["position"] or {}).get(field)
for report in send_reports]
assert set(expect) == set(gotten)
assert set([w["ssid"] for w in send_reports[0]["wifiAccessPoints"]
]) == set(["my-wifi"])
assert metricsmock.has_record("incr",
"data.export.batch",
value=1,
tags=["key:test"])
assert metricsmock.has_record("incr",
"data.export.upload",
value=1,
tags=["key:test", "status:200"])
assert metricsmock.has_record("timing",
"data.export.upload.timing",
tags=["key:test"])
def test_queues(self):
self.add_reports(4)
self.add_reports(1, api_key='test2')
self.add_reports(2, api_key=None)
update_incoming.delay().get()
for queue_key, num in [
('queue_export_test', 1),
('queue_export_everything', 2),
('queue_export_no_test', 1)]:
self.assertEqual(self.queue_length(queue_key), num)
def _post(self, items, api_key=None, status=status, **kw):
url = self.url
if api_key:
url += '?key=%s' % api_key
extra = {'HTTP_X_FORWARDED_FOR': self.geoip_data['London']['ip']}
result = self.app.post_json(
url, {'items': items},
status=status, extra_environ=extra, **kw)
while self.data_queue.size() > 0:
update_incoming.delay().get()
return result
def test_upload(self, celery, session, stats):
ExportConfigFactory(
name='backup', batch=3, schema='s3',
url='s3://bucket/backups/{source}/{api_key}/{year}/{month}/{day}')
ApiKeyFactory(valid_key='e5444-794')
session.flush()
reports = self.add_reports(celery, 3)
self.add_reports(celery, 3, api_key='e5444-794', source='gnss')
self.add_reports(celery, 3, api_key='e5444-794', source='fused')
self.add_reports(celery, 3, api_key=None)
mock_keys = []
with mock_s3(mock_keys):
update_incoming.delay().get()
assert len(mock_keys) == 4
keys = []
test_export = None
for mock_key in mock_keys:
assert mock_key.set_contents_from_string.called
assert mock_key.content_encoding == 'gzip'
assert mock_key.content_type == 'application/json'
assert mock_key.key.startswith('backups/')
assert mock_key.key.endswith('.json.gz')
assert mock_key.close.called
keys.append(mock_key.key)
if 'test' in mock_key.key:
test_export = mock_key
# extract second and third path segment from key names
groups = [tuple(key.split('/')[1:3]) for key in keys]
assert (set(groups) ==
set([('gnss', 'test'), ('gnss', 'no_key'),
('gnss', 'e5444-794'), ('fused', 'e5444-794')]))
# check uploaded content
args, kw = test_export.set_contents_from_string.call_args
uploaded_data = args[0]
uploaded_text = util.decode_gzip(uploaded_data)
send_reports = simplejson.loads(uploaded_text)['items']
assert len(send_reports) == 3
expect = [report['position']['accuracy'] for report in reports]
gotten = [report['position']['accuracy'] for report in send_reports]
assert set(expect) == set(gotten)
stats.check(counter=[
('data.export.batch', 4, 1, ['key:backup']),
('data.export.upload', 4, ['key:backup', 'status:success']),
], timer=[
('data.export.upload', 4, ['key:backup']),
])
def test_gzip(self):
cell, query = self._one_cell_query()
data = {'items': [query]}
body = util.encode_gzip(dumps(data))
headers = {'Content-Encoding': 'gzip'}
res = self.app.post(
self.url, body, headers=headers,
content_type='application/json', status=self.status)
self.assertEqual(res.headers['Access-Control-Allow-Origin'], '*')
self.assertEqual(res.headers['Access-Control-Max-Age'], '2592000')
update_incoming.delay().get()
self._assert_queue_size(1)
def test_upload(self):
ApiKeyFactory(valid_key='e5444-794')
self.session.flush()
reports = self.add_reports(3)
self.add_reports(6, api_key='e5444-794')
self.add_reports(3, api_key=None)
mock_keys = []
with mock_s3(mock_keys):
update_incoming.delay().get()
self.assertEqual(len(mock_keys), 4)
keys = []
test_export = None
for mock_key in mock_keys:
self.assertTrue(mock_key.set_contents_from_string.called)
self.assertEqual(mock_key.content_encoding, 'gzip')
self.assertEqual(mock_key.content_type, 'application/json')
self.assertTrue(mock_key.key.startswith('backups/'))
self.assertTrue(mock_key.key.endswith('.json.gz'))
self.assertTrue(mock_key.close.called)
keys.append(mock_key.key)
if 'test' in mock_key.key:
test_export = mock_key
# extract second path segment from key names
queue_keys = [key.split('/')[1] for key in keys]
self.assertEqual(set(queue_keys), set(['test', 'no_key', 'e5444-794']))
# check uploaded content
args, kw = test_export.set_contents_from_string.call_args
uploaded_data = args[0]
uploaded_text = util.decode_gzip(uploaded_data)
send_reports = simplejson.loads(uploaded_text)['items']
self.assertEqual(len(send_reports), 3)
expect = [report['position']['accuracy'] for report in reports]
gotten = [report['position']['accuracy'] for report in send_reports]
self.assertEqual(set(expect), set(gotten))
self.check_stats(counter=[
('data.export.batch', 4, 1, ['key:backup']),
('data.export.upload', 4, ['key:backup', 'status:success']),
], timer=[
('data.export.upload', 4, ['key:backup']),
])
def _update_all(self, session, datamap_only=False):
ExportConfigFactory(name='internal', batch=0, schema='internal')
session.flush()
update_incoming.delay().get()
if datamap_only:
return
for shard_id in BlueShard.shards().keys():
update_blue.delay(shard_id=shard_id).get()
for shard_id in CellShard.shards().keys():
update_cell.delay(shard_id=shard_id).get()
for shard_id in WifiShard.shards().keys():
update_wifi.delay(shard_id=shard_id).get()
def _update_all(self, session, datamap_only=False):
ExportConfigFactory(name="internal", batch=0, schema="internal")
session.flush()
update_incoming.delay().get()
if datamap_only:
return
for shard_id in BlueShard.shards().keys():
update_blue.delay(shard_id=shard_id).get()
for shard_id in CellShard.shards().keys():
update_cell.delay(shard_id=shard_id).get()
for shard_id in WifiShard.shards().keys():
update_wifi.delay(shard_id=shard_id).get()
def test_upload(self, celery, session, stats):
ApiKeyFactory(valid_key='e5444-794')
ExportConfigFactory(
name='test', batch=3, schema='geosubmit',
url='http://127.0.0.1:9/v2/geosubmit?key=external')
session.flush()
reports = []
reports.extend(self.add_reports(
celery, 1, source='gnss'))
reports.extend(self.add_reports(
celery, 1, api_key='e5444e9f-7946'))
reports.extend(self.add_reports(
celery, 1, api_key=None, source='fused'))
with requests_mock.Mocker() as mock:
mock.register_uri('POST', requests_mock.ANY, text='{}')
update_incoming.delay().get()
assert mock.call_count == 1
req = mock.request_history[0]
# check headers
assert req.headers['Content-Type'] == 'application/json'
assert req.headers['Content-Encoding'] == 'gzip'
assert req.headers['User-Agent'] == 'ichnaea'
body = util.decode_gzip(req.body)
send_reports = simplejson.loads(body)['items']
assert len(send_reports) == 3
for field in ('accuracy', 'source', 'timestamp'):
expect = [report['position'].get(field) for report in reports]
gotten = [report['position'].get(field) for report in send_reports]
assert set(expect) == set(gotten)
assert (
set([w['ssid'] for w in send_reports[0]['wifiAccessPoints']]) ==
set(['my-wifi']))
stats.check(counter=[
('data.export.batch', 1, 1, ['key:test']),
('data.export.upload', 1, ['key:test', 'status:200']),
], timer=[
('data.export.upload', ['key:test']),
])
def test_upload(self, celery, session, stats):
ApiKeyFactory(valid_key='e5444-794')
ExportConfigFactory(name='test',
batch=3,
schema='geosubmit',
url='http://127.0.0.1:9/v2/geosubmit?key=external')
session.flush()
reports = []
reports.extend(self.add_reports(celery, 1, source='gnss'))
reports.extend(self.add_reports(celery, 1, api_key='e5444e9f-7946'))
reports.extend(
self.add_reports(celery, 1, api_key=None, source='fused'))
with requests_mock.Mocker() as mock:
mock.register_uri('POST', requests_mock.ANY, text='{}')
update_incoming.delay().get()
assert mock.call_count == 1
req = mock.request_history[0]
# check headers
assert req.headers['Content-Type'] == 'application/json'
assert req.headers['Content-Encoding'] == 'gzip'
assert req.headers['User-Agent'] == 'ichnaea'
body = util.decode_gzip(req.body)
send_reports = simplejson.loads(body)['items']
assert len(send_reports) == 3
for field in ('accuracy', 'source', 'timestamp'):
expect = [report['position'].get(field) for report in reports]
gotten = [report['position'].get(field) for report in send_reports]
assert set(expect) == set(gotten)
assert (set([w['ssid'] for w in send_reports[0]['wifiAccessPoints']
]) == set(['my-wifi']))
stats.check(counter=[
('data.export.batch', 1, 1, ['key:test']),
('data.export.upload', 1, ['key:test', 'status:200']),
],
timer=[
('data.export.upload', ['key:test']),
])
def test_retry(self):
self.add_reports(3)
num = [0]
orig_wait = DummyExporter._retry_wait
def mock_send(self, data, num=num):
num[0] += 1
if num[0] == 1:
raise IOError()
with mock.patch('ichnaea.data.export.DummyExporter.send', mock_send):
try:
DummyExporter._retry_wait = 0.001
update_incoming.delay().get()
finally:
DummyExporter._retry_wait = orig_wait
self.assertEqual(self.queue_length('queue_export_test'), 0)
def test_retry(self, celery, redis, session):
ExportConfigFactory(name="test", batch=1)
session.flush()
self.add_reports(celery, 1)
num = [0]
orig_wait = DummyExporter._retry_wait
def mock_send(self, data, num=num):
num[0] += 1
if num[0] == 1:
raise IOError()
with mock.patch("ichnaea.data.export.DummyExporter.send", mock_send):
try:
DummyExporter._retry_wait = 0.001
update_incoming.delay().get()
finally:
DummyExporter._retry_wait = orig_wait
assert self.queue_length(redis, "queue_export_test") == 0
def test_retry(self, celery, redis, session):
ExportConfigFactory(name='test', batch=1)
session.flush()
self.add_reports(celery, 1)
num = [0]
orig_wait = DummyExporter._retry_wait
def mock_send(self, data, num=num):
num[0] += 1
if num[0] == 1:
raise IOError()
with mock.patch('ichnaea.data.export.DummyExporter.send', mock_send):
try:
DummyExporter._retry_wait = 0.001
update_incoming.delay().get()
finally:
DummyExporter._retry_wait = orig_wait
assert self.queue_length(redis, 'queue_export_test') == 0
def test_upload(self):
ApiKeyFactory(valid_key='e5444-794')
self.session.flush()
reports = []
reports.extend(self.add_reports(1))
reports.extend(self.add_reports(1, api_key='e5444e9f-7946'))
reports.extend(self.add_reports(1, api_key=None))
with requests_mock.Mocker() as mock:
mock.register_uri('POST', requests_mock.ANY, text='{}')
update_incoming.delay().get()
self.assertEqual(mock.call_count, 1)
req = mock.request_history[0]
# check headers
self.assertEqual(req.headers['Content-Type'], 'application/json')
self.assertEqual(req.headers['Content-Encoding'], 'gzip')
self.assertEqual(req.headers['User-Agent'], 'ichnaea')
body = util.decode_gzip(req.body)
send_reports = simplejson.loads(body)['items']
self.assertEqual(len(send_reports), 3)
expect = [report['position']['accuracy'] for report in reports]
gotten = [report['position']['accuracy'] for report in send_reports]
self.assertEqual(set(expect), set(gotten))
self.assertEqual(
set([w['ssid'] for w in send_reports[0]['wifiAccessPoints']]),
set(['my-wifi']))
self.check_stats(counter=[
('data.export.batch', 1, 1, ['key:test']),
('data.export.upload', 1, ['key:test', 'status:200']),
], timer=[
('data.export.upload', ['key:test']),
])
def test_queues(self, celery, redis, session):
ApiKeyFactory(valid_key='test2')
ExportConfigFactory(name='test', batch=3,
skip_keys=frozenset(['export_source']))
ExportConfigFactory(name='everything', batch=5)
ExportConfigFactory(name='no_test', batch=2,
skip_keys=frozenset(['test', 'test_1']))
ExportConfigFactory(name='query', batch=2,
skip_sources=frozenset(['gnss']))
session.flush()
self.add_reports(celery, 4)
self.add_reports(celery, 1, api_key='test2')
self.add_reports(celery, 2, api_key=None, source='gnss')
self.add_reports(celery, 1, api_key='test', source='query')
update_incoming.delay().get()
for queue_key, num in [
('queue_export_test', 2),
('queue_export_everything', 3),
('queue_export_no_test', 1),
('queue_export_query', 1)]:
assert self.queue_length(redis, queue_key) == num
def test_upload(self, celery, session, metricsmock):
ExportConfigFactory(
name="backup",
batch=3,
schema="s3",
url="s3://bucket/backups/{source}/{api_key}/{year}/{month}/{day}",
)
ApiKeyFactory(valid_key="e5444-794")
session.flush()
reports = self.add_reports(celery, 3)
self.add_reports(celery, 3, api_key="e5444-794", source="gnss")
self.add_reports(celery, 3, api_key="e5444-794", source="fused")
self.add_reports(celery, 3, api_key=None)
mock_conn = mock.MagicMock()
mock_bucket = mock.MagicMock()
mock_obj = mock.MagicMock()
mock_conn.return_value.Bucket.return_value = mock_bucket
mock_bucket.Object.return_value = mock_obj
with mock.patch.object(boto3, "resource", mock_conn):
update_incoming.delay().get()
obj_calls = mock_bucket.Object.call_args_list
put_calls = mock_obj.put.call_args_list
assert len(obj_calls) == 4
assert len(put_calls) == 4
keys = []
test_export = None
for obj_call, put_call in zip(obj_calls, put_calls):
s3_key = obj_call[0][0]
assert s3_key.startswith("backups/")
assert s3_key.endswith(".json.gz")
assert put_call[1]["Body"]
assert put_call[1]["ContentType"] == "application/json"
assert put_call[1]["ContentEncoding"] == "gzip"
keys.append(s3_key)
if "test" in s3_key:
test_export = put_call[1]["Body"]
# extract second and third path segment from key names
groups = [tuple(key.split("/")[1:3]) for key in keys]
assert set(groups) == set([
("gnss", "test"),
("gnss", "no_key"),
("gnss", "e5444-794"),
("fused", "e5444-794"),
])
# check uploaded content
uploaded_text = util.decode_gzip(test_export)
send_reports = json.loads(uploaded_text)["items"]
assert len(send_reports) == 3
expect = [report["position"]["accuracy"] for report in reports]
gotten = [report["position"]["accuracy"] for report in send_reports]
assert set(expect) == set(gotten)
assert (len(
metricsmock.filter_records("incr",
"data.export.batch",
value=1,
tags=["key:backup"])) == 4)
assert (len(
metricsmock.filter_records(
"incr",
"data.export.upload",
value=1,
tags=["key:backup", "status:success"],
)) == 4)
assert (len(
metricsmock.filter_records("timing",
"data.export.upload.timing",
tags=["key:backup"])) == 4)
def test_upload(self, celery, session, stats):
ExportConfigFactory(
name='backup',
batch=3,
schema='s3',
url='s3://bucket/backups/{source}/{api_key}/{year}/{month}/{day}')
ApiKeyFactory(valid_key='e5444-794')
session.flush()
reports = self.add_reports(celery, 3)
self.add_reports(celery, 3, api_key='e5444-794', source='gnss')
self.add_reports(celery, 3, api_key='e5444-794', source='fused')
self.add_reports(celery, 3, api_key=None)
mock_conn = mock.MagicMock()
mock_bucket = mock.MagicMock()
mock_obj = mock.MagicMock()
mock_conn.return_value.Bucket.return_value = mock_bucket
mock_bucket.Object.return_value = mock_obj
with mock.patch.object(boto3, 'resource', mock_conn):
update_incoming.delay().get()
obj_calls = mock_bucket.Object.call_args_list
put_calls = mock_obj.put.call_args_list
assert len(obj_calls) == 4
assert len(put_calls) == 4
keys = []
test_export = None
for obj_call, put_call in zip(obj_calls, put_calls):
s3_key = obj_call[0][0]
assert s3_key.startswith('backups/')
assert s3_key.endswith('.json.gz')
assert put_call[1]['Body']
assert put_call[1]['ContentType'] == 'application/json'
assert put_call[1]['ContentEncoding'] == 'gzip'
keys.append(s3_key)
if 'test' in s3_key:
test_export = put_call[1]['Body']
# extract second and third path segment from key names
groups = [tuple(key.split('/')[1:3]) for key in keys]
assert (set(groups) == set([('gnss', 'test'), ('gnss', 'no_key'),
('gnss', 'e5444-794'),
('fused', 'e5444-794')]))
# check uploaded content
uploaded_text = util.decode_gzip(test_export)
send_reports = simplejson.loads(uploaded_text)['items']
assert len(send_reports) == 3
expect = [report['position']['accuracy'] for report in reports]
gotten = [report['position']['accuracy'] for report in send_reports]
assert set(expect) == set(gotten)
stats.check(counter=[
('data.export.batch', 4, 1, ['key:backup']),
('data.export.upload', 4, ['key:backup', 'status:success']),
],
timer=[
('data.export.upload', 4, ['key:backup']),
])
