コード例 #1
0
ファイル: test_export.py プロジェクト: vamsikarnika/ichnaea
    def test_queues(self, celery, redis, session):
        ApiKeyFactory(valid_key="test2")
        ExportConfigFactory(name="test",
                            batch=3,
                            skip_keys=frozenset(["export_source"]))
        ExportConfigFactory(name="everything", batch=5)
        ExportConfigFactory(name="no_test",
                            batch=2,
                            skip_keys=frozenset(["test", "test_1"]))
        ExportConfigFactory(name="query",
                            batch=2,
                            skip_sources=frozenset(["gnss"]))
        session.flush()

        self.add_reports(celery, 4)
        self.add_reports(celery, 1, api_key="test2")
        self.add_reports(celery, 2, api_key=None, source="gnss")
        self.add_reports(celery, 1, api_key="test", source="query")
        update_incoming.delay().get()

        for queue_key, num in [
            ("queue_export_test", 2),
            ("queue_export_everything", 3),
            ("queue_export_no_test", 1),
            ("queue_export_query", 1),
        ]:
            assert self.queue_length(redis, queue_key) == num
コード例 #2
0
    def test_queues(self, celery, redis, session):
        ApiKeyFactory(valid_key='test2')
        ExportConfigFactory(name='test',
                            batch=3,
                            skip_keys=frozenset(['export_source']))
        ExportConfigFactory(name='everything', batch=5)
        ExportConfigFactory(name='no_test',
                            batch=2,
                            skip_keys=frozenset(['test', 'test_1']))
        ExportConfigFactory(name='query',
                            batch=2,
                            skip_sources=frozenset(['gnss']))
        session.flush()

        self.add_reports(celery, 4)
        self.add_reports(celery, 1, api_key='test2')
        self.add_reports(celery, 2, api_key=None, source='gnss')
        self.add_reports(celery, 1, api_key='test', source='query')
        update_incoming.delay().get()

        for queue_key, num in [('queue_export_test', 2),
                               ('queue_export_everything', 3),
                               ('queue_export_no_test', 1),
                               ('queue_export_query', 1)]:
            assert self.queue_length(redis, queue_key) == num
コード例 #3
0
    def test_null_position(self, celery, redis, session):
        """Reports with null position are queued."""
        ApiKeyFactory(valid_key="no-position")
        ExportConfigFactory(name="everything", batch=5)
        session.flush()

        self.add_reports(celery, 1, api_key="no-position", set_position=False)
        update_incoming.delay().get()
        assert self.queue_length(redis, "queue_export_everything") == 1
コード例 #4
0
    def test_upload(self, celery, session, metricsmock):
        ApiKeyFactory(valid_key="e5444-7946")
        ExportConfigFactory(
            name="test",
            batch=4,
            schema="geosubmit",
            url="http://127.0.0.1:9/v2/geosubmit?key=external",
        )
        session.flush()

        reports = []
        reports.extend(self.add_reports(celery, 1, source="gnss"))
        reports.extend(self.add_reports(celery, 1, api_key="e5444e9f-7946"))
        reports.extend(
            self.add_reports(celery, 1, api_key=None, source="fused"))
        reports.extend(self.add_reports(celery, 1, set_position=False))

        with requests_mock.Mocker() as mock:
            mock.register_uri("POST", requests_mock.ANY, text="{}")
            update_incoming.delay().get()

        assert mock.call_count == 1
        req = mock.request_history[0]

        # check headers
        assert req.headers["Content-Type"] == "application/json"
        assert req.headers["Content-Encoding"] == "gzip"
        assert req.headers["User-Agent"] == "ichnaea"

        body = util.decode_gzip(req.body)
        send_reports = json.loads(body)["items"]
        assert len(send_reports) == 4

        for field in ("accuracy", "source", "timestamp"):
            expect = [(report["position"] or {}).get(field)
                      for report in reports]
            gotten = [(report["position"] or {}).get(field)
                      for report in send_reports]
            assert set(expect) == set(gotten)

        assert set([w["ssid"] for w in send_reports[0]["wifiAccessPoints"]
                    ]) == set(["my-wifi"])

        assert metricsmock.has_record("incr",
                                      "data.export.batch",
                                      value=1,
                                      tags=["key:test"])
        assert metricsmock.has_record("incr",
                                      "data.export.upload",
                                      value=1,
                                      tags=["key:test", "status:200"])
        assert metricsmock.has_record("timing",
                                      "data.export.upload.timing",
                                      tags=["key:test"])
コード例 #5
0
ファイル: test_export.py プロジェクト: vamsikarnika/ichnaea
    def _update_all(self, session, datamap_only=False):
        ExportConfigFactory(name="internal", batch=0, schema="internal")
        session.flush()
        update_incoming.delay().get()

        if datamap_only:
            return

        for shard_id in BlueShard.shards().keys():
            update_blue.delay(shard_id=shard_id).get()

        for shard_id in CellShard.shards().keys():
            update_cell.delay(shard_id=shard_id).get()

        for shard_id in WifiShard.shards().keys():
            update_wifi.delay(shard_id=shard_id).get()
コード例 #6
0
    def test_upload(self, celery, session, stats):
        ApiKeyFactory(valid_key='e5444-794')
        ExportConfigFactory(name='test',
                            batch=3,
                            schema='geosubmit',
                            url='http://127.0.0.1:9/v2/geosubmit?key=external')
        session.flush()

        reports = []
        reports.extend(self.add_reports(celery, 1, source='gnss'))
        reports.extend(self.add_reports(celery, 1, api_key='e5444e9f-7946'))
        reports.extend(
            self.add_reports(celery, 1, api_key=None, source='fused'))

        with requests_mock.Mocker() as mock:
            mock.register_uri('POST', requests_mock.ANY, text='{}')
            update_incoming.delay().get()

        assert mock.call_count == 1
        req = mock.request_history[0]

        # check headers
        assert req.headers['Content-Type'] == 'application/json'
        assert req.headers['Content-Encoding'] == 'gzip'
        assert req.headers['User-Agent'] == 'ichnaea'

        body = util.decode_gzip(req.body)
        send_reports = simplejson.loads(body)['items']
        assert len(send_reports) == 3

        for field in ('accuracy', 'source', 'timestamp'):
            expect = [report['position'].get(field) for report in reports]
            gotten = [report['position'].get(field) for report in send_reports]
            assert set(expect) == set(gotten)

        assert (set([w['ssid'] for w in send_reports[0]['wifiAccessPoints']
                     ]) == set(['my-wifi']))

        stats.check(counter=[
            ('data.export.batch', 1, 1, ['key:test']),
            ('data.export.upload', 1, ['key:test', 'status:200']),
        ],
                    timer=[
                        ('data.export.upload', ['key:test']),
                    ])
コード例 #7
0
ファイル: test_export.py プロジェクト: vamsikarnika/ichnaea
    def test_retry(self, celery, redis, session):
        ExportConfigFactory(name="test", batch=1)
        session.flush()
        self.add_reports(celery, 1)

        num = [0]
        orig_wait = DummyExporter._retry_wait

        def mock_send(self, data, num=num):
            num[0] += 1
            if num[0] == 1:
                raise IOError()

        with mock.patch("ichnaea.data.export.DummyExporter.send", mock_send):
            try:
                DummyExporter._retry_wait = 0.001
                update_incoming.delay().get()
            finally:
                DummyExporter._retry_wait = orig_wait

        assert self.queue_length(redis, "queue_export_test") == 0
コード例 #8
0
ファイル: test_export.py プロジェクト: vamsikarnika/ichnaea
    def test_upload(self, celery, session, metricsmock):
        ExportConfigFactory(
            name="backup",
            batch=3,
            schema="s3",
            url="s3://bucket/backups/{source}/{api_key}/{year}/{month}/{day}",
        )
        ApiKeyFactory(valid_key="e5444-794")
        session.flush()

        reports = self.add_reports(celery, 3)
        self.add_reports(celery, 3, api_key="e5444-794", source="gnss")
        self.add_reports(celery, 3, api_key="e5444-794", source="fused")
        self.add_reports(celery, 3, api_key=None)

        mock_conn = mock.MagicMock()
        mock_bucket = mock.MagicMock()
        mock_obj = mock.MagicMock()
        mock_conn.return_value.Bucket.return_value = mock_bucket
        mock_bucket.Object.return_value = mock_obj

        with mock.patch.object(boto3, "resource", mock_conn):
            update_incoming.delay().get()

        obj_calls = mock_bucket.Object.call_args_list
        put_calls = mock_obj.put.call_args_list
        assert len(obj_calls) == 4
        assert len(put_calls) == 4

        keys = []
        test_export = None
        for obj_call, put_call in zip(obj_calls, put_calls):
            s3_key = obj_call[0][0]
            assert s3_key.startswith("backups/")
            assert s3_key.endswith(".json.gz")
            assert put_call[1]["Body"]
            assert put_call[1]["ContentType"] == "application/json"
            assert put_call[1]["ContentEncoding"] == "gzip"
            keys.append(s3_key)
            if "test" in s3_key:
                test_export = put_call[1]["Body"]

        # extract second and third path segment from key names
        groups = [tuple(key.split("/")[1:3]) for key in keys]
        assert set(groups) == set([
            ("gnss", "test"),
            ("gnss", "no_key"),
            ("gnss", "e5444-794"),
            ("fused", "e5444-794"),
        ])

        # check uploaded content
        uploaded_text = util.decode_gzip(test_export)

        send_reports = json.loads(uploaded_text)["items"]
        assert len(send_reports) == 3
        expect = [report["position"]["accuracy"] for report in reports]
        gotten = [report["position"]["accuracy"] for report in send_reports]
        assert set(expect) == set(gotten)

        assert (len(
            metricsmock.filter_records("incr",
                                       "data.export.batch",
                                       value=1,
                                       tags=["key:backup"])) == 4)
        assert (len(
            metricsmock.filter_records(
                "incr",
                "data.export.upload",
                value=1,
                tags=["key:backup", "status:success"],
            )) == 4)
        assert (len(
            metricsmock.filter_records("timing",
                                       "data.export.upload.timing",
                                       tags=["key:backup"])) == 4)
コード例 #9
0
    def test_upload(self, celery, session, stats):
        ExportConfigFactory(
            name='backup',
            batch=3,
            schema='s3',
            url='s3://bucket/backups/{source}/{api_key}/{year}/{month}/{day}')
        ApiKeyFactory(valid_key='e5444-794')
        session.flush()

        reports = self.add_reports(celery, 3)
        self.add_reports(celery, 3, api_key='e5444-794', source='gnss')
        self.add_reports(celery, 3, api_key='e5444-794', source='fused')
        self.add_reports(celery, 3, api_key=None)

        mock_conn = mock.MagicMock()
        mock_bucket = mock.MagicMock()
        mock_obj = mock.MagicMock()
        mock_conn.return_value.Bucket.return_value = mock_bucket
        mock_bucket.Object.return_value = mock_obj

        with mock.patch.object(boto3, 'resource', mock_conn):
            update_incoming.delay().get()

        obj_calls = mock_bucket.Object.call_args_list
        put_calls = mock_obj.put.call_args_list
        assert len(obj_calls) == 4
        assert len(put_calls) == 4

        keys = []
        test_export = None
        for obj_call, put_call in zip(obj_calls, put_calls):
            s3_key = obj_call[0][0]
            assert s3_key.startswith('backups/')
            assert s3_key.endswith('.json.gz')
            assert put_call[1]['Body']
            assert put_call[1]['ContentType'] == 'application/json'
            assert put_call[1]['ContentEncoding'] == 'gzip'
            keys.append(s3_key)
            if 'test' in s3_key:
                test_export = put_call[1]['Body']

        # extract second and third path segment from key names
        groups = [tuple(key.split('/')[1:3]) for key in keys]
        assert (set(groups) == set([('gnss', 'test'), ('gnss', 'no_key'),
                                    ('gnss', 'e5444-794'),
                                    ('fused', 'e5444-794')]))

        # check uploaded content
        uploaded_text = util.decode_gzip(test_export)

        send_reports = simplejson.loads(uploaded_text)['items']
        assert len(send_reports) == 3
        expect = [report['position']['accuracy'] for report in reports]
        gotten = [report['position']['accuracy'] for report in send_reports]
        assert set(expect) == set(gotten)

        stats.check(counter=[
            ('data.export.batch', 4, 1, ['key:backup']),
            ('data.export.upload', 4, ['key:backup', 'status:success']),
        ],
                    timer=[
                        ('data.export.upload', 4, ['key:backup']),
                    ])