Esempio n. 1
0
    def test_create_batch_simple(self):
        # given:
        data = [
            dict(key=1, value=2),
            dict(key=3, value=4),
            dict(key=5, value=6),
        ]
        jsonlines = [json.dumps(d) for d in data]

        # when:
        result = list(create_document_batches(jsonlines, 'key'))

        # then:
        expected = [[
            {
                'type': 'add',
                'id': 1,
                'fields': dict(key=1, value=2)
            },
            {
                'type': 'add',
                'id': 3,
                'fields': dict(key=3, value=4)
            },
            {
                'type': 'add',
                'id': 5,
                'fields': dict(key=5, value=6)
            },
        ]]
        self.assertEqual(len(expected), len(result))
        for expected_batch, resulting_batch in zip(expected, result):
            self.assertEquals(expected_batch, json.loads(resulting_batch))
            self.assertLessEqual(len(resulting_batch),
                                 CLOUDSEARCH_MAX_BATCH_SIZE)
    def test_create_batch_simple(self):
        # given:
        data = [dict(key=1, value=2), dict(key=3, value=4), dict(key=5, value=6)]
        jsonlines = [json.dumps(d) for d in data]

        # when:
        result = list(create_document_batches(jsonlines, "key"))

        # then:
        expected = [
            [
                {"type": "add", "id": 1, "fields": dict(key=1, value=2)},
                {"type": "add", "id": 3, "fields": dict(key=3, value=4)},
                {"type": "add", "id": 5, "fields": dict(key=5, value=6)},
            ]
        ]
        self.assertEqual(len(expected), len(result))
        for expected_batch, resulting_batch in zip(expected, result):
            self.assertEquals(expected_batch, json.loads(resulting_batch))
            self.assertLessEqual(len(resulting_batch), CLOUDSEARCH_MAX_BATCH_SIZE)
    def test_create_batch_with_size_limit(self):
        # given:
        data = [dict(key="1", value=2), dict(key="3", value=4), dict(key="5", value=6)]
        jsonlines = [json.dumps(d) for d in data]
        max_batch_size = 150

        # when:
        result = list(create_document_batches(jsonlines, "key", max_batch_size=max_batch_size))

        # then:
        expected = [
            [
                {"type": "add", "id": "1", "fields": dict(key="1", value=2)},
                {"type": "add", "id": "3", "fields": dict(key="3", value=4)},
            ],
            [{"type": "add", "id": "5", "fields": dict(key="5", value=6)}],
        ]
        self.assertEqual(len(expected), len(result))
        for expected_batch, resulting_batch in zip(expected, result):
            self.assertEquals(expected_batch, json.loads(resulting_batch))
            self.assertLessEqual(len(resulting_batch), max_batch_size)
Esempio n. 4
0
    def test_create_batch_with_size_limit(self):
        # given:
        data = [
            dict(key="1", value=2),
            dict(key="3", value=4),
            dict(key="5", value=6),
        ]
        jsonlines = [json.dumps(d) for d in data]
        max_batch_size = 150

        # when:
        result = list(
            create_document_batches(jsonlines,
                                    'key',
                                    max_batch_size=max_batch_size))

        # then:
        expected = [[
            {
                'type': 'add',
                'id': "1",
                'fields': dict(key="1", value=2)
            },
            {
                'type': 'add',
                'id': "3",
                'fields': dict(key="3", value=4)
            },
        ], [
            {
                'type': 'add',
                'id': "5",
                'fields': dict(key="5", value=6)
            },
        ]]
        self.assertEqual(len(expected), len(result))
        for expected_batch, resulting_batch in zip(expected, result):
            self.assertEquals(expected_batch, json.loads(resulting_batch))
            self.assertLessEqual(len(resulting_batch), max_batch_size)