def test_create_batch_simple(self):
# given:
data = [
dict(key=1, value=2),
dict(key=3, value=4),
dict(key=5, value=6),
]
jsonlines = [json.dumps(d) for d in data]
# when:
result = list(create_document_batches(jsonlines, 'key'))
# then:
expected = [[
{
'type': 'add',
'id': 1,
'fields': dict(key=1, value=2)
},
{
'type': 'add',
'id': 3,
'fields': dict(key=3, value=4)
},
{
'type': 'add',
'id': 5,
'fields': dict(key=5, value=6)
},
]]
self.assertEqual(len(expected), len(result))
for expected_batch, resulting_batch in zip(expected, result):
self.assertEquals(expected_batch, json.loads(resulting_batch))
self.assertLessEqual(len(resulting_batch),
CLOUDSEARCH_MAX_BATCH_SIZE)
def test_create_batch_simple(self):
# given:
data = [dict(key=1, value=2), dict(key=3, value=4), dict(key=5, value=6)]
jsonlines = [json.dumps(d) for d in data]
# when:
result = list(create_document_batches(jsonlines, "key"))
# then:
expected = [
[
{"type": "add", "id": 1, "fields": dict(key=1, value=2)},
{"type": "add", "id": 3, "fields": dict(key=3, value=4)},
{"type": "add", "id": 5, "fields": dict(key=5, value=6)},
]
]
self.assertEqual(len(expected), len(result))
for expected_batch, resulting_batch in zip(expected, result):
self.assertEquals(expected_batch, json.loads(resulting_batch))
self.assertLessEqual(len(resulting_batch), CLOUDSEARCH_MAX_BATCH_SIZE)
def test_create_batch_with_size_limit(self):
# given:
data = [dict(key="1", value=2), dict(key="3", value=4), dict(key="5", value=6)]
jsonlines = [json.dumps(d) for d in data]
max_batch_size = 150
# when:
result = list(create_document_batches(jsonlines, "key", max_batch_size=max_batch_size))
# then:
expected = [
[
{"type": "add", "id": "1", "fields": dict(key="1", value=2)},
{"type": "add", "id": "3", "fields": dict(key="3", value=4)},
],
[{"type": "add", "id": "5", "fields": dict(key="5", value=6)}],
]
self.assertEqual(len(expected), len(result))
for expected_batch, resulting_batch in zip(expected, result):
self.assertEquals(expected_batch, json.loads(resulting_batch))
self.assertLessEqual(len(resulting_batch), max_batch_size)
def test_create_batch_with_size_limit(self):
# given:
data = [
dict(key="1", value=2),
dict(key="3", value=4),
dict(key="5", value=6),
]
jsonlines = [json.dumps(d) for d in data]
max_batch_size = 150
# when:
result = list(
create_document_batches(jsonlines,
'key',
max_batch_size=max_batch_size))
# then:
expected = [[
{
'type': 'add',
'id': "1",
'fields': dict(key="1", value=2)
},
{
'type': 'add',
'id': "3",
'fields': dict(key="3", value=4)
},
], [
{
'type': 'add',
'id': "5",
'fields': dict(key="5", value=6)
},
]]
self.assertEqual(len(expected), len(result))
for expected_batch, resulting_batch in zip(expected, result):
self.assertEquals(expected_batch, json.loads(resulting_batch))
self.assertLessEqual(len(resulting_batch), max_batch_size)