def test_create_batch_simple(self): # given: data = [ dict(key=1, value=2), dict(key=3, value=4), dict(key=5, value=6), ] jsonlines = [json.dumps(d) for d in data] # when: result = list(create_document_batches(jsonlines, 'key')) # then: expected = [[ { 'type': 'add', 'id': 1, 'fields': dict(key=1, value=2) }, { 'type': 'add', 'id': 3, 'fields': dict(key=3, value=4) }, { 'type': 'add', 'id': 5, 'fields': dict(key=5, value=6) }, ]] self.assertEqual(len(expected), len(result)) for expected_batch, resulting_batch in zip(expected, result): self.assertEquals(expected_batch, json.loads(resulting_batch)) self.assertLessEqual(len(resulting_batch), CLOUDSEARCH_MAX_BATCH_SIZE)
def test_create_batch_simple(self): # given: data = [dict(key=1, value=2), dict(key=3, value=4), dict(key=5, value=6)] jsonlines = [json.dumps(d) for d in data] # when: result = list(create_document_batches(jsonlines, "key")) # then: expected = [ [ {"type": "add", "id": 1, "fields": dict(key=1, value=2)}, {"type": "add", "id": 3, "fields": dict(key=3, value=4)}, {"type": "add", "id": 5, "fields": dict(key=5, value=6)}, ] ] self.assertEqual(len(expected), len(result)) for expected_batch, resulting_batch in zip(expected, result): self.assertEquals(expected_batch, json.loads(resulting_batch)) self.assertLessEqual(len(resulting_batch), CLOUDSEARCH_MAX_BATCH_SIZE)
def test_create_batch_with_size_limit(self): # given: data = [dict(key="1", value=2), dict(key="3", value=4), dict(key="5", value=6)] jsonlines = [json.dumps(d) for d in data] max_batch_size = 150 # when: result = list(create_document_batches(jsonlines, "key", max_batch_size=max_batch_size)) # then: expected = [ [ {"type": "add", "id": "1", "fields": dict(key="1", value=2)}, {"type": "add", "id": "3", "fields": dict(key="3", value=4)}, ], [{"type": "add", "id": "5", "fields": dict(key="5", value=6)}], ] self.assertEqual(len(expected), len(result)) for expected_batch, resulting_batch in zip(expected, result): self.assertEquals(expected_batch, json.loads(resulting_batch)) self.assertLessEqual(len(resulting_batch), max_batch_size)
def test_create_batch_with_size_limit(self): # given: data = [ dict(key="1", value=2), dict(key="3", value=4), dict(key="5", value=6), ] jsonlines = [json.dumps(d) for d in data] max_batch_size = 150 # when: result = list( create_document_batches(jsonlines, 'key', max_batch_size=max_batch_size)) # then: expected = [[ { 'type': 'add', 'id': "1", 'fields': dict(key="1", value=2) }, { 'type': 'add', 'id': "3", 'fields': dict(key="3", value=4) }, ], [ { 'type': 'add', 'id': "5", 'fields': dict(key="5", value=6) }, ]] self.assertEqual(len(expected), len(result)) for expected_batch, resulting_batch in zip(expected, result): self.assertEquals(expected_batch, json.loads(resulting_batch)) self.assertLessEqual(len(resulting_batch), max_batch_size)