def test_load_object_get_persistent(self):
json = '{"a": 1, "b": null, "c": true}'
# Access in order
data = load(StringIO(json), persistent=True)
self.assertEqual(data['a'], 1)
self.assertEqual(data['b'], None)
self.assertEqual(data['c'], True)
with self.assertRaises(KeyError):
_ = data['d']
# Access out of order
data = load(StringIO(json), persistent=True)
self.assertEqual(data['b'], None)
self.assertEqual(data['a'], 1)
self.assertEqual(data['c'], True)
with self.assertRaises(KeyError):
_ = data['d']
# Access with key error first order
data = load(StringIO(json), persistent=True)
with self.assertRaises(KeyError):
_ = data['d']
self.assertEqual(data['a'], 1)
self.assertEqual(data['b'], None)
self.assertEqual(data['c'], True)
def test_load_object_get_transient(self):
json = '{"a": 1, "b": null, "c": true}'
# Access in order
data = load(StringIO(json), persistent=False)
self.assertEqual(data['a'], 1)
self.assertEqual(data['b'], None)
self.assertEqual(data['c'], True)
with self.assertRaises(TransientAccessException):
_ = data['d']
# Access out of order
data = load(StringIO(json), persistent=False)
self.assertEqual(data['b'], None)
with self.assertRaises(TransientAccessException):
_ = data['a']
with self.assertRaises(TransientAccessException):
_ = data['c'] # stream was exhausted in search for 'a'
with self.assertRaises(TransientAccessException):
_ = data[
'd'] # don't know if this was a key error or was in the past
# Access with key error first order
data = load(StringIO(json), persistent=False)
with self.assertRaises(KeyError):
_ = data['d']
with self.assertRaises(TransientAccessException):
_ = data['a'] # stream was exhausted in search for 'd'
def test_not_copiable(self):
json = '[["a"], "b", "c"]'
with self.assertRaisesRegex(
copy.Error,
"^Copying json_steam objects leads to a bad time$"):
copy.copy(load(StringIO(json)))
with self.assertRaisesRegex(
copy.Error,
"^Copying json_steam objects leads to a bad time$"):
copy.deepcopy(load(StringIO(json)))
def _to_data(self, obj, persistent, binary=False):
data = json.dumps(obj)
if binary:
stream = BytesIO(data.encode())
else:
stream = StringIO(data)
return load(stream, persistent)
def test_load_nested_transient(self):
json = '{"count": 3, "results": ["a", "b", "c"]}'
data = load(StringIO(json), persistent=False)
self.assertIsInstance(data, TransientStreamingJSONObject)
results = data['results']
self.assertIsInstance(results, TransientStreamingJSONList)
self.assertEqual(list(results), ['a', 'b', 'c'])
def test_load_list_get_persistent(self):
json = '[1, true, ""]'
# Access in order
data = load(StringIO(json), persistent=True)
self.assertEqual(data[0], 1)
self.assertTrue(data[1])
self.assertEqual(data[2], "")
with self.assertRaises(IndexError):
_ = data[3]
# Access out of order
data = load(StringIO(json), persistent=True)
self.assertEqual(data[0], 1)
self.assertTrue(data[1])
self.assertEqual(data[2], "")
with self.assertRaises(IndexError):
_ = data[3]
def test_load_nested_transient_first_list_item_list(self):
json = '[["a"], "b", "c"]'
data = load(StringIO(json), persistent=False)
self.assertIsInstance(data, TransientStreamingJSONList)
items = iter(data)
item = next(items)
self.assertIsInstance(item, TransientStreamingJSONList)
self.assertListEqual(["a"], list(item))
self.assertEqual(list(items), ['b', 'c'])
def test_load_nested_transient_first_list_item_object(self):
json = '[{"a": 4}, "b", "c"]'
data = load(StringIO(json), persistent=False)
self.assertIsInstance(data, TransientStreamingJSONList)
items = iter(data)
item = next(items)
self.assertIsInstance(item, TransientStreamingJSONObject)
self.assertDictEqual({"a": 4}, dict(item.items()))
self.assertEqual(list(items), ['b', 'c'])
def test_persistent_to_transient(self):
json = """{"a": 1, "x": ["long", "list", "I", "don't", "want", "in", "memory"], "b": 2}"""
data = load(StringIO(json), persistent=True).transient()
self.assertIsInstance(data, PersistentStreamingJSONObject)
self.assertEqual(data["a"], 1)
l = data["x"]
self.assertIsInstance(l, TransientStreamingJSONList)
self.assertEqual(data["b"], 2)
self.assertEqual(data["b"], 2) # would error if data was transient
with self.assertRaisesRegex(TransientAccessException,
"Index 0 already passed in this stream"):
_ = l[0] # cannot access transient list
def test_load_nested_persistent(self):
json = '{"count": 3, "results": ["a", "b", {}]}'
data = load(StringIO(json), persistent=True)
self.assertIsInstance(data, PersistentStreamingJSONObject)
results = data['results']
self.assertIsInstance(results, PersistentStreamingJSONList)
self.assertEqual(results[0], 'a')
self.assertEqual(results[1], 'b')
self.assertIsInstance(results[2], PersistentStreamingJSONObject)
self.assertEqual(len(results), 3)
self.assertEqual(len(results[2]), 0)
self.assertEqual(len(data), 2)
self.assertEqual(data["count"], 3)
def test_transient_to_persistent(self):
json = '{"results": [{"x": 1, "y": 3}, {"y": 4, "x": 2}]}'
xs = iter((1, 2))
ys = iter((3, 4))
data = load(StringIO(json)) # data is a transient dict-like object
self.assertIsInstance(data, TransientStreamingJSONObject)
results = data['results']
self.assertIsInstance(results, TransientStreamingJSONList)
# iterate transient list, but produce persistent items
for result in results.persistent():
# result is a persistent dict-like object
self.assertIsInstance(result, PersistentStreamingJSONObject)
x = next(xs)
y = next(ys)
self.assertEqual(result['x'], x)
self.assertEqual(
result['y'],
y) # would error on second result without .persistent()
self.assertEqual(result['x'],
x) # would error without .persistent()
def load(response, persistent=False):
return json_stream.load(_to_file(response), persistent=persistent)
def test_dump_context(self):
data = json_stream.load(StringIO(self.JSON), persistent=True)
with JSONStreamEncoder():
output = json.dumps(data)
self._assert_json_okay(output)
def test_dump_cls(self):
data = json_stream.load(StringIO(self.JSON), persistent=True)
output = json.dumps(data, cls=JSONStreamEncoder)
self._assert_json_okay(output)
def test_dump_default(self):
data = json_stream.load(StringIO(self.JSON), persistent=True)
output = json.dumps(data, default=default)
self._assert_json_okay(output)
