async def test_complex(make_async_gen, convert, extract_path):
matcher = (streamson.DepthMatcher("2-2")
| streamson.SimpleMatcher('{"users"}')
) & ~streamson.SimpleMatcher('{"users"}[0]')
buff_handler = BufferHandler(use_path=extract_path)
handler = PythonConverterHandler(
convert, extract_path) + buff_handler if convert else buff_handler
async_out = streamson.extract_async(make_async_gen()(),
[(matcher, handler)], extract_path)
res = []
async for rec in async_out:
res.append(rec)
output = list(Output(e for e in res).generator())
assert len(output) == 1
assert output[0] == (
'{"users"}' if extract_path else None,
b'["john", "carl", "bob"]',
)
convert = convert if convert else (lambda x: x)
assert buff_handler.pop_front() == (
'{"users"}' if extract_path else None,
[e for e in convert(b'["john", "carl", "bob"]')],
)
assert buff_handler.pop_front() is None
def test_all(io_reader, data, kind, convert, extract_path):
matcher = streamson.SimpleMatcher('{"users"}[]') & streamson.SimpleMatcher(
"{}[1]")
buff_handler = BufferHandler(use_path=extract_path)
handler = PythonConverterHandler(
convert, extract_path) + buff_handler if convert else buff_handler
if kind == Kind.ITER:
extracted = streamson.extract_iter((e for e in data),
[(matcher, handler)], extract_path)
elif kind == Kind.FD:
extracted = streamson.extract_fd(io_reader, [(matcher, handler)], 5,
extract_path)
output = Output(extracted).generator()
assert next(output) == ('{"users"}[1]' if extract_path else None,
b'"carl"')
with pytest.raises(StopIteration):
next(output)
convert = convert if convert else (lambda x: x)
assert buff_handler.pop_front() == (
'{"users"}[1]' if extract_path else None,
[e for e in convert(b'"carl"')],
)
assert buff_handler.pop_front() is None
async def test_all(make_async_gen, convert, extract_path):
matcher = streamson.SimpleMatcher('{"users"}[]') & streamson.SimpleMatcher(
"{}[1]")
buff_handler = BufferHandler(use_path=extract_path)
handler = PythonConverterHandler(
convert, extract_path) + buff_handler if convert else buff_handler
async_out = streamson.extract_async(make_async_gen()(),
[(matcher, handler)], extract_path)
res = []
async for rec in async_out:
res.append(rec)
output = list(Output(e for e in res).generator())
assert len(output) == 1
assert output[0] == ('{"users"}[1]' if extract_path else None, b'"carl"')
convert = convert if convert else (lambda x: x)
assert buff_handler.pop_front() == (
'{"users"}[1]' if extract_path else None,
[e for e in convert(b'"carl"')],
)
assert buff_handler.pop_front() is None
def test_simple(io_reader, data, kind, convert):
matcher = streamson.SimpleMatcher('{"users"}[]')
buff_handler = BufferHandler()
handler = PythonConverterHandler(
convert) + buff_handler if convert else buff_handler
if kind == Kind.ITER:
filtered = streamson.filter_iter((e for e in data),
[(matcher, handler)])
elif kind == Kind.FD:
filtered = streamson.filter_fd(io_reader, [(matcher, handler)], 5)
output = Output(filtered).generator()
assert next(output) == (None,
b'{"users": [], "groups": ["admins", "users"]}')
with pytest.raises(StopIteration):
next(output)
convert = convert if convert else (lambda x: x)
assert buff_handler.pop_front() == ('{"users"}[0]',
[e for e in convert(b'"john"')])
assert buff_handler.pop_front() == ('{"users"}[1]',
[e for e in convert(b'"carl"')])
assert buff_handler.pop_front() == ('{"users"}[2]',
[e for e in convert(b'"bob"')])
assert buff_handler.pop_front() is None
async def test_simple(make_async_gen, replace_handler):
matcher = streamson.SimpleMatcher('{"users"}[1]')
output_data = b""
async for e in streamson.convert_async(make_async_gen()(), [(matcher, replace_handler)]):
if e is not None and e[1] is not None:
output_data += e[1]
assert output_data == b'{"users": ["john", "***", "bob"]}'
def test_nested(io_reader, data, kind, extract_path):
handler = BufferHandler(use_path=extract_path)
matcher = streamson.SimpleMatcher('{"users"}') | streamson.SimpleMatcher(
'{"users"}[0]')
output_data = b""
if kind == Kind.ITER:
for e in streamson.trigger_iter((e for e in data),
[(matcher, handler)]):
output_data += e
elif kind == Kind.FD:
for e in streamson.trigger_fd(io_reader, [(matcher, handler)], 5):
output_data += e
assert output_data == b'{"users": ["john", "carl", "bob"], "groups": ["admins", "users"]}'
assert handler.pop_front() == ('{"users"}[0]' if extract_path else None,
[e for e in b'"john"'])
assert handler.pop_front() == ('{"users"}' if extract_path else None,
[e for e in b'["john", "carl", "bob"]'])
assert handler.pop_front() is None
def test_complex(io_reader, data, kind, convert, extract_path):
matcher = (streamson.DepthMatcher("2-2")
| streamson.SimpleMatcher('{"users"}')
) & ~streamson.SimpleMatcher('{"groups"}[0]')
buff_handler = BufferHandler(use_path=extract_path)
handler = PythonConverterHandler(
convert, extract_path) + buff_handler if convert else buff_handler
if kind == Kind.ITER:
extracted = streamson.extract_iter((e for e in data),
[(matcher, handler)], extract_path)
elif kind == Kind.FD:
extracted = streamson.extract_fd(io_reader, [(matcher, handler)],
55555, extract_path)
output = Output(extracted).generator()
assert next(output) == (
'{"users"}' if extract_path else None,
b'["john", "carl", "bob"]',
)
assert next(output) == (
'{"groups"}[1]' if extract_path else None,
b'"users"',
)
with pytest.raises(StopIteration):
next(output)
convert = convert if convert else (lambda x: x)
assert buff_handler.pop_front() == (
'{"users"}' if extract_path else None,
[e for e in convert(b'["john", "carl", "bob"]')],
)
assert buff_handler.pop_front() == (
'{"groups"}[1]' if extract_path else None,
[e for e in convert(b'"users"')],
)
assert buff_handler.pop_front() is None
async def test_simple(make_async_gen, buffer_handler):
matcher = streamson.SimpleMatcher('{"users"}[]')
async_out = streamson.filter_async(make_async_gen()(),
[(matcher, buffer_handler)])
res = []
async for rec in async_out:
res.append(rec)
output = list(Output(e for e in res).generator())
assert len(output) == 1
assert output[0] == (None, b'{"users": []}')
def test_simple(io_reader, data, replace_handler, kind):
matcher = streamson.SimpleMatcher('{"users"}[1]')
output_data = b""
if kind == Kind.ITER:
for e in streamson.convert_iter((e for e in data),
[(matcher, replace_handler)]):
if e is not None and e[1] is not None:
output_data += e[1]
elif kind == Kind.FD:
for e in streamson.convert_fd(io_reader, [(matcher, replace_handler)],
5):
if e is not None and e[1] is not None:
output_data += e[1]
assert output_data == b'{"users": ["john", "***", "bob"], "groups": ["admins", "users"]}'
async def test_simple(make_async_gen, extract_path):
handler = BufferHandler(use_path=extract_path)
matcher = streamson.SimpleMatcher('{"users"}[]')
async_out = streamson.trigger_async(make_async_gen()(),
[(matcher, handler)])
output_data = b""
async for rec in async_out:
output_data += rec
assert output_data == b'{"users": ["john", "carl", "bob"]}'
assert handler.pop_front() == ('{"users"}[0]' if extract_path else None,
[e for e in b'"john"'])
assert handler.pop_front() == ('{"users"}[1]' if extract_path else None,
[e for e in b'"carl"'])
assert handler.pop_front() == ('{"users"}[2]' if extract_path else None,
[e for e in b'"bob"'])