def test_command_parsing_ok(self, executor, ndjson_bytes):
"""Sanity test that hits most elements of parsing."""
BulkAPI.from_byte_stream(ndjson_bytes,
executor=executor,
observer=Observer())
executor.configure.assert_called_with(
config=Any.instance_of(Configuration))
executor.execute_batch.assert_has_calls([
call(
command_type=CommandType.UPSERT,
data_type=DataType.USER,
batch=[Any.instance_of(DataCommand)],
default_config={},
),
call(
command_type=CommandType.UPSERT,
data_type=DataType.GROUP,
batch=[Any.instance_of(DataCommand)],
default_config={},
),
call(
command_type=CommandType.CREATE,
data_type=DataType.GROUP_MEMBERSHIP,
batch=[Any.instance_of(DataCommand)],
default_config={"on_duplicate": "continue"},
),
])
def test_from_string(self, nd_json, executor, CommandProcessor):
BulkAPI.from_string(nd_json, executor)
CommandProcessor.assert_called_once_with(
executor=executor, observer=Any.instance_of(Observer))
self._assert_process_called_with_generator_of_commands(
CommandProcessor)
def test_to_stream(self, commands):
handle = StringIO()
expected = deepcopy([command.raw for command in commands])
BulkAPI.to_stream(handle, commands)
assert self._decode_ndjson(handle.getvalue()) == expected
def test_round_tripping(self, commands, collecting_observer):
"""Check that sending and decoding results in the same data."""
original_raw = [command.raw for command in commands]
BulkAPI.from_byte_stream(
BytesIO(BulkAPI.to_string(commands).encode("utf-8")),
executor=AutomaticReportExecutor(),
observer=collecting_observer,
)
final_raw = [command.raw for command in collecting_observer.commands]
assert original_raw == final_raw
def test__bytes_to_lines(self):
bytes = BytesIO(b"\nline_1\n\nlong_middle_line_2\nline_3")
lines = BulkAPI._bytes_to_lines(bytes, chunk_size=8)
assert isinstance(lines, GeneratorType)
assert list(lines) == [b"line_1", b"long_middle_line_2", b"line_3"]
def test_command_serialisation_ok(self, commands):
"""A sanity check that hits most of the behavior of creation.
This is a happy path check. We expect this to work."""
ndjson = BulkAPI.to_string(commands)
lines = ndjson.strip().split("\n")
command_data = [json.loads(line) for line in lines]
assert len(command_data) == 4
assert command_data == [
["configure", Any.dict()],
["upsert", {
"data": Any.dict.containing({"type": "user"})
}],
["upsert", {
"data": Any.dict.containing({"type": "group"})
}],
[
"create", {
"data": Any.dict.containing({"type": "group_membership"})
}
],
]
def bulk(request):
"""
Perform a bulk request which can modify multiple records in on go.
This end-point can:
* Upsert users
* Upsert groups
* Add users to groups
This end-point is intended to be called using the classes provided by
`h.h_api.bulk_api`.
"""
results = BulkAPI.from_byte_stream(request.body_file,
executor=BulkExecutor(request.db))
if results is None:
return Response(status=204)
# When we get an iterator we must force the first return value to be
# created to be sure input validation has occurred. Otherwise we might
# raise errors outside of the view when called
try:
results = chain([next(results)], results)
except StopIteration:
results = []
# An NDJSON response is required
return Response(
app_iter=((json.dumps(result) + "\n").encode("utf-8")
for result in results),
status=200,
content_type="application/x-ndjson",
)
def test_we_catch_json_parsing_errors(self, config_command, executor):
bad_string = json.dumps(config_command.raw) + '\n["Nonsense'
with pytest.raises(InvalidJSONError):
BulkAPI.from_string(bad_string, executor)
def test_to_string(self, commands):
nd_json = BulkAPI.to_string(commands)
self._assert_nd_json_matches_commands(nd_json, commands)
def test_to_stream(self, commands):
handle = StringIO()
BulkAPI.to_stream(handle, commands)
self._assert_nd_json_matches_commands(handle.getvalue(), commands)
def test__string_to_lines(self):
lines = BulkAPI._string_to_lines(
"\nline_1\n\nlong_middle_line_2\nline_3")
assert isinstance(lines, GeneratorType)
assert list(lines) == ["line_1", "long_middle_line_2", "line_3"]
def test_to_string(self, commands):
expected = deepcopy([command.raw for command in commands])
nd_json = BulkAPI.to_string(commands)
assert self._decode_ndjson(nd_json) == expected
