def test_describe_message(self):
class Input(BaseModel):
key: int
title: str
class Output(BaseModel):
key: int
class MyProcessor(Processor):
def process(self, msg: Input, id: str) -> Output:
return Output(key=1)
logger = mock.MagicMock()
msgs = [DescribeMessage(), {"key": "3", "title": "title"}]
settings = ProcessorSettings(
name="processor",
version="0.0.0",
description="",
in_kind=TapKind.MEM,
out_kind=TapKind.MEM,
)
processor = MyProcessor(settings,
input_class=Input,
output_class=Output,
logger=logger)
processor.parse_args(args="--out-topic test".split())
processor.source.data = msgs
processor.start()
assert len(processor.destination.results) == 2
result = processor.destination.results[0]
assert result.input_schema is not None
assert result.output_schema is not None
def test_processor_invalid_output(self):
class Output(BaseModel):
language: str
class Input(BaseModel):
key: int
outputs = [Output(language="en"), Input(key=1), Output(language="en")]
class MyProcessor(Processor):
def process(self, msg, id):
return outputs.pop(0)
settings = ProcessorSettings(
name="processor",
version="0.1.0",
description="",
in_kind=TapKind.MEM,
out_kind=TapKind.MEM,
)
processor = MyProcessor(settings,
input_class=Input,
output_class=Output)
msgs = [{"key": 1}, {"key": 2}, {"key": 3}]
processor.parse_args()
processor.source.data = msgs
with pytest.raises(PipelineOutputError):
processor.start()
def test_mem_processor_limit(self):
class Input(BaseModel):
pass
class Output(BaseModel):
pass
class MyProcessor(Processor):
def process(self, msg: Input, id: str) -> Output:
return Output()
msgs = [{}, {}, {}]
settings = ProcessorSettings(
name="processor",
version="0.0.0",
description="",
in_kind=TapKind.MEM,
out_kind=TapKind.MEM,
)
processor = MyProcessor(settings,
input_class=Input,
output_class=Output)
processor.parse_args(args=["--limit", "1", "--out-topic", "test"])
processor.source.data = msgs
processor.start()
assert len(processor.destination.results) == 1
def test_logging(self, monkeypatch):
class Input(BaseModel):
key: int
class Output(BaseModel):
key: int
class MyProcessor(Processor):
def process(self, msg, id):
self.logger.info("logging")
return msg
logger = mock.MagicMock()
msgs = [{"key": "1"}, {"key": "2"}, {"key": "3"}]
settings = ProcessorSettings(
name="processor",
version="0.0.0",
description="",
in_kind=TapKind.MEM,
out_kind=TapKind.MEM,
debug=True,
monitoring=True,
)
processor = MyProcessor(settings,
input_class=Input,
output_class=Output,
logger=logger)
processor.parse_args(args="--out-topic test".split())
processor.source.data = msgs
monkeypatch.setattr(processor, "monitor", mock.MagicMock())
processor.start()
assert len(processor.destination.results) == 3
logger.info.assert_any_call("logging")
def test_file_stdin_stdout(self):
class Input(BaseModel):
key: int
language: str
class Output(BaseModel):
key: int
language: str
class MyProcessor(Processor):
def process(self, msg: Input, id: str) -> Output:
return Output(key=msg.key, language=msg.language)
with tempfile.NamedTemporaryFile() as tmpInFile:
make_output(tmpInFile.name)
settings = ProcessorSettings(name="processor",
version="0.0.0",
description="")
processor = MyProcessor(settings,
input_class=Input,
output_class=Output)
processor.parse_args(args=[
"--in-kind",
"FILE",
"--in-filename",
tmpInFile.name,
"--out-kind",
"FILE",
"--out-filename",
"-",
])
processor.start()
def test_processor(self):
class Input(BaseModel):
key: str
class Output(BaseModel):
key: str
newkey: str
class MyProcessor(Processor):
def process(self, input, id):
return Output(key=input.key, newkey="newval")
msgs = [{"key": "1"}, {"key": "2"}, {"key": "3"}]
settings = ProcessorSettings(name="processor",
version="0.0.0",
description="")
processor = MyProcessor(settings,
input_class=Input,
output_class=Output)
processor.parse_args(
args="--in-kind MEM --out-kind MEM --out-topic test".split())
processor.source.data = msgs
processor.start()
assert len(processor.destination.results) == 3
m = processor.destination.results[0]
assert m.get("newkey") == "newval"
def test_processor_invalid_input(self):
class Output(BaseModel):
language: str
class Input(BaseModel):
language: str
class MyProcessor(Processor):
def process(self, msg: Input, id: str) -> Output:
return Output(language="en")
settings = ProcessorSettings(
name="processor",
version="0.1.0",
description="",
in_kind=TapKind.MEM,
out_kind=TapKind.MEM,
)
processor = MyProcessor(settings,
input_class=Input,
output_class=Output)
msgs = [{"language": "en"}, {"key": 1}, {"key": 2}, {"language": "en"}]
processor.parse_args()
processor.source.data = msgs
processor.start()
assert len(processor.destination.results) == 2
def __init__(self):
settings = ProcessorSettings(name=__worker__,
version=__version__,
description="Test Processor")
super().__init__(
settings,
input_class=Input,
output_class=Output,
)
self.counter = 0
def test_mem_processor_raise(self):
class Input(BaseModel):
pass
class Output(BaseModel):
pass
settings = ProcessorSettings(name="processor",
version="0.0.0",
description="")
processor = Processor(settings, input_class=Input, output_class=Output)
processor.parse_args(args=["--in-kind", "MEM"])
assert processor.settings.out_kind is None
assert not hasattr(processor, "destination")
def test_mem_processor(self):
class Input(BaseModel):
pass
class Output(BaseModel):
pass
class MyProcessor(Processor):
def process(self, msg: Input, id: str) -> Output:
return Output()
msgs = [{}, {}, {}]
settings = ProcessorSettings(name="processor",
version="0.0.0",
description="")
processor = MyProcessor(settings,
input_class=Input,
output_class=Output)
processor.parse_args(
args="--in-kind MEM --out-kind MEM --out-topic test".split())
processor.source.data = msgs
processor.start()
assert len(processor.destination.results) == 3
def test_mem_processor_nooutput(self):
class Input(BaseModel):
pass
class Output(BaseModel):
pass
class MyProcessor(Processor):
def process(self, msg: Input, id: str) -> Output:
assert self.message is not None
raise NotImplementedError
msgs = [{}, {}, {}]
settings = ProcessorSettings(name="processor",
version="0.0.0",
description="")
processor = MyProcessor(settings,
input_class=Input,
output_class=Output)
processor.parse_args(args="--in-kind MEM --out-kind MEM".split())
processor.source.data = msgs
processor.start()
assert len(processor.destination.results) == 0
assert not hasattr(processor, "message")