def run(self, context: PipelineContext, transition: Transition) -> Dict:
local_context = transition.prepare_input(context.data, self.definition)
try:
result = self.do_work(local_context)
output = transition.prepare_output(result, self.definition)
transition.check_failure(output)
return output
except Exception as err:
raise TransformationError(err)
def run(self, context: PipelineContext, transition: Transition) -> Dict:
local_context = transition.prepare_input(context.data, self.definition)
try:
result = self.do_work(local_context, context.kafka_producer)
output = transition.prepare_output(result, self.definition)
transition.check_failure(output)
return output
except Exception as err:
msg = f'Error sending: {err}' if not self.last_call_kafka_error else \
f'Error sending: {self.last_call_kafka_error}'
raise TransformationError(msg) from err
finally:
self.last_call_kafka_error = None
def run(self, context: PipelineContext, transition: Transition) -> Dict:
try:
if context.zeebe is None:
raise RuntimeError('Expected pipeline context to have '
' a ZeebeConnection, found None')
local_context = transition.prepare_input(context.data,
self.definition)
transition.check_failure(local_context)
for spawn in self._prepare_spawns(mapping=transition.output_map,
**local_context):
self._handle_spawn(context.zeebe, **spawn)
return {'success': True}
except Exception as err:
raise TransformationError(err)
def test__pipelineset_simple():
def _getter(*args, **kwargs):
return transforms.JavascriptCall('_id', examples.XF_JS_ADDER, None)
context = PipelineContext()
context.register_result('source', {'one': 1})
transition = {
'input_map': {
'a': '$.source.one',
'b': '$.source.one'
},
'output_map': {
'result': '$.result'
}
}
stages = []
for x in range(1, 10):
name = f'stage{x}'
stage = Stage(name, 'jscall', 'adder',
Transition(**deepcopy(transition)), _getter)
transition['input_map']['a'] = f'$.{name}.result'
stages.append(stage)
pipeline = PipelineSet('some-pipeline', 'sometenant', stages=stages)
res = pipeline.run(context)
assert (res.context.data['stage9'] == {'result': 10})
def test__restcall_request_methods(definition, transition_override, transition,
config, exception, status):
url = definition.get('url')
method_str = config.get('method')
method = responses.GET if method_str == 'get' else None
body = Exception() if exception else '"{}"'
responses.add(method, url, body=body, status=status)
if transition_override:
transition[transition_override[0]] = transition_override[1]
transition = Transition(**transition)
def fn():
rc = transforms.RestCall('_id', definition, None)
context = PipelineContext()
context.register_result('source', config)
res = rc.run(context, transition)
assert (res.get('status_code') == status)
if exception:
with pytest.raises(exception):
fn()
else:
fn()
def test__Transformation_basic(BaseTransition):
trans = transforms.Transformation('_id', examples.BASE_TRANSFORMATION,
None)
context = PipelineContext()
context.register_result('source', {'ref': 200})
assert (trans.run(context,
Transition(**examples.BASE_TRANSITION_PASS)) == {
'ref': 200,
'list': [200],
'const': 'c',
'dict': {
'f': 200
}
})
context.register_result('source', {'ref': 500})
with pytest.raises(TransformationError):
trans.run(context, Transition(**examples.BASE_TRANSITION_PASS))
def run(self, context: PipelineContext, transition: Transition) -> Dict:
input_context = context.data
job = context.source_event
try:
local_context = transition.prepare_input(context.data,
self.definition)
# failure / pass based on global context
transition.check_failure(input_context)
if isinstance(job, TestEvent):
return local_context
if not isinstance(job, ZeebeJob):
raise TypeError('Expected source event to be ZeebeJob'
f' found {type(job)}.')
job.complete(variables=local_context)
return local_context
except Exception as err:
try:
job.fail(message=str(err))
except Exception:
pass
raise TransformationError(err)
def _prepare_spawns(self,
mode=None,
message_id=None,
listener_name=None,
mapping=None,
message_iterator=None,
message_destination=None,
**local_context) -> Iterable[Tuple[str, Dict]]:
# returns (workflow, msg) generator
if mode == 'single':
yield {
'message_id': message_id,
**Transition.apply_map(mapping, local_context)
}
elif mode == 'multiple':
if not message_iterator:
raise RuntimeError('Expected message_iterator in'
'mode `multiple` found None')
res = Transition.handle_parser_results(
CachedParser.find(message_iterator, local_context))
for msg in res:
if message_destination:
yield {
'message_id': message_id,
**{
message_destination: msg
},
**Transition.apply_map(mapping, local_context)
}
else:
yield {
'message_id': message_id,
**msg,
**Transition.apply_map(mapping, local_context)
}
else:
raise RuntimeError(
f'Expected mode in [single, multiple], got {mode}')
def test__xf_ZeebeComplete_basic():
transition = dict(examples.BASE_TRANSITION_PASS)
transition['input_map'] = {'ref': '$.source.ref'}
transition['pass_condition'] = '$.source.ref.`match(200, null)`'
transition = Transition(**transition)
transformation = transforms.ZeebeComplete('_id',
examples.BASE_TRANSFORMATION,
None)
context = PipelineContext(TestEvent(**{'ref': 200}))
# context.register_result('source', {'ref': 200})
LOG.debug(json.dumps(context.data, indent=2))
assert (transformation.run(context, transition) == {'ref': 200})
context.register_result('source', {'ref': 500})
with pytest.raises(TransformationError):
transformation.run(context, transition)
def test__stage_simple():
transition = {
'input_map': {
'res': '$.source.res'
},
'output_map': {
'res': '$.res'
}
}
def _getter(*args, **kwargs):
return transforms.Transformation('_id', examples.BASE_TRANSFORMATION,
None)
context = PipelineContext()
context.register_result('source', {'res': 1})
stage = Stage('test', '__transformation', '_id', Transition(**transition),
_getter)
res = stage.run(context)
assert (res['res'] == 1)
def BaseTransition():
return Transition(**examples.BASE_TRANSITION)