def test_workflow01(self):
class GenericWEWithXChooser(GenericWorkflowEngine):
def callback_chooser(self, obj):
return self.callbacks.get('x')
we0 = GenericWorkflowEngine()
we1 = GenericWorkflowEngine()
we2 = GenericWEWithXChooser()
we0.addManyCallbacks('*', [
obj_append('mouse'),
[obj_append('dog'), jump_call(1), obj_append('cat'), obj_append('puppy')],
obj_append('horse'),
])
we1.setWorkflow([
obj_append('mouse'),
[obj_append('dog'), jump_call(1), obj_append('cat'), obj_append('puppy')],
obj_append('horse'),
])
we2.addManyCallbacks('x', [
obj_append('mouse'),
[obj_append('dog'), jump_call(1), obj_append('cat'), obj_append('puppy')],
obj_append('horse'),
])
we0.process(self.d0)
we1.process(self.d1)
we2.process(self.d2)
assert self.d0 == self.d1
assert self.d0 == self.d2
def before_processing(objects, self):
"""
Executed before processing the workflow.
"""
# 1. Save workflow (ourselves).
if not self.db_obj.uuid:
self.save()
self.setCounterInitial(objects)
self.log.info("[%s], %s has been started" %
(str(datetime.now()), str(self.db_obj.name)))
# 2. We want to save all the objects as version 0.
for o in objects:
same_workflow = \
o.db_obj.workflow_id and \
o.db_obj.workflow_id == self.db_obj.uuid
if o.db_obj.id and same_workflow:
# If object exists and we are running the same workflow,
# do nothing
continue
# Set the current workflow id in the object
if o.db_obj.version == CFG_OBJECT_VERSION.INITIAL \
and o.db_obj.workflow_id is not None:
pass
else:
o.db_obj.workflow_id = self.db_obj.uuid
o.save(CFG_OBJECT_VERSION.INITIAL)
GenericWorkflowEngine.before_processing(objects, self)
def test_CHOICE03(self):
we = GenericWorkflowEngine()
doc = self.getDoc()[0:1]
def arbiter(obj, eng):
return obj[-1]
we.setWorkflow([i('start'),
cf.CHOICE(arbiter,
('bam', lambda obj, eng: obj.append('bom')),
('end', lambda obj,
eng: obj.append('error')),
('bim', lambda obj, eng: obj.append('bam')),
bom=(lambda obj, eng: obj.append('bum')),
one=(lambda obj, eng: obj.append('bim')),
bum=cf.STOP(),
),
cf.TASK_JUMP_BWD(-1)])
we.process(doc)
d = ' '.join(doc[0])
assert 'bim bam bom bum' in d
assert 'error' not in d
assert len(doc[0]) == 6
def test_PARALLEL_SPLIT01(self):
we = GenericWorkflowEngine()
doc = self.getDoc()
we.setWorkflow([i('start'),
cf.PARALLEL_SPLIT(
printer('p1'),
printer('p2'),
printer('p3'),
printer('p4'),
printer('p5')),
lambda o, e: time.sleep(.1),
a('end')
])
we.process(doc)
r = [' '.join(doc[x]) for x in range(len(doc))]
assert doc[0][0] == 'start'
assert doc[0][1] == 'one'
assert doc[1][0] == 'start'
assert doc[1][1] == 'two'
# end must have been inserted while printers were running
# mixed together with them
all_pos = set()
for x in range(len(doc)):
pos = doc[x].index('end')
assert pos > 2
assert pos < len(doc[x])
all_pos.add(pos)
def test_nested_workflow_halt(self):
other_wfe = GenericWorkflowEngine()
wfe = self.wfe
other_wfe.callbacks.add_many([
m('mouse'),
[
m('dog'),
[m('cat'), m('puppy')],
[m('python'), halt_processing()],
m('horse'),
]
], self.key)
wfe.callbacks.add_many([
m('mouse'),
[
m('dog'),
[m('cat'), m('puppy')],
[m('python'), lambda o, e: other_wfe.process(self.tokens)],
m('horse'),
]
], self.key)
with pytest.raises(HaltProcessing):
wfe.process(self.tokens)
t = get_first(self.tokens)
assert get_xth(self.tokens, 0) == 'mouse dog cat puppy python mouse dog cat puppy python'
assert get_xth(self.tokens, 1) is None
assert get_xth(self.tokens, 2) is None
def before_processing(objects, self):
"""
Executed before processing the workflow.
"""
# 1. Save workflow (ourselves).
if not self.db_obj.uuid:
self.save()
self.set_counter_initial(len(objects))
self.log_info("Workflow has been started")
# 2. We want to save all the objects as version 0.
for o in objects:
same_workflow = \
o.id_workflow and \
o.id_workflow == self.db_obj.uuid
if o.id and same_workflow:
# If object exists and we are running the same workflow,
# do nothing
o.log_info("object saving process : was already existing")
continue
# Set the current workflow id in the object
if o.version == CFG_OBJECT_VERSION.INITIAL \
and o.id_workflow is not None:
o.log_info("object saving process : was already existing")
pass
else:
o.id_workflow = self.uuid
o.save(o.version)
GenericWorkflowEngine.before_processing(objects, self)
def test_nested_workflow_halt(self):
other_wfe = GenericWorkflowEngine()
wfe = self.wfe
other_wfe.callbacks.add_many([
m('mouse'),
[
m('dog'),
[m('cat'), m('puppy')],
[m('python'), halt_processing()],
m('horse'),
]
], self.key)
wfe.callbacks.add_many([
m('mouse'),
[
m('dog'),
[m('cat'), m('puppy')],
[m('python'), lambda o, e: other_wfe.process(self.tokens)],
m('horse'),
]
], self.key)
with pytest.raises(HaltProcessing):
wfe.process(self.tokens)
t = get_first(self.tokens)
assert get_xth(
self.tokens,
0) == 'mouse dog cat puppy python mouse dog cat puppy python'
assert get_xth(self.tokens, 1) is None
assert get_xth(self.tokens, 2) is None
def automated_process(target): # pylint: disable=unused-argument
"""
Work out the current point in the automated workflow and process the next
step.
"""
my_engine = GenericWorkflowEngine()
my_engine.callbacks.replace(
[task_add_repo, task_collect_nonwords, task_submit, task_cleanup])
my_engine.process([State(target)])
def test_init(self):
# init with empty to full parameters
we1 = GenericWorkflowEngine()
callbacks = [
obj_append('mouse'),
[obj_append('dog'), jump_call(1), obj_append('cat'), obj_append('puppy')],
obj_append('horse'),
]
we1.addManyCallbacks('*', deepcopy(callbacks))
we1.process(self.d1)
def run_workflow_2():
# Create the engine as in the previous example
my_engine_2 = GenericWorkflowEngine()
my_engine_2.setWorkflow(my_workflow_2)
try:
# Note how we don't need to keep a reference to our tokens - the engine
# allows us to access them via `my_engine.objects` later.
my_engine_2.process([[], [0,1], [0,1,0,1]])
except HaltProcessing:
# Our engine was built to throw this exception every time an object is
# completed. At this point we can inspect the object to decide what to
# do next. In any case, we will ask it to move to the next object,
# until it stops throwing the exception (which, in our case, means it
# has finished with all objects).
while True:
try:
# Restart the engine with the next object, starting from the
# first task.
my_engine_2.restart('next', 'first')
except HaltProcessing:
continue
else:
print "Done!", my_engine_2.objects
break
def test_PARALLEL_SPLIT02(self):
"""TODO: this test is failing, but that is because sometimes it does
not take into accounts threads being executed in random mannger"""
we = GenericWorkflowEngine()
doc = self.getDoc()[0:1]
we.setWorkflow([i('start'),
cf.PARALLEL_SPLIT(
[
cf.PARALLEL_SPLIT(
printer('p0'),
printer('p0a'),
cf.PARALLEL_SPLIT(
printer('p0b'),
printer('p0c')
),
),
printer('xx')
],
[
a('AAA'),
printer('p2b')
],
printer('p3'),
[
a('p4a'),
printer('p4b'),
printer('p4c')
],
[printer('p5'), cf.PARALLEL_SPLIT(
printer('p6'),
printer('p7'),
[printer('p8a'), printer('p8b')],
)]),
a('end')
])
we.process(doc)
# give threads time to finish
time.sleep(2)
assert doc[0][0] == 'start'
assert doc[0][1] == 'one'
# at least the fist object should have them all
# print doc[0]
for x in ['p0', 'p0a', 'p0b', 'p0c', 'xx', 'AAA', 'p2b', 'p3', 'p4a',
'p4b', 'p4c', 'p5', 'p6', 'p8a', 'p8b']:
doc[0].index(x) # will fail if not present
def test_init(self):
d1 = self.getDoc()
d2 = self.getDoc()
d3 = self.getDoc()
# init with empty to full parameters
we1 = GenericWorkflowEngine()
we2 = GenericWorkflowEngine(callback_chooser=asterisk_chooser)
try:
we3 = GenericWorkflowEngine(processing_factory='x',
callback_chooser='x',
before_processing='x',
after_processing='x')
except Exception, msg:
assert 'must be a callable' not in msg
def run_workflow(records, name, **kwargs):
"""Run the uploader workflow itself.
:param records: List of tuples `(blob, json_record)` from :func:`translate`
:param name: Name of the workflow to be run.
:parma kwargs: Additional arguments to be used by the tasks of the workflow
:returns: Typically the list of record Ids that has been process, although
this value could be modify by the `post_tasks`.
"""
def _run_pre_post_tasks(tasks):
"""Helper function to run list of functions."""
for task in tasks:
task(records, **kwargs)
#FIXME: don't know why this is needed but IT IS!
records = records[0]
if name in cfg['UPLOADER_WORKFLOWS']:
workflow = workflows.get(name)
else:
raise UploaderException("Workflow {0} not in UPLOADER_WORKFLOWS".format(name))
_run_pre_post_tasks(workflow.pre_tasks)
wfe = WorkflowEngine()
wfe.setWorkflow(workflow.tasks)
wfe.setVar('options', kwargs)
wfe.process(records)
_run_pre_post_tasks(workflow.post_tasks)
signals.uploader_finished.send(uploader_workflow=name,
result=records, **kwargs)
return records
def run_workflow(records, name, **kwargs):
"""Run the uploader workflow itself.
:param records: List of tuples `(blob, json_record)` from :func:`translate`
:param name: Name of the workflow to be run.
:parma kwargs: Additional arguments to be used by the tasks of the workflow
:returns: Typically the list of record Ids that has been process, although
this value could be modify by the `post_tasks`.
"""
def _run_pre_post_tasks(tasks):
"""Helper function to run list of functions."""
for task in tasks:
task(records, **kwargs)
#FIXME: don't know why this is needed but IT IS!
records = records[0]
if name in cfg['UPLOADER_WORKFLOWS']:
workflow = workflows.get(name)
else:
raise UploaderException(
"Workflow {0} not in UPLOADER_WORKFLOWS".format(name))
_run_pre_post_tasks(workflow.pre_tasks)
wfe = WorkflowEngine()
wfe.setWorkflow(workflow.tasks)
wfe.setVar('options', kwargs)
wfe.process(records)
_run_pre_post_tasks(workflow.post_tasks)
signals.uploader_finished.send(uploader_workflow=name,
result=records,
**kwargs)
return records
def test_RUN_WF02(self):
"""Test wfe is reinit=True - eng must not remember"""
we = GenericWorkflowEngine()
doc = self.getDoc()[0:1]
we.callbacks.replace(
[
i('start'),
ut.RUN_WF(
[
lambda obj, eng: obj.append('bom'),
lambda obj, eng: obj.append('bam'),
lambda obj, eng: obj.append('bum'),
lambda obj, eng: obj.append('end'),
lambda obj, eng: obj.append(
eng.store.setdefault('eng-end', '')),
e('eng-end', 'eng-end')
],
data_connector=lambda obj, eng: [obj],
outkey='#wfe',
reinit=True
),
]
)
we.process(doc)
d = ' '.join(doc[0])
assert 'start' in d
assert 'bom' in d
assert 'bam' in d
assert 'bum' in d
assert 'end' in d
assert 'eng-end' not in d
# run the same thing again
we.process(doc)
d = ' '.join(doc[0])
assert 'start' in d
assert d.count('bom') == 2
assert d.count('bam') == 2
assert d.count('bum') == 2
assert 'end' in d
assert 'eng-end' not in d # it must not be present if reinit=True
def test_IF_ELSE02(self):
we = GenericWorkflowEngine()
doc = self.getDoc()
we.setWorkflow([i('add'),
cf.IF_ELSE(lambda o, e: o[1] == 'three',
a('3'),
a('other'))
])
we.process(doc)
r = [' '.join(doc[x]) for x in range(len(doc))]
assert r[0] == 'add one other'
assert r[1] == 'add two other'
assert r[2] == 'add three 3'
assert r[3] == 'add four other'
assert r[4] == 'add five other'
def test_PARALLEL_SPLIT03(self):
we = GenericWorkflowEngine()
doc = self.getDoc()
we.setWorkflow([i('start'),
cf.PARALLEL_SPLIT(
[cf.IF(lambda obj, eng: 'jump-verified' in obj, a('error')),
cf.PARALLEL_SPLIT(
[cf.IF(lambda obj, eng: 'nasty-jump' in obj,
[a('jump-ok'),
lambda obj, eng: ('nasty-jump' in obj and
obj.append('jump-verified'))]),
cf.PARALLEL_SPLIT(
a('ok-1'),
a('ok-2'),
cf.IF(lambda obj, eng: 'ok-3' not in obj,
lambda obj, eng: (obj.append('ok-3') and
eng.breakFromThisLoop())),
a('ok-4')),
a('xx'),
lambda obj, eng: ('jump-verified' in obj and
eng.breakFromThisLoop()),
a('nasty-jump'),
cf.TASK_JUMP_IF(
lambda obj, eng: 'jump-verified' not in obj, -100)]),
],
[a('AAA'), a('p2b')]),
a('end')
])
we.process(doc)
# give threads time to finish
time.sleep(.5)
d = doc[0]
# at least the fist object should have them all
# print doc[0]
for x in ['nasty-jump', 'jump-verified', 'ok-3']:
d.index(x) # will fail if not present
assert d.count('ok-1') > 1
def interactive_old_word(state, jsonobj, word):
"""
Single word processing
"""
my_engine = GenericWorkflowEngine()
my_engine.callbacks.replace(
[check_websearch, is_nonword, is_typo, what_now])
newstate = NonwordState(
interaction=state.interaction,
target=state.target,
word=word,
details=jsonobj[word],
repopath=state.repopath,
nonword_delegate=state.nonword_delegate,
)
try:
my_engine.process([newstate])
except HaltProcessing:
if newstate.done:
return True
return False
def test_SIMPLE_MERGE03(self):
we = GenericWorkflowEngine()
doc = self.getDoc()[0:1]
we.setWorkflow([i('start'),
cf.SIMPLE_MERGE(
lambda obj, eng: obj.append('bom'),
lambda obj, eng: obj.append('error'),
lambda obj, eng: obj.append('bam'),
lambda obj, eng: obj.append('bum'),
lambda obj, eng: obj.append('end'),
),
])
we.process(doc)
d = ' '.join(doc[0])
assert 'start' in d
assert 'bom' in d
assert 'error' not in d
assert 'end' in d
def interactive_task_collect_nonwords( # pylint: disable=unused-argument
reponame,
target,
nonword_delegate=None,
nonstop=False):
"""
Saves nonwords until a typo is found
"""
if nonword_delegate is None:
nonword_delegate = interactive_nonword_delegate(target)
key = "repository_map"
repository_map = get_json_value(key, {})
repodir = repository_map[reponame]
repodirpath = Path(repodir)
jsonpath = repodirpath / "spelling.json"
if not jsonpath.is_file():
print(f"Unable to locate spelling at {jsonpath}", file=sys.stderr)
return
with io.open(jsonpath, "r", encoding="utf-8") as fobj:
jsonobj = json.load(fobj)
words = get_sorted_words(jsonobj)
my_engine = GenericWorkflowEngine()
my_engine.callbacks.replace(
[check_websearch, is_nonword, is_typo, what_now])
for word in words:
state = NonwordState(
target=target,
word=word,
details=jsonobj[word],
repopath=repodirpath,
nonword_delegate=nonword_delegate,
)
try:
my_engine.process([state])
except HaltProcessing:
if state.done and not nonstop:
return
print(
f"{Fore.YELLOW}Completed checking all words for {reponame}!{Style.RESET_ALL}"
)
def test_CHOICE03(self):
we = GenericWorkflowEngine()
doc = self.getDoc()[0:1]
def arbiter(obj, eng):
return obj[-1]
we.setWorkflow([
i('start'),
cf.CHOICE(
arbiter,
('bam', lambda obj, eng: obj.append('bom')),
('end', lambda obj, eng: obj.append('error')),
('bim', lambda obj, eng: obj.append('bam')),
bom=(lambda obj, eng: obj.append('bum')),
one=(lambda obj, eng: obj.append('bim')),
bum=cf.STOP(),
),
cf.TASK_JUMP_BWD(-1)
])
we.process(doc)
d = ' '.join(doc[0])
assert 'bim bam bom bum' in d
assert 'error' not in d
assert len(doc[0]) == 6
def test_IF_ELSE03(self):
we = GenericWorkflowEngine()
doc = self.getDoc()
doc[3].append('4')
def test(v):
return lambda o, e: v in o
we.setWorkflow([
i('add'),
cf.IF_ELSE(test('three'), [
a('xxx'),
cf.IF_ELSE(test('xxx'), [
a('6'),
cf.IF_ELSE(test('6'), a('six'), (a('only-3s'), a('error')))
], a('ok'))
], [
cf.IF_ELSE(
test('4'),
cf.IF_ELSE(test('four'), [a('44'), [[[a('forty')]]]],
a('error')), a('not-four'))
]),
a('end'),
cf.IF_ELSE(test('error'), a('gosh!'), a('OK'))
])
we.process(doc)
r = [' '.join(doc[x]) for x in range(len(doc))]
assert r[0] == 'add one not-four end OK'
assert r[1] == 'add two not-four end OK'
assert r[2] == 'add three xxx 6 six end OK'
assert r[3] == 'add four 4 44 forty end OK'
assert r[4] == 'add five not-four end OK'
def run_workflow(message):
jsonObj = json.loads(message)
print "Workflow input header message %s" % (jsonObj['Header'])
data = jsonObj['Body']
my_engine = GenericWorkflowEngine()
my_engine.setWorkflow(workflow_model_1())
my_engine.process(data)
def test_PARALLEL_SPLIT01(self):
we = GenericWorkflowEngine()
doc = self.getDoc()
we.setWorkflow([
i('start'),
cf.PARALLEL_SPLIT(printer('p1'), printer('p2'), printer('p3'),
printer('p4'), printer('p5')),
lambda o, e: time.sleep(.1),
a('end')
])
we.process(doc)
r = [' '.join(doc[x]) for x in range(len(doc))]
assert doc[0][0] == 'start'
assert doc[0][1] == 'one'
assert doc[1][0] == 'start'
assert doc[1][1] == 'two'
# end must have been inserted while printers were running
# mixed together with them
all_pos = set()
for x in range(len(doc)):
pos = doc[x].index('end')
assert pos > 2
assert pos < len(doc[x])
all_pos.add(pos)
def test_configure(self):
callbacks_list = [
obj_append('mouse'),
[obj_append('dog'), jump_call(1), obj_append('cat'), obj_append('puppy')],
obj_append('horse'),
]
we = GenericWorkflowEngine()
we.addManyCallbacks('*', callbacks_list)
# process using defaults
we.process(self.d1)
r = 'one mouse dog cat puppy horse'.split()
we = GenericWorkflowEngine()
we.addManyCallbacks('*', callbacks_list)
we.process(self.d2)
assert self.d1[0] == r
assert self.d2[0] == r
assert self.d1 == self.d2
def run_workflow_1(): my_object0 = MyObject(0) my_object1 = MyObject(1) my_engine_1 = GenericWorkflowEngine() my_engine_1.setWorkflow(my_workflow_1) my_engine_1.process([my_object0, my_object1])
def test_IF_ELSE03(self):
we = GenericWorkflowEngine()
doc = self.getDoc()
doc[3].append('4')
def test(v):
return lambda o, e: v in o
we.setWorkflow([i('add'),
cf.IF_ELSE(
test('three'),
[a('xxx'), cf.IF_ELSE(test('xxx'),
[a('6'), cf.IF_ELSE(
test('6'),
a('six'),
(a('only-3s'), a('error')))],
a('ok'))],
[cf.IF_ELSE(
test('4'),
cf.IF_ELSE(test('four'),
[a('44'), [[[a('forty')]]]],
a('error')),
a('not-four'))]),
a('end'),
cf.IF_ELSE(test('error'),
a('gosh!'),
a('OK'))
])
we.process(doc)
r = [' '.join(doc[x]) for x in range(len(doc))]
assert r[0] == 'add one not-four end OK'
assert r[1] == 'add two not-four end OK'
assert r[2] == 'add three xxx 6 six end OK'
assert r[3] == 'add four 4 44 forty end OK'
assert r[4] == 'add five not-four end OK'
def test_init(self):
d1 = self.getDoc()
d2 = self.getDoc()
d3 = self.getDoc()
# init with empty to full parameters
we1 = GenericWorkflowEngine()
we2 = GenericWorkflowEngine(callback_chooser=asterisk_chooser)
try:
we3 = GenericWorkflowEngine(processing_factory='x',
callback_chooser='x',
before_processing='x',
after_processing='x')
except Exception as msg:
assert 'must be a callable' in str(msg)
try:
we3 = GenericWorkflowEngine(callback_chooser=asterisk_chooser,
after_processing='x')
except Exception as msg:
assert 'must be a callable' in str(msg)
we1.addManyCallbacks('*', [
m('mouse'),
[m('dog'), call_forward(1), m('cat'), m('puppy')],
m('horse'),
])
we2.addManyCallbacks('*', [
m('mouse'),
[m('dog'), call_forward(1), m('cat'), m('puppy')],
m('horse'),
])
we1.process(d1)
we2.process(d2)
def WHILE(cond, branch):
"""Keeps executing branch as long as the condition cond is True
@var cond: callable, function that decides
@var branch: block of functions to run [if=true]
"""
# quite often i passed a function, which results in errors
if callable(branch):
branch = (branch,)
# we don't know what is hiding inside branch
branch = tuple(engine._cleanUpCallables(branch))
def x(obj, eng):
if not cond(obj, eng):
eng.breakFromThisLoop()
x.__name__ = 'WHILE'
return [x, branch, TASK_JUMP_BWD(-(len(branch)+1))]
def WHILE(cond, branch):
"""Keeps executing branch as long as the condition cond is True
:param cond: callable, function that decides
:param branch: block of functions to run [if=true]
"""
# quite often i passed a function, which results in errors
if callable(branch):
branch = (branch, )
# we don't know what is hiding inside branch
branch = tuple(engine._cleanUpCallables(branch))
def x(obj, eng):
if not cond(obj, eng):
eng.breakFromThisLoop()
x.__name__ = 'WHILE'
return [x, branch, TASK_JUMP_BWD(-(len(branch) + 1))]
def test_RUN_WF01(self):
"""Test wfe is reinit=False, eng must remember previous invocations"""
we = GenericWorkflowEngine()
doc = self.getDoc()[0:1]
we.setWorkflow([
i('start'),
ut.RUN_WF(
[
lambda obj, eng: obj.append('bom'),
lambda obj, eng: obj.append('bam'),
lambda obj, eng: obj.append('bum'),
lambda obj, eng: obj.append('end'), lambda obj, eng: obj.
append(eng.store.setdefault('eng-end', '')),
e('eng-end', 'eng-end')
],
data_connector=lambda obj, eng: [obj],
outkey='#wfe',
),
])
we.process(doc)
d = ' '.join(doc[0])
assert 'start' in d
assert 'bom' in d
assert 'bam' in d
assert 'bum' in d
assert 'end' in d
assert 'eng-end' not in d
# run the same thing again
we.process(doc)
d = ' '.join(doc[0])
assert 'start' in d
assert d.count('bom') == 2
assert d.count('bam') == 2
assert d.count('bum') == 2
assert 'end' in d
assert 'eng-end' in d # now it must be present
def test_RUN_WF02(self):
"""Test wfe is reinit=True - eng must not remember"""
we = GenericWorkflowEngine()
doc = self.getDoc()[0:1]
we.setWorkflow([
i('start'),
ut.RUN_WF([
lambda obj, eng: obj.append('bom'),
lambda obj, eng: obj.append('bam'),
lambda obj, eng: obj.append('bum'),
lambda obj, eng: obj.append('end'),
lambda obj, eng: obj.append(eng.getVar('eng-end', '')),
e('eng-end', 'eng-end')
],
data_connector=lambda obj, eng: [obj],
outkey='#wfe',
reinit=True),
])
we.process(doc)
d = ' '.join(doc[0])
assert 'start' in d
assert 'bom' in d
assert 'bam' in d
assert 'bum' in d
assert 'end' in d
assert 'eng-end' not in d
# run the same thing again
we.process(doc)
d = ' '.join(doc[0])
assert 'start' in d
assert d.count('bom') == 2
assert d.count('bam') == 2
assert d.count('bum') == 2
assert 'end' in d
assert 'eng-end' not in d # it must not be present if reinit=True
def test_RUN_WF01(self):
"""Test wfe is reinit=False, eng must remember previous invocations"""
we = GenericWorkflowEngine()
doc = self.getDoc()[0:1]
we.setWorkflow(
[
i('start'),
ut.RUN_WF(
[
lambda obj, eng: obj.append('bom'),
lambda obj, eng: obj.append('bam'),
lambda obj, eng: obj.append('bum'),
lambda obj, eng: obj.append('end'),
lambda obj, eng: obj.append(
eng.store.setdefault('eng-end', '')),
e('eng-end', 'eng-end')
],
data_connector=lambda obj, eng: [obj],
outkey='#wfe',
),
]
)
we.process(doc)
d = ' '.join(doc[0])
assert 'start' in d
assert 'bom' in d
assert 'bam' in d
assert 'bum' in d
assert 'end' in d
assert 'eng-end' not in d
# run the same thing again
we.process(doc)
d = ' '.join(doc[0])
assert 'start' in d
assert d.count('bom') == 2
assert d.count('bam') == 2
assert d.count('bum') == 2
assert 'end' in d
assert 'eng-end' in d # now it must be present
class TestGenericWorkflowEngine(unittest.TestCase):
"""Tests of the WE interface"""
def setUp(self):
self.key = '*'
def tearDown(self):
pass
def getDoc(self, val=None):
if val:
return [[x] for x in val.split()]
return [[x] for x in u"one two three four five".split()]
def addTestCallbacks(self, no, eng):
if type == 1:
eng.addManyCallbacks()
# --------- initialization ---------------
def test_init(self):
d1 = self.getDoc()
d2 = self.getDoc()
d3 = self.getDoc()
# init with empty to full parameters
we1 = GenericWorkflowEngine()
we2 = GenericWorkflowEngine(callback_chooser=asterisk_chooser)
try:
we3 = GenericWorkflowEngine(processing_factory='x',
callback_chooser='x',
before_processing='x',
after_processing='x')
except Exception, msg:
assert 'must be a callable' not in msg
try:
we3 = GenericWorkflowEngine(callback_chooser=asterisk_chooser,
after_processing='x')
except Exception, msg:
assert 'must be a callable' not in msg
def test_IF_ELSE02(self):
we = GenericWorkflowEngine()
doc = self.getDoc()
we.setWorkflow([
i('add'),
cf.IF_ELSE(lambda o, e: o[1] == 'three', a('3'), a('other'))
])
we.process(doc)
r = [' '.join(doc[x]) for x in range(len(doc))]
assert r[0] == 'add one other'
assert r[1] == 'add two other'
assert r[2] == 'add three 3'
assert r[3] == 'add four other'
assert r[4] == 'add five other'
def test_PARALLEL_SPLIT02(self):
"""TODO: this test is failing, but that is because sometimes it does
not take into accounts threads being executed in random mannger"""
we = GenericWorkflowEngine()
doc = self.getDoc()[0:1]
we.setWorkflow([
i('start'),
cf.PARALLEL_SPLIT(
[
cf.PARALLEL_SPLIT(
printer('p0'),
printer('p0a'),
cf.PARALLEL_SPLIT(printer('p0b'), printer('p0c')),
),
printer('xx')
], [a('AAA'), printer('p2b')], printer('p3'),
[a('p4a'), printer('p4b'),
printer('p4c')], [
printer('p5'),
cf.PARALLEL_SPLIT(
printer('p6'),
printer('p7'),
[printer('p8a'), printer('p8b')],
)
]),
a('end')
])
we.process(doc)
# give threads time to finish
time.sleep(2)
assert doc[0][0] == 'start'
assert doc[0][1] == 'one'
# at least the fist object should have them all
# print doc[0]
for x in [
'p0', 'p0a', 'p0b', 'p0c', 'xx', 'AAA', 'p2b', 'p3', 'p4a',
'p4b', 'p4c', 'p5', 'p6', 'p8a', 'p8b'
]:
doc[0].index(x) # will fail if not present
def test_init(self):
# init with empty to full parameters
we1 = GenericWorkflowEngine()
callbacks = [
obj_append('mouse'),
[
obj_append('dog'),
jump_call(1),
obj_append('cat'),
obj_append('puppy')
],
obj_append('horse'),
]
we1.addManyCallbacks('*', deepcopy(callbacks))
we1.process(self.d1)
def test_PARALLEL_SPLIT03(self):
we = GenericWorkflowEngine()
doc = self.getDoc()
we.setWorkflow([
i('start'),
cf.PARALLEL_SPLIT([
cf.IF(lambda obj, eng: 'jump-verified' in obj, a('error')),
cf.PARALLEL_SPLIT([
cf.IF(lambda obj, eng: 'nasty-jump' in obj, [
a('jump-ok'), lambda obj, eng:
('nasty-jump' in obj and obj.append('jump-verified'))
]),
cf.PARALLEL_SPLIT(
a('ok-1'), a('ok-2'),
cf.IF(
lambda obj, eng: 'ok-3' not in obj,
lambda obj, eng:
(obj.append('ok-3') and eng.breakFromThisLoop())),
a('ok-4')),
a('xx'), lambda obj, eng:
('jump-verified' in obj and eng.breakFromThisLoop()),
a('nasty-jump'),
cf.TASK_JUMP_IF(
lambda obj, eng: 'jump-verified' not in obj, -100)
]),
], [a('AAA'), a('p2b')]),
a('end')
])
we.process(doc)
# give threads time to finish
time.sleep(.5)
d = doc[0]
# at least the fist object should have them all
# print doc[0]
for x in ['nasty-jump', 'jump-verified', 'ok-3']:
d.index(x) # will fail if not present
assert d.count('ok-1') > 1
def test_SIMPLE_MERGE03(self):
we = GenericWorkflowEngine()
doc = self.getDoc()[0:1]
we.setWorkflow([i('start'),
cf.SIMPLE_MERGE(
lambda obj, eng: obj.append('bom'),
lambda obj, eng: obj.append('error'),
lambda obj, eng: obj.append('bam'),
lambda obj, eng: obj.append('bum'),
lambda obj, eng: obj.append('end'),
),
])
we.process(doc)
d = ' '.join(doc[0])
assert 'start' in d
assert 'bom' in d
assert 'error' not in d
assert 'end' in d
def setup_method(self, method):
self.key = '*'
self.wfe = GenericWorkflowEngine()
self.data = ['one', 'two', 'three', 'four', 'five']
self.tokens = [FakeToken(x, type='*') for x in self.data]
def test_configure(self):
d1 = self.getDoc()
d2 = self.getDoc()
d3 = self.getDoc()
we = GenericWorkflowEngine()
we.addManyCallbacks('*', [
m('mouse'),
[m('dog'), call_forward(1), m('cat'), m('puppy')],
m('horse'),
])
# process using defaults
we.process(d1)
r = 'one mouse dog cat puppy horse'.split()
# pass our own callback chooser
we.configure(callback_chooser=asterisk_chooser)
we.process(d2)
assert d1[0] == r
assert d2[0] == r
assert d1 == d2
# configure it wrongly
we.configure(callback_chooser='')
self.failUnlessRaises(Exception, we.process, d3)
assert d3 == self.getDoc()
def test_workflow01(self):
we0 = GenericWorkflowEngine()
we1 = GenericWorkflowEngine()
we2 = GenericWorkflowEngine()
d0 = self.getDoc()
d1 = self.getDoc()
d2 = self.getDoc()
we0.addManyCallbacks('*', [
m('mouse'),
[m('dog'), call_forward(1), m('cat'), m('puppy')],
m('horse'),
])
we1.setWorkflow([
m('mouse'),
[m('dog'), call_forward(1), m('cat'), m('puppy')],
m('horse'),
])
we2.addManyCallbacks('x', [
m('mouse'),
[m('dog'), call_forward(1), m('cat'), m('puppy')],
m('horse'),
])
we2.configure(callback_chooser=lambda o, e: e.getCallbacks('x'))
we0.process(d0)
we1.process(d1)
we2.process(d2)
assert d0 == d1
assert d0 == d2
def test_workflow01(self):
we0 = GenericWorkflowEngine()
we1 = GenericWorkflowEngine()
we2 = GenericWorkflowEngine()
d0 = self.getDoc()
d1 = self.getDoc()
d2 = self.getDoc()
we0.addManyCallbacks('*', [
m('mouse'),
[m('dog'), call_forward(1),
m('cat'), m('puppy')],
m('horse'),
])
we1.setWorkflow([
m('mouse'),
[m('dog'), call_forward(1),
m('cat'), m('puppy')],
m('horse'),
])
we2.addManyCallbacks('x', [
m('mouse'),
[m('dog'), call_forward(1),
m('cat'), m('puppy')],
m('horse'),
])
we2.configure(callback_chooser=lambda o, e: e.getCallbacks('x'))
we0.process(d0)
we1.process(d1)
we2.process(d2)
assert d0 == d1
assert d0 == d2
from workflow.engine import GenericWorkflowEngine
def flow1(object, eng):
print(object + 1)
def flow2(object, eng):
print(object + 2)
flowEngine = GenericWorkflowEngine()
taskFlow = [flow1, flow2]
flowEngine.callbacks.replace(taskFlow)
flowEngine.process([1])
def test_configure(self):
d1 = self.getDoc()
d2 = self.getDoc()
d3 = self.getDoc()
we = GenericWorkflowEngine()
we.addManyCallbacks('*', [
m('mouse'),
[m('dog'), call_forward(1),
m('cat'), m('puppy')],
m('horse'),
])
# process using defaults
we.process(d1)
r = 'one mouse dog cat puppy horse'.split()
# pass our own callback chooser
we.configure(callback_chooser=asterisk_chooser)
we.process(d2)
assert d1[0] == r
assert d2[0] == r
assert d1 == d2
# configure it wrongly
we.configure(callback_chooser='')
self.failUnlessRaises(Exception, we.process, d3)
assert d3 == self.getDoc()
class TestWorkflowEngine(object):
"""Tests using FakeTokens in place of strings"""
def setup_method(self, method):
self.key = '*'
self.wfe = GenericWorkflowEngine()
self.data = ['one', 'two', 'three', 'four', 'five']
self.tokens = [FakeToken(x, type='*') for x in self.data]
def teardown_method(self, method):
pass
@pytest.mark.parametrize("_,tokens,exception,exception_msg", (
("int", 49, WorkflowError, "not an iterable"),
("str", "hello", WorkflowError, "not an iterable"),
("object", object, WorkflowError, "not an iterable"),
))
def test_objects_are_of_bad_type(self, _, tokens, exception,
exception_msg):
with pytest.raises(exception) as exc_info:
self.wfe.process(tokens)
assert exception_msg in exc_info.value.args[0]
def test_empty_object_list_logs_warning(self):
assert hasattr(self.wfe, 'log')
self.wfe.log = mock.Mock()
self.wfe.callbacks.replace([lambda o, e: None])
self.wfe.process([])
self.wfe.log.warning.assert_called_once_with(
'List of objects is empty. Running workflow '
'on empty set has no effect.')
def test_current_taskname_resolution(self):
workflow = [m('test')]
self.wfe.callbacks.replace(workflow, self.key)
self.wfe.process(self.tokens)
assert self.wfe.current_taskname == 'string appender'
workflow = [lambda obj, eng: 1]
self.wfe.callbacks.replace(workflow, self.key)
self.wfe.process(self.tokens)
assert self.wfe.current_taskname == '<lambda>'
workflow = [
IF_ELSE(
lambda obj, eng: True,
[lambda obj, eng: 1],
[lambda obj, eng: 2],
)
]
self.wfe.callbacks.replace(workflow, self.key)
# This test will break if someone changes IF_ELSE. TODO: Mock
# Note: Python3 has much stronger introspection, thus the `.*`.
assert re.match(
r'\[<function IF_ELSE.* at 0x[0-f]+>, '
r'\[<function .*<lambda> at 0x[0-f]+>\], '
r'<function BREAK.* at 0x[0-f]+>, '
r'\[<function .*<lambda> at 0x[0-f]+>\]\]',
self.wfe.current_taskname)
def test_current_object_returns_correct_object(self):
self.wfe.callbacks.replace([halt_processing()])
assert self.wfe.current_object is None
with pytest.raises(HaltProcessing):
self.wfe.process(self.tokens)
assert self.wfe.current_object is self.tokens[0]
with pytest.raises(HaltProcessing):
self.wfe.restart('current', 'next')
assert self.wfe.current_object is self.tokens[1]
@pytest.mark.parametrize(
"_,callbacks,expected_result",
(('skips_forward_with_acceptable_increment', [
m('mouse'),
[m('dog'),
jump_call(2),
m('cat'),
m('puppy'),
m('python')],
m('horse'),
], 'mouse dog puppy python horse'),
('skips_forward_with_increment_that_is_too_large', [
m('mouse'),
[m('dog'),
jump_call(50),
m('cat'),
m('puppy'),
m('python')],
m('horse'),
], 'mouse dog horse'), ('jumps_forward_outside_of_nest', [
jump_call(3),
m('mouse'),
[m('dog'), m('cat'), m('puppy'),
m('python')],
m('horse'),
], 'horse'), ('skips_backwards_with_acceptable_decrement', [
m('mouse'),
[m('dog'), jump_call(-1),
m('cat'), m('puppy')],
m('horse'),
], 'mouse dog dog cat puppy horse'),
('skips_backwards_with_decrement_that_is_too_large', [
m('mouse'),
[m('dog'),
m('cat'),
jump_call(-50),
m('puppy'),
m('python')],
m('horse'),
], 'mouse dog cat dog cat puppy python horse'),
('skips_backwards_outside_of_nest', [
m('mouse'),
[m('dog'), m('cat'), m('puppy'),
m('python')],
m('horse'),
jump_call(-2)
], 'mouse dog cat puppy python horse dog cat puppy python horse')))
def test_jump_call(self, _, callbacks, expected_result):
self.wfe.callbacks.add_many(callbacks, self.key)
self.wfe.process(self.tokens)
t = get_first(self.tokens)
assert t == expected_result
# --------- complicated loop -----------
@pytest.mark.parametrize("_,workflow,expected_result", (('simple', [
m('mouse'),
[
m('dog'),
[m('cat'), m('puppy')],
[m('python'), [m('wasp'), m('leon')]],
m('horse'),
]
], 'mouse dog cat puppy python wasp leon horse'), ('with_nested_jumps', [
jump_call(2),
m('mouse'),
[
m('dog'),
[m('cat'), m('puppy')],
[m('python'), jump_call(-2), [m('wasp'), m('leon')]],
m('horse'),
]
], 'dog cat puppy python python wasp leon horse')))
def test_multi_nested_workflows(self, _, workflow, expected_result):
self.wfe.callbacks.add_many(workflow, self.key)
self.wfe.process(self.tokens)
t = get_first(self.tokens)
assert t == expected_result
@pytest.mark.parametrize("_,workflow,expected_result", (('simple', [
m('mouse'),
[
m('dog'),
[m('cat'), m('puppy')],
[m('python'), break_loop(), [m('wasp'), m('leon')]],
m('horse'),
]
], 'mouse dog cat puppy python horse'), ('break_loop_outside_of_nest', [
break_loop(),
m('mouse'),
[
m('dog'),
[m('cat'), m('puppy')],
[m('python'), [m('wasp'), m('leon')]],
m('horse'),
]
], None)))
def test_break_from_this_loop(self, _, workflow, expected_result):
self.wfe.callbacks.add_many(workflow, self.key)
self.wfe.process(self.tokens)
t = get_first(self.tokens)
assert t == expected_result
# ----------- StopProcessing --------------------------
def test_engine_immediatelly_stops(self):
self.wfe.callbacks.add_many([
stop_processing(),
m('mouse'),
[
m('dog'),
[m('cat'), m('puppy')],
[m('python'), [m('wasp'), m('leon')]],
m('horse'),
]
], self.key)
self.wfe.process(self.tokens)
t = get_first(self.tokens)
assert get_xth(self.tokens, 0) is None
assert get_xth(self.tokens, 1) is None
assert get_xth(self.tokens, 2) is None
def test_engine_stops_half_way_through(self):
self.wfe.callbacks.add_many([
m('mouse'),
[
m('dog'),
[m('cat'), m('puppy')],
[
m('python'),
stop_if_token_equals('four'), [m('wasp'),
m('leon')]
],
m('horse'),
]
], self.key)
self.wfe.process(self.tokens)
full_result = 'mouse dog cat puppy python wasp leon horse'
result_until_stop = 'mouse dog cat puppy python'
assert get_xth(self.tokens, 0) == full_result # 'one'
assert get_xth(self.tokens, 1) == full_result # 'two'
assert get_xth(self.tokens, 2) == full_result # 'three'
assert get_xth(self.tokens, 3) == result_until_stop # 'four'
assert get_xth(self.tokens, 4) is None # 'five', engine stopped
# ---------- jump_token -------------
def test_engine_moves_to_next_token(self):
self.wfe.callbacks.add_many([
m('mouse'),
[
m('dog'),
[m('cat'), m('puppy')],
[m('python'),
next_token(), [m('wasp'), m('leon')]],
m('horse'),
]
], self.key)
self.wfe.process(self.tokens)
result_until_next_token = 'mouse dog cat puppy python'
for i in range(5):
assert get_xth(self.tokens, i) == result_until_next_token
def test_workflow_09a(self):
self.wfe.callbacks.add_many([
m('mouse'),
[
m('dog'),
if_str_token_jump('four', -2),
[m('cat'), m('puppy')],
m('horse'),
]
], self.key)
self.wfe.process(self.tokens)
t = get_first(self.tokens)
r1 = 'mouse dog cat puppy horse' # one, five
r2 = 'mouse dog cat puppy horse mouse dog cat puppy horse' # two, three
r3 = 'mouse dog mouse dog cat puppy horse' # four
assert get_xth(self.tokens, 0) == r1
assert get_xth(self.tokens, 1) == r2
assert get_xth(self.tokens, 2) == r2
assert get_xth(self.tokens, 3) == r3
assert get_xth(self.tokens, 4) == r1
def test_workflow_09b(self):
self.wfe.callbacks.add_many([
m('mouse'),
[
m('dog'),
if_str_token_jump('two', 2),
[m('cat'), m('puppy')],
m('horse'),
]
], self.key)
self.wfe.process(self.tokens)
t = get_first(self.tokens)
r1 = 'mouse dog cat puppy horse' # one, four, five
r2 = 'mouse dog' # two
r3 = None # three
assert get_xth(self.tokens, 0) == r1
assert get_xth(self.tokens, 1) == r2
assert get_xth(self.tokens, 2) == r3
assert get_xth(self.tokens, 3) == r1
assert get_xth(self.tokens, 4) == r1
# ----------------- HaltProcessing --------------------
def test_50_halt_processing_mid_workflow(self):
other_wfe = GenericWorkflowEngine()
other_wfe.callbacks.add_many([
m('mouse'),
[
m('dog'),
[m('cat'), m('puppy')],
[m('python'), halt_processing()],
m('horse'),
]
], self.key)
with pytest.raises(HaltProcessing):
other_wfe.process(self.tokens)
t = get_first(self.tokens)
assert get_xth(self.tokens, 0) == 'mouse dog cat puppy python'
assert get_xth(self.tokens, 1) is None
assert get_xth(self.tokens, 2) is None
compl = 'mouse dog cat puppy python horse'
compl1 = 'mouse dog cat puppy python'
@pytest.mark.parametrize(
"obj,task,results",
(
('prev', 'prev', (compl + " python", compl1, None)),
('prev', 'current',
(compl, compl1, None)), # current task is to halt
('prev', 'next', (compl + " horse", compl1 + " " + compl1, None)),
('prev', 'first', (compl + " " + compl1, compl1, None)),
('current', 'prev', (compl, compl1 + " python", None)),
('current', 'current', (compl, compl1, None)),
('current', 'next', (compl, compl1 + " horse", compl1)),
('current', 'first', (compl, compl1 + " " + compl1, None)),
('next', 'prev', (compl, compl1, "python")),
('next', 'current', (compl, compl1, None)),
('next', 'next', (compl, compl1, "horse")),
('next', 'first', (compl, compl1, compl1)),
('first', 'prev', (compl + " python", compl1, None)),
('first', 'current',
(compl, compl1, None)), # current task is to halt
('first', 'next', (compl + " horse", compl1 + " " + compl1, None)),
('first', 'first', (compl + " " + compl1, compl1, None)),
))
def test_51_workflow_restart_after_halt(self, obj, task, results):
self.wfe.callbacks.add_many([
m('mouse'),
[
m('dog'),
[m('cat'), m('puppy')],
[m('python'), halt_processing()],
m('horse'),
]
], self.key)
with pytest.raises(HaltProcessing):
self.wfe.process(self.tokens)
assert get_xth(self.tokens, 0) == 'mouse dog cat puppy python'
assert get_xth(self.tokens, 1) is None
assert get_xth(self.tokens, 2) is None
# this should pick up from the point where we stopped
with pytest.raises(HaltProcessing):
self.wfe.restart('current', 'next')
assert get_xth(self.tokens, 0) == 'mouse dog cat puppy python horse'
assert get_xth(self.tokens, 1) == 'mouse dog cat puppy python'
assert get_xth(self.tokens, 2) is None
with pytest.raises(HaltProcessing):
self.wfe.restart(obj, task)
assert (get_xth(self.tokens, 0), get_xth(self.tokens,
1), get_xth(self.tokens,
2)) == results
def test_restart_accepts_new_objects(self):
workflow = [m('test')]
self.wfe.callbacks.replace(workflow, self.key)
self.wfe.process(self.tokens)
new_data = ['a', 'b', 'c', 'd', 'e']
new_tokens = [FakeToken(x, type='*') for x in new_data]
self.wfe.restart('first', 'first', objects=new_tokens)
assert self.wfe.objects == new_tokens
def test_has_completed(self):
self.wfe.callbacks.replace([
m('mouse'),
halt_processing(),
m('horse'),
])
assert self.wfe.has_completed is False
with pytest.raises(HaltProcessing):
self.wfe.process([self.tokens[0]])
assert self.wfe.has_completed is False
self.wfe.restart('current', 'next')
assert self.wfe.has_completed is True
def test_nested_workflow_halt(self):
other_wfe = GenericWorkflowEngine()
wfe = self.wfe
other_wfe.callbacks.add_many([
m('mouse'),
[
m('dog'),
[m('cat'), m('puppy')],
[m('python'), halt_processing()],
m('horse'),
]
], self.key)
wfe.callbacks.add_many([
m('mouse'),
[
m('dog'),
[m('cat'), m('puppy')],
[m('python'), lambda o, e: other_wfe.process(self.tokens)],
m('horse'),
]
], self.key)
with pytest.raises(HaltProcessing):
wfe.process(self.tokens)
t = get_first(self.tokens)
assert get_xth(
self.tokens,
0) == 'mouse dog cat puppy python mouse dog cat puppy python'
assert get_xth(self.tokens, 1) is None
assert get_xth(self.tokens, 2) is None
@pytest.mark.parametrize("callbacks,kwargs,result,exception", (
(
[
m('mouse'),
[
m('dog'),
[m('cat'), m('puppy')],
[m('python'), workflow_error()],
m('horse'),
]
],
{},
'mouse dog cat puppy python',
WorkflowError,
),
(
[
m('mouse'),
[
m('dog'),
[m('cat'), m('puppy')],
[m('python'), workflow_error()],
m('horse'),
]
],
{
'stop_on_error': False,
},
'mouse dog cat puppy python',
None,
),
(
[
m('mouse'),
[
m('dog'),
[m('cat'), m('puppy')],
[m('python'), halt_processing()],
m('horse'),
]
],
{
'stop_on_halt': False,
},
'mouse dog cat puppy python',
None,
),
))
def test_process_smash_through(self, callbacks, kwargs, result, exception):
self.wfe.callbacks.add_many(callbacks, self.key)
if exception:
with pytest.raises(exception):
self.wfe.process(self.tokens, **kwargs)
else:
self.wfe.process(self.tokens, **kwargs)
for idx, dummy in enumerate(self.tokens):
assert get_xth(self.tokens, idx) == result
def before_processing(objects, self):
"""Executed before processing the workflow."""
self.save(status=WorkflowStatus.RUNNING)
self.set_counter_initial(len(objects))
workflow_started.send(self)
GenericWorkflowEngine.before_processing(objects, self)
def run_workflow_3():
my_engine_3 = GenericWorkflowEngine()
my_engine_3.setWorkflow(my_workflow_3)
my_engine_3.process([[]])
def setup_method(self, method):
self.key = '*'
self.wfe = GenericWorkflowEngine()
self.data = ['one', 'two', 'three', 'four', 'five']
self.tokens = [FakeToken(x, type='*') for x in self.data]
def before_processing(objects, self):
"""Executed before processing the workflow."""
self.save(status=WorkflowStatus.RUNNING)
self.set_counter_initial(len(objects))
workflow_started.send(self)
GenericWorkflowEngine.before_processing(objects, self)
class TestWorkflowEngine(object):
"""Tests using FakeTokens in place of strings"""
def setup_method(self, method):
self.key = '*'
self.wfe = GenericWorkflowEngine()
self.data = ['one', 'two', 'three', 'four', 'five']
self.tokens = [FakeToken(x, type='*') for x in self.data]
def teardown_method(self, method):
pass
@pytest.mark.parametrize("_,tokens,exception,exception_msg", (
("int", 49, WorkflowError, "not an iterable"),
("str", "hello", WorkflowError, "not an iterable"),
("object", object, WorkflowError, "not an iterable"),
))
def test_objects_are_of_bad_type(self, _, tokens, exception, exception_msg):
with pytest.raises(exception) as exc_info:
self.wfe.process(tokens)
assert exception_msg in exc_info.value.args[0]
def test_empty_object_list_logs_warning(self):
assert hasattr(self.wfe, 'log')
self.wfe.log = mock.Mock()
self.wfe.callbacks.replace([lambda o, e: None])
self.wfe.process([])
self.wfe.log.warning.assert_called_once_with('List of objects is empty. Running workflow '
'on empty set has no effect.')
def test_current_taskname_resolution(self):
workflow = [m('test')]
self.wfe.callbacks.replace(workflow, self.key)
self.wfe.process(self.tokens)
assert self.wfe.current_taskname == 'string appender'
workflow = [lambda obj, eng: 1]
self.wfe.callbacks.replace(workflow, self.key)
self.wfe.process(self.tokens)
assert self.wfe.current_taskname == '<lambda>'
workflow = [
IF_ELSE(
lambda obj, eng: True,
[lambda obj, eng: 1],
[lambda obj, eng: 2],
)
]
self.wfe.callbacks.replace(workflow, self.key)
# This test will break if someone changes IF_ELSE. TODO: Mock
# Note: Python3 has much stronger introspection, thus the `.*`.
assert re.match(r'\[<function IF_ELSE.* at 0x[0-f]+>, '
r'\[<function .*<lambda> at 0x[0-f]+>\], '
r'<function BREAK.* at 0x[0-f]+>, '
r'\[<function .*<lambda> at 0x[0-f]+>\]\]',
self.wfe.current_taskname)
def test_current_object_returns_correct_object(self):
self.wfe.callbacks.replace([halt_processing()])
assert self.wfe.current_object is None
with pytest.raises(HaltProcessing):
self.wfe.process(self.tokens)
assert self.wfe.current_object is self.tokens[0]
with pytest.raises(HaltProcessing):
self.wfe.restart('current', 'next')
assert self.wfe.current_object is self.tokens[1]
@pytest.mark.parametrize("_,callbacks,expected_result", (
(
'skips_forward_with_acceptable_increment',
[
m('mouse'),
[m('dog'), jump_call(2), m('cat'), m('puppy'), m('python')],
m('horse'),
],
'mouse dog puppy python horse'
),
(
'skips_forward_with_increment_that_is_too_large',
[
m('mouse'),
[m('dog'), jump_call(50), m('cat'), m('puppy'), m('python')],
m('horse'),
],
'mouse dog horse'
),
(
'jumps_forward_outside_of_nest',
[
jump_call(3),
m('mouse'),
[m('dog'), m('cat'), m('puppy'), m('python')],
m('horse'),
],
'horse'
),
(
'skips_backwards_with_acceptable_decrement',
[
m('mouse'),
[m('dog'), jump_call(-1), m('cat'), m('puppy')],
m('horse'),
],
'mouse dog dog cat puppy horse'
),
(
'skips_backwards_with_decrement_that_is_too_large',
[
m('mouse'),
[m('dog'), m('cat'), jump_call(-50), m('puppy'), m('python')],
m('horse'),
],
'mouse dog cat dog cat puppy python horse'
),
(
'skips_backwards_outside_of_nest',
[
m('mouse'),
[m('dog'), m('cat'), m('puppy'), m('python')],
m('horse'),
jump_call(-2)
],
'mouse dog cat puppy python horse dog cat puppy python horse'
)
))
def test_jump_call(self, _, callbacks, expected_result):
self.wfe.callbacks.add_many(callbacks, self.key)
self.wfe.process(self.tokens)
t = get_first(self.tokens)
assert t == expected_result
# --------- complicated loop -----------
@pytest.mark.parametrize("_,workflow,expected_result", (
(
'simple',
[
m('mouse'),
[
m('dog'),
[m('cat'), m('puppy')],
[m('python'), [m('wasp'), m('leon')]],
m('horse'),
]
],
'mouse dog cat puppy python wasp leon horse'
),
(
'with_nested_jumps',
[
jump_call(2),
m('mouse'),
[
m('dog'),
[m('cat'), m('puppy')],
[m('python'), jump_call(-2), [m('wasp'), m('leon')]],
m('horse'),
]
],
'dog cat puppy python python wasp leon horse'
)
))
def test_multi_nested_workflows(self, _, workflow, expected_result):
self.wfe.callbacks.add_many(workflow, self.key)
self.wfe.process(self.tokens)
t = get_first(self.tokens)
assert t == expected_result
@pytest.mark.parametrize("_,workflow,expected_result", (
(
'simple',
[
m('mouse'),
[
m('dog'),
[m('cat'), m('puppy')],
[m('python'), break_loop(), [m('wasp'), m('leon')]],
m('horse'),
]
],
'mouse dog cat puppy python horse'
),
(
'break_loop_outside_of_nest',
[
break_loop(),
m('mouse'),
[
m('dog'),
[m('cat'), m('puppy')],
[m('python'), [m('wasp'), m('leon')]],
m('horse'),
]
],
None
)
))
def test_break_from_this_loop(self, _, workflow, expected_result):
self.wfe.callbacks.add_many(workflow, self.key)
self.wfe.process(self.tokens)
t = get_first(self.tokens)
assert t == expected_result
# ----------- StopProcessing --------------------------
def test_engine_immediatelly_stops(self):
self.wfe.callbacks.add_many([
stop_processing(),
m('mouse'),
[
m('dog'),
[m('cat'), m('puppy')],
[
m('python'),
[m('wasp'), m('leon')]
],
m('horse'),
]
], self.key)
self.wfe.process(self.tokens)
t = get_first(self.tokens)
assert get_xth(self.tokens, 0) is None
assert get_xth(self.tokens, 1) is None
assert get_xth(self.tokens, 2) is None
def test_engine_stops_half_way_through(self):
self.wfe.callbacks.add_many([
m('mouse'),
[
m('dog'),
[m('cat'), m('puppy')],
[m('python'), stop_if_token_equals('four'), [m('wasp'), m('leon')]],
m('horse'),
]
], self.key)
self.wfe.process(self.tokens)
full_result = 'mouse dog cat puppy python wasp leon horse'
result_until_stop = 'mouse dog cat puppy python'
assert get_xth(self.tokens, 0) == full_result # 'one'
assert get_xth(self.tokens, 1) == full_result # 'two'
assert get_xth(self.tokens, 2) == full_result # 'three'
assert get_xth(self.tokens, 3) == result_until_stop # 'four'
assert get_xth(self.tokens, 4) is None # 'five', engine stopped
# ---------- jump_token -------------
def test_engine_moves_to_next_token(self):
self.wfe.callbacks.add_many(
[
m('mouse'),
[
m('dog'),
[m('cat'), m('puppy')],
[m('python'), next_token(), [m('wasp'), m('leon')]],
m('horse'),
]
], self.key)
self.wfe.process(self.tokens)
result_until_next_token = 'mouse dog cat puppy python'
for i in range(5):
assert get_xth(self.tokens, i) == result_until_next_token
def test_workflow_09a(self):
self.wfe.callbacks.add_many([
m('mouse'),
[
m('dog'), if_str_token_jump('four', -2),
[m('cat'), m('puppy')],
m('horse'),
]
], self.key)
self.wfe.process(self.tokens)
t = get_first(self.tokens)
r1 = 'mouse dog cat puppy horse' # one, five
r2 = 'mouse dog cat puppy horse mouse dog cat puppy horse' # two, three
r3 = 'mouse dog mouse dog cat puppy horse' # four
assert get_xth(self.tokens, 0) == r1
assert get_xth(self.tokens, 1) == r2
assert get_xth(self.tokens, 2) == r2
assert get_xth(self.tokens, 3) == r3
assert get_xth(self.tokens, 4) == r1
def test_workflow_09b(self):
self.wfe.callbacks.add_many([
m('mouse'),
[
m('dog'),
if_str_token_jump('two', 2),
[m('cat'), m('puppy')],
m('horse'),
]
], self.key)
self.wfe.process(self.tokens)
t = get_first(self.tokens)
r1 = 'mouse dog cat puppy horse' # one, four, five
r2 = 'mouse dog' # two
r3 = None # three
assert get_xth(self.tokens, 0) == r1
assert get_xth(self.tokens, 1) == r2
assert get_xth(self.tokens, 2) == r3
assert get_xth(self.tokens, 3) == r1
assert get_xth(self.tokens, 4) == r1
# ----------------- HaltProcessing --------------------
def test_50_halt_processing_mid_workflow(self):
other_wfe = GenericWorkflowEngine()
other_wfe.callbacks.add_many([
m('mouse'),
[
m('dog'),
[m('cat'), m('puppy')],
[m('python'), halt_processing()],
m('horse'),
]
], self.key)
with pytest.raises(HaltProcessing):
other_wfe.process(self.tokens)
t = get_first(self.tokens)
assert get_xth(self.tokens, 0) == 'mouse dog cat puppy python'
assert get_xth(self.tokens, 1) is None
assert get_xth(self.tokens, 2) is None
compl = 'mouse dog cat puppy python horse'
compl1 = 'mouse dog cat puppy python'
@pytest.mark.parametrize("obj,task,results", (
('prev', 'prev', (compl + " python", compl1, None)),
('prev', 'current', (compl, compl1, None)), # current task is to halt
('prev', 'next', (compl + " horse", compl1 + " " + compl1, None)),
('prev', 'first', (compl + " " + compl1, compl1, None)),
('current', 'prev', (compl, compl1 + " python", None)),
('current', 'current', (compl, compl1, None)),
('current', 'next', (compl, compl1 + " horse", compl1)),
('current', 'first', (compl, compl1 + " " + compl1, None)),
('next', 'prev', (compl, compl1, "python")),
('next', 'current', (compl, compl1, None)),
('next', 'next', (compl, compl1, "horse")),
('next', 'first', (compl, compl1, compl1)),
('first', 'prev', (compl + " python", compl1, None)),
('first', 'current', (compl, compl1, None)), # current task is to halt
('first', 'next', (compl + " horse", compl1 + " " + compl1, None)),
('first', 'first', (compl + " " + compl1, compl1, None)),
))
def test_51_workflow_restart_after_halt(self, obj, task, results):
self.wfe.callbacks.add_many([
m('mouse'),
[
m('dog'),
[m('cat'), m('puppy')],
[m('python'), halt_processing()],
m('horse'),
]
], self.key)
with pytest.raises(HaltProcessing):
self.wfe.process(self.tokens)
assert get_xth(self.tokens, 0) == 'mouse dog cat puppy python'
assert get_xth(self.tokens, 1) is None
assert get_xth(self.tokens, 2) is None
# this should pick up from the point where we stopped
with pytest.raises(HaltProcessing):
self.wfe.restart('current', 'next')
assert get_xth(self.tokens, 0) == 'mouse dog cat puppy python horse'
assert get_xth(self.tokens, 1) == 'mouse dog cat puppy python'
assert get_xth(self.tokens, 2) is None
with pytest.raises(HaltProcessing):
self.wfe.restart(obj, task)
assert (get_xth(self.tokens, 0),
get_xth(self.tokens, 1),
get_xth(self.tokens, 2)) == results
def test_restart_accepts_new_objects(self):
workflow = [m('test')]
self.wfe.callbacks.replace(workflow, self.key)
self.wfe.process(self.tokens)
new_data = ['a', 'b', 'c', 'd', 'e']
new_tokens = [FakeToken(x, type='*') for x in new_data]
self.wfe.restart('first', 'first', objects=new_tokens)
assert self.wfe.objects == new_tokens
def test_has_completed(self):
self.wfe.callbacks.replace([
m('mouse'),
halt_processing(),
m('horse'),
])
assert self.wfe.has_completed is False
with pytest.raises(HaltProcessing):
self.wfe.process([self.tokens[0]])
assert self.wfe.has_completed is False
self.wfe.restart('current', 'next')
assert self.wfe.has_completed is True
def test_nested_workflow_halt(self):
other_wfe = GenericWorkflowEngine()
wfe = self.wfe
other_wfe.callbacks.add_many([
m('mouse'),
[
m('dog'),
[m('cat'), m('puppy')],
[m('python'), halt_processing()],
m('horse'),
]
], self.key)
wfe.callbacks.add_many([
m('mouse'),
[
m('dog'),
[m('cat'), m('puppy')],
[m('python'), lambda o, e: other_wfe.process(self.tokens)],
m('horse'),
]
], self.key)
with pytest.raises(HaltProcessing):
wfe.process(self.tokens)
t = get_first(self.tokens)
assert get_xth(self.tokens, 0) == 'mouse dog cat puppy python mouse dog cat puppy python'
assert get_xth(self.tokens, 1) is None
assert get_xth(self.tokens, 2) is None
@pytest.mark.parametrize("callbacks,kwargs,result,exception", (
(
[
m('mouse'),
[
m('dog'),
[m('cat'), m('puppy')],
[m('python'), workflow_error()],
m('horse'),
]
],
{},
'mouse dog cat puppy python',
WorkflowError,
),
(
[
m('mouse'),
[
m('dog'),
[m('cat'), m('puppy')],
[m('python'), workflow_error()],
m('horse'),
]
],
{
'stop_on_error': False,
},
'mouse dog cat puppy python',
None,
),
(
[
m('mouse'),
[
m('dog'),
[m('cat'), m('puppy')],
[m('python'), halt_processing()],
m('horse'),
]
],
{
'stop_on_halt': False,
},
'mouse dog cat puppy python',
None,
),
))
def test_process_smash_through(self, callbacks, kwargs, result, exception):
self.wfe.callbacks.add_many(callbacks, self.key)
if exception:
with pytest.raises(exception):
self.wfe.process(self.tokens, **kwargs)
else:
self.wfe.process(self.tokens, **kwargs)
for idx, dummy in enumerate(self.tokens):
assert get_xth(self.tokens, idx) == result
