def test_workflow01(self): class GenericWEWithXChooser(GenericWorkflowEngine): def callback_chooser(self, obj): return self.callbacks.get('x') we0 = GenericWorkflowEngine() we1 = GenericWorkflowEngine() we2 = GenericWEWithXChooser() we0.addManyCallbacks('*', [ obj_append('mouse'), [obj_append('dog'), jump_call(1), obj_append('cat'), obj_append('puppy')], obj_append('horse'), ]) we1.setWorkflow([ obj_append('mouse'), [obj_append('dog'), jump_call(1), obj_append('cat'), obj_append('puppy')], obj_append('horse'), ]) we2.addManyCallbacks('x', [ obj_append('mouse'), [obj_append('dog'), jump_call(1), obj_append('cat'), obj_append('puppy')], obj_append('horse'), ]) we0.process(self.d0) we1.process(self.d1) we2.process(self.d2) assert self.d0 == self.d1 assert self.d0 == self.d2
def before_processing(objects, self): """ Executed before processing the workflow. """ # 1. Save workflow (ourselves). if not self.db_obj.uuid: self.save() self.setCounterInitial(objects) self.log.info("[%s], %s has been started" % (str(datetime.now()), str(self.db_obj.name))) # 2. We want to save all the objects as version 0. for o in objects: same_workflow = \ o.db_obj.workflow_id and \ o.db_obj.workflow_id == self.db_obj.uuid if o.db_obj.id and same_workflow: # If object exists and we are running the same workflow, # do nothing continue # Set the current workflow id in the object if o.db_obj.version == CFG_OBJECT_VERSION.INITIAL \ and o.db_obj.workflow_id is not None: pass else: o.db_obj.workflow_id = self.db_obj.uuid o.save(CFG_OBJECT_VERSION.INITIAL) GenericWorkflowEngine.before_processing(objects, self)
def test_CHOICE03(self): we = GenericWorkflowEngine() doc = self.getDoc()[0:1] def arbiter(obj, eng): return obj[-1] we.setWorkflow([i('start'), cf.CHOICE(arbiter, ('bam', lambda obj, eng: obj.append('bom')), ('end', lambda obj, eng: obj.append('error')), ('bim', lambda obj, eng: obj.append('bam')), bom=(lambda obj, eng: obj.append('bum')), one=(lambda obj, eng: obj.append('bim')), bum=cf.STOP(), ), cf.TASK_JUMP_BWD(-1)]) we.process(doc) d = ' '.join(doc[0]) assert 'bim bam bom bum' in d assert 'error' not in d assert len(doc[0]) == 6
def test_PARALLEL_SPLIT01(self): we = GenericWorkflowEngine() doc = self.getDoc() we.setWorkflow([i('start'), cf.PARALLEL_SPLIT( printer('p1'), printer('p2'), printer('p3'), printer('p4'), printer('p5')), lambda o, e: time.sleep(.1), a('end') ]) we.process(doc) r = [' '.join(doc[x]) for x in range(len(doc))] assert doc[0][0] == 'start' assert doc[0][1] == 'one' assert doc[1][0] == 'start' assert doc[1][1] == 'two' # end must have been inserted while printers were running # mixed together with them all_pos = set() for x in range(len(doc)): pos = doc[x].index('end') assert pos > 2 assert pos < len(doc[x]) all_pos.add(pos)
def test_nested_workflow_halt(self): other_wfe = GenericWorkflowEngine() wfe = self.wfe other_wfe.callbacks.add_many([ m('mouse'), [ m('dog'), [m('cat'), m('puppy')], [m('python'), halt_processing()], m('horse'), ] ], self.key) wfe.callbacks.add_many([ m('mouse'), [ m('dog'), [m('cat'), m('puppy')], [m('python'), lambda o, e: other_wfe.process(self.tokens)], m('horse'), ] ], self.key) with pytest.raises(HaltProcessing): wfe.process(self.tokens) t = get_first(self.tokens) assert get_xth(self.tokens, 0) == 'mouse dog cat puppy python mouse dog cat puppy python' assert get_xth(self.tokens, 1) is None assert get_xth(self.tokens, 2) is None
def before_processing(objects, self): """ Executed before processing the workflow. """ # 1. Save workflow (ourselves). if not self.db_obj.uuid: self.save() self.set_counter_initial(len(objects)) self.log_info("Workflow has been started") # 2. We want to save all the objects as version 0. for o in objects: same_workflow = \ o.id_workflow and \ o.id_workflow == self.db_obj.uuid if o.id and same_workflow: # If object exists and we are running the same workflow, # do nothing o.log_info("object saving process : was already existing") continue # Set the current workflow id in the object if o.version == CFG_OBJECT_VERSION.INITIAL \ and o.id_workflow is not None: o.log_info("object saving process : was already existing") pass else: o.id_workflow = self.uuid o.save(o.version) GenericWorkflowEngine.before_processing(objects, self)
def test_nested_workflow_halt(self): other_wfe = GenericWorkflowEngine() wfe = self.wfe other_wfe.callbacks.add_many([ m('mouse'), [ m('dog'), [m('cat'), m('puppy')], [m('python'), halt_processing()], m('horse'), ] ], self.key) wfe.callbacks.add_many([ m('mouse'), [ m('dog'), [m('cat'), m('puppy')], [m('python'), lambda o, e: other_wfe.process(self.tokens)], m('horse'), ] ], self.key) with pytest.raises(HaltProcessing): wfe.process(self.tokens) t = get_first(self.tokens) assert get_xth( self.tokens, 0) == 'mouse dog cat puppy python mouse dog cat puppy python' assert get_xth(self.tokens, 1) is None assert get_xth(self.tokens, 2) is None
def automated_process(target): # pylint: disable=unused-argument """ Work out the current point in the automated workflow and process the next step. """ my_engine = GenericWorkflowEngine() my_engine.callbacks.replace( [task_add_repo, task_collect_nonwords, task_submit, task_cleanup]) my_engine.process([State(target)])
def test_init(self): # init with empty to full parameters we1 = GenericWorkflowEngine() callbacks = [ obj_append('mouse'), [obj_append('dog'), jump_call(1), obj_append('cat'), obj_append('puppy')], obj_append('horse'), ] we1.addManyCallbacks('*', deepcopy(callbacks)) we1.process(self.d1)
def run_workflow_2(): # Create the engine as in the previous example my_engine_2 = GenericWorkflowEngine() my_engine_2.setWorkflow(my_workflow_2) try: # Note how we don't need to keep a reference to our tokens - the engine # allows us to access them via `my_engine.objects` later. my_engine_2.process([[], [0,1], [0,1,0,1]]) except HaltProcessing: # Our engine was built to throw this exception every time an object is # completed. At this point we can inspect the object to decide what to # do next. In any case, we will ask it to move to the next object, # until it stops throwing the exception (which, in our case, means it # has finished with all objects). while True: try: # Restart the engine with the next object, starting from the # first task. my_engine_2.restart('next', 'first') except HaltProcessing: continue else: print "Done!", my_engine_2.objects break
def test_PARALLEL_SPLIT02(self): """TODO: this test is failing, but that is because sometimes it does not take into accounts threads being executed in random mannger""" we = GenericWorkflowEngine() doc = self.getDoc()[0:1] we.setWorkflow([i('start'), cf.PARALLEL_SPLIT( [ cf.PARALLEL_SPLIT( printer('p0'), printer('p0a'), cf.PARALLEL_SPLIT( printer('p0b'), printer('p0c') ), ), printer('xx') ], [ a('AAA'), printer('p2b') ], printer('p3'), [ a('p4a'), printer('p4b'), printer('p4c') ], [printer('p5'), cf.PARALLEL_SPLIT( printer('p6'), printer('p7'), [printer('p8a'), printer('p8b')], )]), a('end') ]) we.process(doc) # give threads time to finish time.sleep(2) assert doc[0][0] == 'start' assert doc[0][1] == 'one' # at least the fist object should have them all # print doc[0] for x in ['p0', 'p0a', 'p0b', 'p0c', 'xx', 'AAA', 'p2b', 'p3', 'p4a', 'p4b', 'p4c', 'p5', 'p6', 'p8a', 'p8b']: doc[0].index(x) # will fail if not present
def test_init(self): d1 = self.getDoc() d2 = self.getDoc() d3 = self.getDoc() # init with empty to full parameters we1 = GenericWorkflowEngine() we2 = GenericWorkflowEngine(callback_chooser=asterisk_chooser) try: we3 = GenericWorkflowEngine(processing_factory='x', callback_chooser='x', before_processing='x', after_processing='x') except Exception, msg: assert 'must be a callable' not in msg
def run_workflow(records, name, **kwargs): """Run the uploader workflow itself. :param records: List of tuples `(blob, json_record)` from :func:`translate` :param name: Name of the workflow to be run. :parma kwargs: Additional arguments to be used by the tasks of the workflow :returns: Typically the list of record Ids that has been process, although this value could be modify by the `post_tasks`. """ def _run_pre_post_tasks(tasks): """Helper function to run list of functions.""" for task in tasks: task(records, **kwargs) #FIXME: don't know why this is needed but IT IS! records = records[0] if name in cfg['UPLOADER_WORKFLOWS']: workflow = workflows.get(name) else: raise UploaderException("Workflow {0} not in UPLOADER_WORKFLOWS".format(name)) _run_pre_post_tasks(workflow.pre_tasks) wfe = WorkflowEngine() wfe.setWorkflow(workflow.tasks) wfe.setVar('options', kwargs) wfe.process(records) _run_pre_post_tasks(workflow.post_tasks) signals.uploader_finished.send(uploader_workflow=name, result=records, **kwargs) return records
def run_workflow(records, name, **kwargs): """Run the uploader workflow itself. :param records: List of tuples `(blob, json_record)` from :func:`translate` :param name: Name of the workflow to be run. :parma kwargs: Additional arguments to be used by the tasks of the workflow :returns: Typically the list of record Ids that has been process, although this value could be modify by the `post_tasks`. """ def _run_pre_post_tasks(tasks): """Helper function to run list of functions.""" for task in tasks: task(records, **kwargs) #FIXME: don't know why this is needed but IT IS! records = records[0] if name in cfg['UPLOADER_WORKFLOWS']: workflow = workflows.get(name) else: raise UploaderException( "Workflow {0} not in UPLOADER_WORKFLOWS".format(name)) _run_pre_post_tasks(workflow.pre_tasks) wfe = WorkflowEngine() wfe.setWorkflow(workflow.tasks) wfe.setVar('options', kwargs) wfe.process(records) _run_pre_post_tasks(workflow.post_tasks) signals.uploader_finished.send(uploader_workflow=name, result=records, **kwargs) return records
def test_RUN_WF02(self): """Test wfe is reinit=True - eng must not remember""" we = GenericWorkflowEngine() doc = self.getDoc()[0:1] we.callbacks.replace( [ i('start'), ut.RUN_WF( [ lambda obj, eng: obj.append('bom'), lambda obj, eng: obj.append('bam'), lambda obj, eng: obj.append('bum'), lambda obj, eng: obj.append('end'), lambda obj, eng: obj.append( eng.store.setdefault('eng-end', '')), e('eng-end', 'eng-end') ], data_connector=lambda obj, eng: [obj], outkey='#wfe', reinit=True ), ] ) we.process(doc) d = ' '.join(doc[0]) assert 'start' in d assert 'bom' in d assert 'bam' in d assert 'bum' in d assert 'end' in d assert 'eng-end' not in d # run the same thing again we.process(doc) d = ' '.join(doc[0]) assert 'start' in d assert d.count('bom') == 2 assert d.count('bam') == 2 assert d.count('bum') == 2 assert 'end' in d assert 'eng-end' not in d # it must not be present if reinit=True
def test_IF_ELSE02(self): we = GenericWorkflowEngine() doc = self.getDoc() we.setWorkflow([i('add'), cf.IF_ELSE(lambda o, e: o[1] == 'three', a('3'), a('other')) ]) we.process(doc) r = [' '.join(doc[x]) for x in range(len(doc))] assert r[0] == 'add one other' assert r[1] == 'add two other' assert r[2] == 'add three 3' assert r[3] == 'add four other' assert r[4] == 'add five other'
def test_PARALLEL_SPLIT03(self): we = GenericWorkflowEngine() doc = self.getDoc() we.setWorkflow([i('start'), cf.PARALLEL_SPLIT( [cf.IF(lambda obj, eng: 'jump-verified' in obj, a('error')), cf.PARALLEL_SPLIT( [cf.IF(lambda obj, eng: 'nasty-jump' in obj, [a('jump-ok'), lambda obj, eng: ('nasty-jump' in obj and obj.append('jump-verified'))]), cf.PARALLEL_SPLIT( a('ok-1'), a('ok-2'), cf.IF(lambda obj, eng: 'ok-3' not in obj, lambda obj, eng: (obj.append('ok-3') and eng.breakFromThisLoop())), a('ok-4')), a('xx'), lambda obj, eng: ('jump-verified' in obj and eng.breakFromThisLoop()), a('nasty-jump'), cf.TASK_JUMP_IF( lambda obj, eng: 'jump-verified' not in obj, -100)]), ], [a('AAA'), a('p2b')]), a('end') ]) we.process(doc) # give threads time to finish time.sleep(.5) d = doc[0] # at least the fist object should have them all # print doc[0] for x in ['nasty-jump', 'jump-verified', 'ok-3']: d.index(x) # will fail if not present assert d.count('ok-1') > 1
def interactive_old_word(state, jsonobj, word): """ Single word processing """ my_engine = GenericWorkflowEngine() my_engine.callbacks.replace( [check_websearch, is_nonword, is_typo, what_now]) newstate = NonwordState( interaction=state.interaction, target=state.target, word=word, details=jsonobj[word], repopath=state.repopath, nonword_delegate=state.nonword_delegate, ) try: my_engine.process([newstate]) except HaltProcessing: if newstate.done: return True return False
def test_SIMPLE_MERGE03(self): we = GenericWorkflowEngine() doc = self.getDoc()[0:1] we.setWorkflow([i('start'), cf.SIMPLE_MERGE( lambda obj, eng: obj.append('bom'), lambda obj, eng: obj.append('error'), lambda obj, eng: obj.append('bam'), lambda obj, eng: obj.append('bum'), lambda obj, eng: obj.append('end'), ), ]) we.process(doc) d = ' '.join(doc[0]) assert 'start' in d assert 'bom' in d assert 'error' not in d assert 'end' in d
def interactive_task_collect_nonwords( # pylint: disable=unused-argument reponame, target, nonword_delegate=None, nonstop=False): """ Saves nonwords until a typo is found """ if nonword_delegate is None: nonword_delegate = interactive_nonword_delegate(target) key = "repository_map" repository_map = get_json_value(key, {}) repodir = repository_map[reponame] repodirpath = Path(repodir) jsonpath = repodirpath / "spelling.json" if not jsonpath.is_file(): print(f"Unable to locate spelling at {jsonpath}", file=sys.stderr) return with io.open(jsonpath, "r", encoding="utf-8") as fobj: jsonobj = json.load(fobj) words = get_sorted_words(jsonobj) my_engine = GenericWorkflowEngine() my_engine.callbacks.replace( [check_websearch, is_nonword, is_typo, what_now]) for word in words: state = NonwordState( target=target, word=word, details=jsonobj[word], repopath=repodirpath, nonword_delegate=nonword_delegate, ) try: my_engine.process([state]) except HaltProcessing: if state.done and not nonstop: return print( f"{Fore.YELLOW}Completed checking all words for {reponame}!{Style.RESET_ALL}" )
def test_CHOICE03(self): we = GenericWorkflowEngine() doc = self.getDoc()[0:1] def arbiter(obj, eng): return obj[-1] we.setWorkflow([ i('start'), cf.CHOICE( arbiter, ('bam', lambda obj, eng: obj.append('bom')), ('end', lambda obj, eng: obj.append('error')), ('bim', lambda obj, eng: obj.append('bam')), bom=(lambda obj, eng: obj.append('bum')), one=(lambda obj, eng: obj.append('bim')), bum=cf.STOP(), ), cf.TASK_JUMP_BWD(-1) ]) we.process(doc) d = ' '.join(doc[0]) assert 'bim bam bom bum' in d assert 'error' not in d assert len(doc[0]) == 6
def test_IF_ELSE03(self): we = GenericWorkflowEngine() doc = self.getDoc() doc[3].append('4') def test(v): return lambda o, e: v in o we.setWorkflow([ i('add'), cf.IF_ELSE(test('three'), [ a('xxx'), cf.IF_ELSE(test('xxx'), [ a('6'), cf.IF_ELSE(test('6'), a('six'), (a('only-3s'), a('error'))) ], a('ok')) ], [ cf.IF_ELSE( test('4'), cf.IF_ELSE(test('four'), [a('44'), [[[a('forty')]]]], a('error')), a('not-four')) ]), a('end'), cf.IF_ELSE(test('error'), a('gosh!'), a('OK')) ]) we.process(doc) r = [' '.join(doc[x]) for x in range(len(doc))] assert r[0] == 'add one not-four end OK' assert r[1] == 'add two not-four end OK' assert r[2] == 'add three xxx 6 six end OK' assert r[3] == 'add four 4 44 forty end OK' assert r[4] == 'add five not-four end OK'
def run_workflow(message): jsonObj = json.loads(message) print "Workflow input header message %s" % (jsonObj['Header']) data = jsonObj['Body'] my_engine = GenericWorkflowEngine() my_engine.setWorkflow(workflow_model_1()) my_engine.process(data)
def test_PARALLEL_SPLIT01(self): we = GenericWorkflowEngine() doc = self.getDoc() we.setWorkflow([ i('start'), cf.PARALLEL_SPLIT(printer('p1'), printer('p2'), printer('p3'), printer('p4'), printer('p5')), lambda o, e: time.sleep(.1), a('end') ]) we.process(doc) r = [' '.join(doc[x]) for x in range(len(doc))] assert doc[0][0] == 'start' assert doc[0][1] == 'one' assert doc[1][0] == 'start' assert doc[1][1] == 'two' # end must have been inserted while printers were running # mixed together with them all_pos = set() for x in range(len(doc)): pos = doc[x].index('end') assert pos > 2 assert pos < len(doc[x]) all_pos.add(pos)
def test_configure(self): callbacks_list = [ obj_append('mouse'), [obj_append('dog'), jump_call(1), obj_append('cat'), obj_append('puppy')], obj_append('horse'), ] we = GenericWorkflowEngine() we.addManyCallbacks('*', callbacks_list) # process using defaults we.process(self.d1) r = 'one mouse dog cat puppy horse'.split() we = GenericWorkflowEngine() we.addManyCallbacks('*', callbacks_list) we.process(self.d2) assert self.d1[0] == r assert self.d2[0] == r assert self.d1 == self.d2
def run_workflow_1(): my_object0 = MyObject(0) my_object1 = MyObject(1) my_engine_1 = GenericWorkflowEngine() my_engine_1.setWorkflow(my_workflow_1) my_engine_1.process([my_object0, my_object1])
def test_IF_ELSE03(self): we = GenericWorkflowEngine() doc = self.getDoc() doc[3].append('4') def test(v): return lambda o, e: v in o we.setWorkflow([i('add'), cf.IF_ELSE( test('three'), [a('xxx'), cf.IF_ELSE(test('xxx'), [a('6'), cf.IF_ELSE( test('6'), a('six'), (a('only-3s'), a('error')))], a('ok'))], [cf.IF_ELSE( test('4'), cf.IF_ELSE(test('four'), [a('44'), [[[a('forty')]]]], a('error')), a('not-four'))]), a('end'), cf.IF_ELSE(test('error'), a('gosh!'), a('OK')) ]) we.process(doc) r = [' '.join(doc[x]) for x in range(len(doc))] assert r[0] == 'add one not-four end OK' assert r[1] == 'add two not-four end OK' assert r[2] == 'add three xxx 6 six end OK' assert r[3] == 'add four 4 44 forty end OK' assert r[4] == 'add five not-four end OK'
def test_init(self): d1 = self.getDoc() d2 = self.getDoc() d3 = self.getDoc() # init with empty to full parameters we1 = GenericWorkflowEngine() we2 = GenericWorkflowEngine(callback_chooser=asterisk_chooser) try: we3 = GenericWorkflowEngine(processing_factory='x', callback_chooser='x', before_processing='x', after_processing='x') except Exception as msg: assert 'must be a callable' in str(msg) try: we3 = GenericWorkflowEngine(callback_chooser=asterisk_chooser, after_processing='x') except Exception as msg: assert 'must be a callable' in str(msg) we1.addManyCallbacks('*', [ m('mouse'), [m('dog'), call_forward(1), m('cat'), m('puppy')], m('horse'), ]) we2.addManyCallbacks('*', [ m('mouse'), [m('dog'), call_forward(1), m('cat'), m('puppy')], m('horse'), ]) we1.process(d1) we2.process(d2)
def WHILE(cond, branch): """Keeps executing branch as long as the condition cond is True @var cond: callable, function that decides @var branch: block of functions to run [if=true] """ # quite often i passed a function, which results in errors if callable(branch): branch = (branch,) # we don't know what is hiding inside branch branch = tuple(engine._cleanUpCallables(branch)) def x(obj, eng): if not cond(obj, eng): eng.breakFromThisLoop() x.__name__ = 'WHILE' return [x, branch, TASK_JUMP_BWD(-(len(branch)+1))]
def WHILE(cond, branch): """Keeps executing branch as long as the condition cond is True :param cond: callable, function that decides :param branch: block of functions to run [if=true] """ # quite often i passed a function, which results in errors if callable(branch): branch = (branch, ) # we don't know what is hiding inside branch branch = tuple(engine._cleanUpCallables(branch)) def x(obj, eng): if not cond(obj, eng): eng.breakFromThisLoop() x.__name__ = 'WHILE' return [x, branch, TASK_JUMP_BWD(-(len(branch) + 1))]
def test_RUN_WF01(self): """Test wfe is reinit=False, eng must remember previous invocations""" we = GenericWorkflowEngine() doc = self.getDoc()[0:1] we.setWorkflow([ i('start'), ut.RUN_WF( [ lambda obj, eng: obj.append('bom'), lambda obj, eng: obj.append('bam'), lambda obj, eng: obj.append('bum'), lambda obj, eng: obj.append('end'), lambda obj, eng: obj. append(eng.store.setdefault('eng-end', '')), e('eng-end', 'eng-end') ], data_connector=lambda obj, eng: [obj], outkey='#wfe', ), ]) we.process(doc) d = ' '.join(doc[0]) assert 'start' in d assert 'bom' in d assert 'bam' in d assert 'bum' in d assert 'end' in d assert 'eng-end' not in d # run the same thing again we.process(doc) d = ' '.join(doc[0]) assert 'start' in d assert d.count('bom') == 2 assert d.count('bam') == 2 assert d.count('bum') == 2 assert 'end' in d assert 'eng-end' in d # now it must be present
def test_RUN_WF02(self): """Test wfe is reinit=True - eng must not remember""" we = GenericWorkflowEngine() doc = self.getDoc()[0:1] we.setWorkflow([ i('start'), ut.RUN_WF([ lambda obj, eng: obj.append('bom'), lambda obj, eng: obj.append('bam'), lambda obj, eng: obj.append('bum'), lambda obj, eng: obj.append('end'), lambda obj, eng: obj.append(eng.getVar('eng-end', '')), e('eng-end', 'eng-end') ], data_connector=lambda obj, eng: [obj], outkey='#wfe', reinit=True), ]) we.process(doc) d = ' '.join(doc[0]) assert 'start' in d assert 'bom' in d assert 'bam' in d assert 'bum' in d assert 'end' in d assert 'eng-end' not in d # run the same thing again we.process(doc) d = ' '.join(doc[0]) assert 'start' in d assert d.count('bom') == 2 assert d.count('bam') == 2 assert d.count('bum') == 2 assert 'end' in d assert 'eng-end' not in d # it must not be present if reinit=True
def test_RUN_WF01(self): """Test wfe is reinit=False, eng must remember previous invocations""" we = GenericWorkflowEngine() doc = self.getDoc()[0:1] we.setWorkflow( [ i('start'), ut.RUN_WF( [ lambda obj, eng: obj.append('bom'), lambda obj, eng: obj.append('bam'), lambda obj, eng: obj.append('bum'), lambda obj, eng: obj.append('end'), lambda obj, eng: obj.append( eng.store.setdefault('eng-end', '')), e('eng-end', 'eng-end') ], data_connector=lambda obj, eng: [obj], outkey='#wfe', ), ] ) we.process(doc) d = ' '.join(doc[0]) assert 'start' in d assert 'bom' in d assert 'bam' in d assert 'bum' in d assert 'end' in d assert 'eng-end' not in d # run the same thing again we.process(doc) d = ' '.join(doc[0]) assert 'start' in d assert d.count('bom') == 2 assert d.count('bam') == 2 assert d.count('bum') == 2 assert 'end' in d assert 'eng-end' in d # now it must be present
class TestGenericWorkflowEngine(unittest.TestCase): """Tests of the WE interface""" def setUp(self): self.key = '*' def tearDown(self): pass def getDoc(self, val=None): if val: return [[x] for x in val.split()] return [[x] for x in u"one two three four five".split()] def addTestCallbacks(self, no, eng): if type == 1: eng.addManyCallbacks() # --------- initialization --------------- def test_init(self): d1 = self.getDoc() d2 = self.getDoc() d3 = self.getDoc() # init with empty to full parameters we1 = GenericWorkflowEngine() we2 = GenericWorkflowEngine(callback_chooser=asterisk_chooser) try: we3 = GenericWorkflowEngine(processing_factory='x', callback_chooser='x', before_processing='x', after_processing='x') except Exception, msg: assert 'must be a callable' not in msg try: we3 = GenericWorkflowEngine(callback_chooser=asterisk_chooser, after_processing='x') except Exception, msg: assert 'must be a callable' not in msg
def test_IF_ELSE02(self): we = GenericWorkflowEngine() doc = self.getDoc() we.setWorkflow([ i('add'), cf.IF_ELSE(lambda o, e: o[1] == 'three', a('3'), a('other')) ]) we.process(doc) r = [' '.join(doc[x]) for x in range(len(doc))] assert r[0] == 'add one other' assert r[1] == 'add two other' assert r[2] == 'add three 3' assert r[3] == 'add four other' assert r[4] == 'add five other'
def test_PARALLEL_SPLIT02(self): """TODO: this test is failing, but that is because sometimes it does not take into accounts threads being executed in random mannger""" we = GenericWorkflowEngine() doc = self.getDoc()[0:1] we.setWorkflow([ i('start'), cf.PARALLEL_SPLIT( [ cf.PARALLEL_SPLIT( printer('p0'), printer('p0a'), cf.PARALLEL_SPLIT(printer('p0b'), printer('p0c')), ), printer('xx') ], [a('AAA'), printer('p2b')], printer('p3'), [a('p4a'), printer('p4b'), printer('p4c')], [ printer('p5'), cf.PARALLEL_SPLIT( printer('p6'), printer('p7'), [printer('p8a'), printer('p8b')], ) ]), a('end') ]) we.process(doc) # give threads time to finish time.sleep(2) assert doc[0][0] == 'start' assert doc[0][1] == 'one' # at least the fist object should have them all # print doc[0] for x in [ 'p0', 'p0a', 'p0b', 'p0c', 'xx', 'AAA', 'p2b', 'p3', 'p4a', 'p4b', 'p4c', 'p5', 'p6', 'p8a', 'p8b' ]: doc[0].index(x) # will fail if not present
def test_init(self): # init with empty to full parameters we1 = GenericWorkflowEngine() callbacks = [ obj_append('mouse'), [ obj_append('dog'), jump_call(1), obj_append('cat'), obj_append('puppy') ], obj_append('horse'), ] we1.addManyCallbacks('*', deepcopy(callbacks)) we1.process(self.d1)
def test_PARALLEL_SPLIT03(self): we = GenericWorkflowEngine() doc = self.getDoc() we.setWorkflow([ i('start'), cf.PARALLEL_SPLIT([ cf.IF(lambda obj, eng: 'jump-verified' in obj, a('error')), cf.PARALLEL_SPLIT([ cf.IF(lambda obj, eng: 'nasty-jump' in obj, [ a('jump-ok'), lambda obj, eng: ('nasty-jump' in obj and obj.append('jump-verified')) ]), cf.PARALLEL_SPLIT( a('ok-1'), a('ok-2'), cf.IF( lambda obj, eng: 'ok-3' not in obj, lambda obj, eng: (obj.append('ok-3') and eng.breakFromThisLoop())), a('ok-4')), a('xx'), lambda obj, eng: ('jump-verified' in obj and eng.breakFromThisLoop()), a('nasty-jump'), cf.TASK_JUMP_IF( lambda obj, eng: 'jump-verified' not in obj, -100) ]), ], [a('AAA'), a('p2b')]), a('end') ]) we.process(doc) # give threads time to finish time.sleep(.5) d = doc[0] # at least the fist object should have them all # print doc[0] for x in ['nasty-jump', 'jump-verified', 'ok-3']: d.index(x) # will fail if not present assert d.count('ok-1') > 1
def setup_method(self, method): self.key = '*' self.wfe = GenericWorkflowEngine() self.data = ['one', 'two', 'three', 'four', 'five'] self.tokens = [FakeToken(x, type='*') for x in self.data]
def test_configure(self): d1 = self.getDoc() d2 = self.getDoc() d3 = self.getDoc() we = GenericWorkflowEngine() we.addManyCallbacks('*', [ m('mouse'), [m('dog'), call_forward(1), m('cat'), m('puppy')], m('horse'), ]) # process using defaults we.process(d1) r = 'one mouse dog cat puppy horse'.split() # pass our own callback chooser we.configure(callback_chooser=asterisk_chooser) we.process(d2) assert d1[0] == r assert d2[0] == r assert d1 == d2 # configure it wrongly we.configure(callback_chooser='') self.failUnlessRaises(Exception, we.process, d3) assert d3 == self.getDoc()
def test_workflow01(self): we0 = GenericWorkflowEngine() we1 = GenericWorkflowEngine() we2 = GenericWorkflowEngine() d0 = self.getDoc() d1 = self.getDoc() d2 = self.getDoc() we0.addManyCallbacks('*', [ m('mouse'), [m('dog'), call_forward(1), m('cat'), m('puppy')], m('horse'), ]) we1.setWorkflow([ m('mouse'), [m('dog'), call_forward(1), m('cat'), m('puppy')], m('horse'), ]) we2.addManyCallbacks('x', [ m('mouse'), [m('dog'), call_forward(1), m('cat'), m('puppy')], m('horse'), ]) we2.configure(callback_chooser=lambda o, e: e.getCallbacks('x')) we0.process(d0) we1.process(d1) we2.process(d2) assert d0 == d1 assert d0 == d2
from workflow.engine import GenericWorkflowEngine def flow1(object, eng): print(object + 1) def flow2(object, eng): print(object + 2) flowEngine = GenericWorkflowEngine() taskFlow = [flow1, flow2] flowEngine.callbacks.replace(taskFlow) flowEngine.process([1])
class TestWorkflowEngine(object): """Tests using FakeTokens in place of strings""" def setup_method(self, method): self.key = '*' self.wfe = GenericWorkflowEngine() self.data = ['one', 'two', 'three', 'four', 'five'] self.tokens = [FakeToken(x, type='*') for x in self.data] def teardown_method(self, method): pass @pytest.mark.parametrize("_,tokens,exception,exception_msg", ( ("int", 49, WorkflowError, "not an iterable"), ("str", "hello", WorkflowError, "not an iterable"), ("object", object, WorkflowError, "not an iterable"), )) def test_objects_are_of_bad_type(self, _, tokens, exception, exception_msg): with pytest.raises(exception) as exc_info: self.wfe.process(tokens) assert exception_msg in exc_info.value.args[0] def test_empty_object_list_logs_warning(self): assert hasattr(self.wfe, 'log') self.wfe.log = mock.Mock() self.wfe.callbacks.replace([lambda o, e: None]) self.wfe.process([]) self.wfe.log.warning.assert_called_once_with( 'List of objects is empty. Running workflow ' 'on empty set has no effect.') def test_current_taskname_resolution(self): workflow = [m('test')] self.wfe.callbacks.replace(workflow, self.key) self.wfe.process(self.tokens) assert self.wfe.current_taskname == 'string appender' workflow = [lambda obj, eng: 1] self.wfe.callbacks.replace(workflow, self.key) self.wfe.process(self.tokens) assert self.wfe.current_taskname == '<lambda>' workflow = [ IF_ELSE( lambda obj, eng: True, [lambda obj, eng: 1], [lambda obj, eng: 2], ) ] self.wfe.callbacks.replace(workflow, self.key) # This test will break if someone changes IF_ELSE. TODO: Mock # Note: Python3 has much stronger introspection, thus the `.*`. assert re.match( r'\[<function IF_ELSE.* at 0x[0-f]+>, ' r'\[<function .*<lambda> at 0x[0-f]+>\], ' r'<function BREAK.* at 0x[0-f]+>, ' r'\[<function .*<lambda> at 0x[0-f]+>\]\]', self.wfe.current_taskname) def test_current_object_returns_correct_object(self): self.wfe.callbacks.replace([halt_processing()]) assert self.wfe.current_object is None with pytest.raises(HaltProcessing): self.wfe.process(self.tokens) assert self.wfe.current_object is self.tokens[0] with pytest.raises(HaltProcessing): self.wfe.restart('current', 'next') assert self.wfe.current_object is self.tokens[1] @pytest.mark.parametrize( "_,callbacks,expected_result", (('skips_forward_with_acceptable_increment', [ m('mouse'), [m('dog'), jump_call(2), m('cat'), m('puppy'), m('python')], m('horse'), ], 'mouse dog puppy python horse'), ('skips_forward_with_increment_that_is_too_large', [ m('mouse'), [m('dog'), jump_call(50), m('cat'), m('puppy'), m('python')], m('horse'), ], 'mouse dog horse'), ('jumps_forward_outside_of_nest', [ jump_call(3), m('mouse'), [m('dog'), m('cat'), m('puppy'), m('python')], m('horse'), ], 'horse'), ('skips_backwards_with_acceptable_decrement', [ m('mouse'), [m('dog'), jump_call(-1), m('cat'), m('puppy')], m('horse'), ], 'mouse dog dog cat puppy horse'), ('skips_backwards_with_decrement_that_is_too_large', [ m('mouse'), [m('dog'), m('cat'), jump_call(-50), m('puppy'), m('python')], m('horse'), ], 'mouse dog cat dog cat puppy python horse'), ('skips_backwards_outside_of_nest', [ m('mouse'), [m('dog'), m('cat'), m('puppy'), m('python')], m('horse'), jump_call(-2) ], 'mouse dog cat puppy python horse dog cat puppy python horse'))) def test_jump_call(self, _, callbacks, expected_result): self.wfe.callbacks.add_many(callbacks, self.key) self.wfe.process(self.tokens) t = get_first(self.tokens) assert t == expected_result # --------- complicated loop ----------- @pytest.mark.parametrize("_,workflow,expected_result", (('simple', [ m('mouse'), [ m('dog'), [m('cat'), m('puppy')], [m('python'), [m('wasp'), m('leon')]], m('horse'), ] ], 'mouse dog cat puppy python wasp leon horse'), ('with_nested_jumps', [ jump_call(2), m('mouse'), [ m('dog'), [m('cat'), m('puppy')], [m('python'), jump_call(-2), [m('wasp'), m('leon')]], m('horse'), ] ], 'dog cat puppy python python wasp leon horse'))) def test_multi_nested_workflows(self, _, workflow, expected_result): self.wfe.callbacks.add_many(workflow, self.key) self.wfe.process(self.tokens) t = get_first(self.tokens) assert t == expected_result @pytest.mark.parametrize("_,workflow,expected_result", (('simple', [ m('mouse'), [ m('dog'), [m('cat'), m('puppy')], [m('python'), break_loop(), [m('wasp'), m('leon')]], m('horse'), ] ], 'mouse dog cat puppy python horse'), ('break_loop_outside_of_nest', [ break_loop(), m('mouse'), [ m('dog'), [m('cat'), m('puppy')], [m('python'), [m('wasp'), m('leon')]], m('horse'), ] ], None))) def test_break_from_this_loop(self, _, workflow, expected_result): self.wfe.callbacks.add_many(workflow, self.key) self.wfe.process(self.tokens) t = get_first(self.tokens) assert t == expected_result # ----------- StopProcessing -------------------------- def test_engine_immediatelly_stops(self): self.wfe.callbacks.add_many([ stop_processing(), m('mouse'), [ m('dog'), [m('cat'), m('puppy')], [m('python'), [m('wasp'), m('leon')]], m('horse'), ] ], self.key) self.wfe.process(self.tokens) t = get_first(self.tokens) assert get_xth(self.tokens, 0) is None assert get_xth(self.tokens, 1) is None assert get_xth(self.tokens, 2) is None def test_engine_stops_half_way_through(self): self.wfe.callbacks.add_many([ m('mouse'), [ m('dog'), [m('cat'), m('puppy')], [ m('python'), stop_if_token_equals('four'), [m('wasp'), m('leon')] ], m('horse'), ] ], self.key) self.wfe.process(self.tokens) full_result = 'mouse dog cat puppy python wasp leon horse' result_until_stop = 'mouse dog cat puppy python' assert get_xth(self.tokens, 0) == full_result # 'one' assert get_xth(self.tokens, 1) == full_result # 'two' assert get_xth(self.tokens, 2) == full_result # 'three' assert get_xth(self.tokens, 3) == result_until_stop # 'four' assert get_xth(self.tokens, 4) is None # 'five', engine stopped # ---------- jump_token ------------- def test_engine_moves_to_next_token(self): self.wfe.callbacks.add_many([ m('mouse'), [ m('dog'), [m('cat'), m('puppy')], [m('python'), next_token(), [m('wasp'), m('leon')]], m('horse'), ] ], self.key) self.wfe.process(self.tokens) result_until_next_token = 'mouse dog cat puppy python' for i in range(5): assert get_xth(self.tokens, i) == result_until_next_token def test_workflow_09a(self): self.wfe.callbacks.add_many([ m('mouse'), [ m('dog'), if_str_token_jump('four', -2), [m('cat'), m('puppy')], m('horse'), ] ], self.key) self.wfe.process(self.tokens) t = get_first(self.tokens) r1 = 'mouse dog cat puppy horse' # one, five r2 = 'mouse dog cat puppy horse mouse dog cat puppy horse' # two, three r3 = 'mouse dog mouse dog cat puppy horse' # four assert get_xth(self.tokens, 0) == r1 assert get_xth(self.tokens, 1) == r2 assert get_xth(self.tokens, 2) == r2 assert get_xth(self.tokens, 3) == r3 assert get_xth(self.tokens, 4) == r1 def test_workflow_09b(self): self.wfe.callbacks.add_many([ m('mouse'), [ m('dog'), if_str_token_jump('two', 2), [m('cat'), m('puppy')], m('horse'), ] ], self.key) self.wfe.process(self.tokens) t = get_first(self.tokens) r1 = 'mouse dog cat puppy horse' # one, four, five r2 = 'mouse dog' # two r3 = None # three assert get_xth(self.tokens, 0) == r1 assert get_xth(self.tokens, 1) == r2 assert get_xth(self.tokens, 2) == r3 assert get_xth(self.tokens, 3) == r1 assert get_xth(self.tokens, 4) == r1 # ----------------- HaltProcessing -------------------- def test_50_halt_processing_mid_workflow(self): other_wfe = GenericWorkflowEngine() other_wfe.callbacks.add_many([ m('mouse'), [ m('dog'), [m('cat'), m('puppy')], [m('python'), halt_processing()], m('horse'), ] ], self.key) with pytest.raises(HaltProcessing): other_wfe.process(self.tokens) t = get_first(self.tokens) assert get_xth(self.tokens, 0) == 'mouse dog cat puppy python' assert get_xth(self.tokens, 1) is None assert get_xth(self.tokens, 2) is None compl = 'mouse dog cat puppy python horse' compl1 = 'mouse dog cat puppy python' @pytest.mark.parametrize( "obj,task,results", ( ('prev', 'prev', (compl + " python", compl1, None)), ('prev', 'current', (compl, compl1, None)), # current task is to halt ('prev', 'next', (compl + " horse", compl1 + " " + compl1, None)), ('prev', 'first', (compl + " " + compl1, compl1, None)), ('current', 'prev', (compl, compl1 + " python", None)), ('current', 'current', (compl, compl1, None)), ('current', 'next', (compl, compl1 + " horse", compl1)), ('current', 'first', (compl, compl1 + " " + compl1, None)), ('next', 'prev', (compl, compl1, "python")), ('next', 'current', (compl, compl1, None)), ('next', 'next', (compl, compl1, "horse")), ('next', 'first', (compl, compl1, compl1)), ('first', 'prev', (compl + " python", compl1, None)), ('first', 'current', (compl, compl1, None)), # current task is to halt ('first', 'next', (compl + " horse", compl1 + " " + compl1, None)), ('first', 'first', (compl + " " + compl1, compl1, None)), )) def test_51_workflow_restart_after_halt(self, obj, task, results): self.wfe.callbacks.add_many([ m('mouse'), [ m('dog'), [m('cat'), m('puppy')], [m('python'), halt_processing()], m('horse'), ] ], self.key) with pytest.raises(HaltProcessing): self.wfe.process(self.tokens) assert get_xth(self.tokens, 0) == 'mouse dog cat puppy python' assert get_xth(self.tokens, 1) is None assert get_xth(self.tokens, 2) is None # this should pick up from the point where we stopped with pytest.raises(HaltProcessing): self.wfe.restart('current', 'next') assert get_xth(self.tokens, 0) == 'mouse dog cat puppy python horse' assert get_xth(self.tokens, 1) == 'mouse dog cat puppy python' assert get_xth(self.tokens, 2) is None with pytest.raises(HaltProcessing): self.wfe.restart(obj, task) assert (get_xth(self.tokens, 0), get_xth(self.tokens, 1), get_xth(self.tokens, 2)) == results def test_restart_accepts_new_objects(self): workflow = [m('test')] self.wfe.callbacks.replace(workflow, self.key) self.wfe.process(self.tokens) new_data = ['a', 'b', 'c', 'd', 'e'] new_tokens = [FakeToken(x, type='*') for x in new_data] self.wfe.restart('first', 'first', objects=new_tokens) assert self.wfe.objects == new_tokens def test_has_completed(self): self.wfe.callbacks.replace([ m('mouse'), halt_processing(), m('horse'), ]) assert self.wfe.has_completed is False with pytest.raises(HaltProcessing): self.wfe.process([self.tokens[0]]) assert self.wfe.has_completed is False self.wfe.restart('current', 'next') assert self.wfe.has_completed is True def test_nested_workflow_halt(self): other_wfe = GenericWorkflowEngine() wfe = self.wfe other_wfe.callbacks.add_many([ m('mouse'), [ m('dog'), [m('cat'), m('puppy')], [m('python'), halt_processing()], m('horse'), ] ], self.key) wfe.callbacks.add_many([ m('mouse'), [ m('dog'), [m('cat'), m('puppy')], [m('python'), lambda o, e: other_wfe.process(self.tokens)], m('horse'), ] ], self.key) with pytest.raises(HaltProcessing): wfe.process(self.tokens) t = get_first(self.tokens) assert get_xth( self.tokens, 0) == 'mouse dog cat puppy python mouse dog cat puppy python' assert get_xth(self.tokens, 1) is None assert get_xth(self.tokens, 2) is None @pytest.mark.parametrize("callbacks,kwargs,result,exception", ( ( [ m('mouse'), [ m('dog'), [m('cat'), m('puppy')], [m('python'), workflow_error()], m('horse'), ] ], {}, 'mouse dog cat puppy python', WorkflowError, ), ( [ m('mouse'), [ m('dog'), [m('cat'), m('puppy')], [m('python'), workflow_error()], m('horse'), ] ], { 'stop_on_error': False, }, 'mouse dog cat puppy python', None, ), ( [ m('mouse'), [ m('dog'), [m('cat'), m('puppy')], [m('python'), halt_processing()], m('horse'), ] ], { 'stop_on_halt': False, }, 'mouse dog cat puppy python', None, ), )) def test_process_smash_through(self, callbacks, kwargs, result, exception): self.wfe.callbacks.add_many(callbacks, self.key) if exception: with pytest.raises(exception): self.wfe.process(self.tokens, **kwargs) else: self.wfe.process(self.tokens, **kwargs) for idx, dummy in enumerate(self.tokens): assert get_xth(self.tokens, idx) == result
def before_processing(objects, self): """Executed before processing the workflow.""" self.save(status=WorkflowStatus.RUNNING) self.set_counter_initial(len(objects)) workflow_started.send(self) GenericWorkflowEngine.before_processing(objects, self)
def run_workflow_3(): my_engine_3 = GenericWorkflowEngine() my_engine_3.setWorkflow(my_workflow_3) my_engine_3.process([[]])
class TestWorkflowEngine(object): """Tests using FakeTokens in place of strings""" def setup_method(self, method): self.key = '*' self.wfe = GenericWorkflowEngine() self.data = ['one', 'two', 'three', 'four', 'five'] self.tokens = [FakeToken(x, type='*') for x in self.data] def teardown_method(self, method): pass @pytest.mark.parametrize("_,tokens,exception,exception_msg", ( ("int", 49, WorkflowError, "not an iterable"), ("str", "hello", WorkflowError, "not an iterable"), ("object", object, WorkflowError, "not an iterable"), )) def test_objects_are_of_bad_type(self, _, tokens, exception, exception_msg): with pytest.raises(exception) as exc_info: self.wfe.process(tokens) assert exception_msg in exc_info.value.args[0] def test_empty_object_list_logs_warning(self): assert hasattr(self.wfe, 'log') self.wfe.log = mock.Mock() self.wfe.callbacks.replace([lambda o, e: None]) self.wfe.process([]) self.wfe.log.warning.assert_called_once_with('List of objects is empty. Running workflow ' 'on empty set has no effect.') def test_current_taskname_resolution(self): workflow = [m('test')] self.wfe.callbacks.replace(workflow, self.key) self.wfe.process(self.tokens) assert self.wfe.current_taskname == 'string appender' workflow = [lambda obj, eng: 1] self.wfe.callbacks.replace(workflow, self.key) self.wfe.process(self.tokens) assert self.wfe.current_taskname == '<lambda>' workflow = [ IF_ELSE( lambda obj, eng: True, [lambda obj, eng: 1], [lambda obj, eng: 2], ) ] self.wfe.callbacks.replace(workflow, self.key) # This test will break if someone changes IF_ELSE. TODO: Mock # Note: Python3 has much stronger introspection, thus the `.*`. assert re.match(r'\[<function IF_ELSE.* at 0x[0-f]+>, ' r'\[<function .*<lambda> at 0x[0-f]+>\], ' r'<function BREAK.* at 0x[0-f]+>, ' r'\[<function .*<lambda> at 0x[0-f]+>\]\]', self.wfe.current_taskname) def test_current_object_returns_correct_object(self): self.wfe.callbacks.replace([halt_processing()]) assert self.wfe.current_object is None with pytest.raises(HaltProcessing): self.wfe.process(self.tokens) assert self.wfe.current_object is self.tokens[0] with pytest.raises(HaltProcessing): self.wfe.restart('current', 'next') assert self.wfe.current_object is self.tokens[1] @pytest.mark.parametrize("_,callbacks,expected_result", ( ( 'skips_forward_with_acceptable_increment', [ m('mouse'), [m('dog'), jump_call(2), m('cat'), m('puppy'), m('python')], m('horse'), ], 'mouse dog puppy python horse' ), ( 'skips_forward_with_increment_that_is_too_large', [ m('mouse'), [m('dog'), jump_call(50), m('cat'), m('puppy'), m('python')], m('horse'), ], 'mouse dog horse' ), ( 'jumps_forward_outside_of_nest', [ jump_call(3), m('mouse'), [m('dog'), m('cat'), m('puppy'), m('python')], m('horse'), ], 'horse' ), ( 'skips_backwards_with_acceptable_decrement', [ m('mouse'), [m('dog'), jump_call(-1), m('cat'), m('puppy')], m('horse'), ], 'mouse dog dog cat puppy horse' ), ( 'skips_backwards_with_decrement_that_is_too_large', [ m('mouse'), [m('dog'), m('cat'), jump_call(-50), m('puppy'), m('python')], m('horse'), ], 'mouse dog cat dog cat puppy python horse' ), ( 'skips_backwards_outside_of_nest', [ m('mouse'), [m('dog'), m('cat'), m('puppy'), m('python')], m('horse'), jump_call(-2) ], 'mouse dog cat puppy python horse dog cat puppy python horse' ) )) def test_jump_call(self, _, callbacks, expected_result): self.wfe.callbacks.add_many(callbacks, self.key) self.wfe.process(self.tokens) t = get_first(self.tokens) assert t == expected_result # --------- complicated loop ----------- @pytest.mark.parametrize("_,workflow,expected_result", ( ( 'simple', [ m('mouse'), [ m('dog'), [m('cat'), m('puppy')], [m('python'), [m('wasp'), m('leon')]], m('horse'), ] ], 'mouse dog cat puppy python wasp leon horse' ), ( 'with_nested_jumps', [ jump_call(2), m('mouse'), [ m('dog'), [m('cat'), m('puppy')], [m('python'), jump_call(-2), [m('wasp'), m('leon')]], m('horse'), ] ], 'dog cat puppy python python wasp leon horse' ) )) def test_multi_nested_workflows(self, _, workflow, expected_result): self.wfe.callbacks.add_many(workflow, self.key) self.wfe.process(self.tokens) t = get_first(self.tokens) assert t == expected_result @pytest.mark.parametrize("_,workflow,expected_result", ( ( 'simple', [ m('mouse'), [ m('dog'), [m('cat'), m('puppy')], [m('python'), break_loop(), [m('wasp'), m('leon')]], m('horse'), ] ], 'mouse dog cat puppy python horse' ), ( 'break_loop_outside_of_nest', [ break_loop(), m('mouse'), [ m('dog'), [m('cat'), m('puppy')], [m('python'), [m('wasp'), m('leon')]], m('horse'), ] ], None ) )) def test_break_from_this_loop(self, _, workflow, expected_result): self.wfe.callbacks.add_many(workflow, self.key) self.wfe.process(self.tokens) t = get_first(self.tokens) assert t == expected_result # ----------- StopProcessing -------------------------- def test_engine_immediatelly_stops(self): self.wfe.callbacks.add_many([ stop_processing(), m('mouse'), [ m('dog'), [m('cat'), m('puppy')], [ m('python'), [m('wasp'), m('leon')] ], m('horse'), ] ], self.key) self.wfe.process(self.tokens) t = get_first(self.tokens) assert get_xth(self.tokens, 0) is None assert get_xth(self.tokens, 1) is None assert get_xth(self.tokens, 2) is None def test_engine_stops_half_way_through(self): self.wfe.callbacks.add_many([ m('mouse'), [ m('dog'), [m('cat'), m('puppy')], [m('python'), stop_if_token_equals('four'), [m('wasp'), m('leon')]], m('horse'), ] ], self.key) self.wfe.process(self.tokens) full_result = 'mouse dog cat puppy python wasp leon horse' result_until_stop = 'mouse dog cat puppy python' assert get_xth(self.tokens, 0) == full_result # 'one' assert get_xth(self.tokens, 1) == full_result # 'two' assert get_xth(self.tokens, 2) == full_result # 'three' assert get_xth(self.tokens, 3) == result_until_stop # 'four' assert get_xth(self.tokens, 4) is None # 'five', engine stopped # ---------- jump_token ------------- def test_engine_moves_to_next_token(self): self.wfe.callbacks.add_many( [ m('mouse'), [ m('dog'), [m('cat'), m('puppy')], [m('python'), next_token(), [m('wasp'), m('leon')]], m('horse'), ] ], self.key) self.wfe.process(self.tokens) result_until_next_token = 'mouse dog cat puppy python' for i in range(5): assert get_xth(self.tokens, i) == result_until_next_token def test_workflow_09a(self): self.wfe.callbacks.add_many([ m('mouse'), [ m('dog'), if_str_token_jump('four', -2), [m('cat'), m('puppy')], m('horse'), ] ], self.key) self.wfe.process(self.tokens) t = get_first(self.tokens) r1 = 'mouse dog cat puppy horse' # one, five r2 = 'mouse dog cat puppy horse mouse dog cat puppy horse' # two, three r3 = 'mouse dog mouse dog cat puppy horse' # four assert get_xth(self.tokens, 0) == r1 assert get_xth(self.tokens, 1) == r2 assert get_xth(self.tokens, 2) == r2 assert get_xth(self.tokens, 3) == r3 assert get_xth(self.tokens, 4) == r1 def test_workflow_09b(self): self.wfe.callbacks.add_many([ m('mouse'), [ m('dog'), if_str_token_jump('two', 2), [m('cat'), m('puppy')], m('horse'), ] ], self.key) self.wfe.process(self.tokens) t = get_first(self.tokens) r1 = 'mouse dog cat puppy horse' # one, four, five r2 = 'mouse dog' # two r3 = None # three assert get_xth(self.tokens, 0) == r1 assert get_xth(self.tokens, 1) == r2 assert get_xth(self.tokens, 2) == r3 assert get_xth(self.tokens, 3) == r1 assert get_xth(self.tokens, 4) == r1 # ----------------- HaltProcessing -------------------- def test_50_halt_processing_mid_workflow(self): other_wfe = GenericWorkflowEngine() other_wfe.callbacks.add_many([ m('mouse'), [ m('dog'), [m('cat'), m('puppy')], [m('python'), halt_processing()], m('horse'), ] ], self.key) with pytest.raises(HaltProcessing): other_wfe.process(self.tokens) t = get_first(self.tokens) assert get_xth(self.tokens, 0) == 'mouse dog cat puppy python' assert get_xth(self.tokens, 1) is None assert get_xth(self.tokens, 2) is None compl = 'mouse dog cat puppy python horse' compl1 = 'mouse dog cat puppy python' @pytest.mark.parametrize("obj,task,results", ( ('prev', 'prev', (compl + " python", compl1, None)), ('prev', 'current', (compl, compl1, None)), # current task is to halt ('prev', 'next', (compl + " horse", compl1 + " " + compl1, None)), ('prev', 'first', (compl + " " + compl1, compl1, None)), ('current', 'prev', (compl, compl1 + " python", None)), ('current', 'current', (compl, compl1, None)), ('current', 'next', (compl, compl1 + " horse", compl1)), ('current', 'first', (compl, compl1 + " " + compl1, None)), ('next', 'prev', (compl, compl1, "python")), ('next', 'current', (compl, compl1, None)), ('next', 'next', (compl, compl1, "horse")), ('next', 'first', (compl, compl1, compl1)), ('first', 'prev', (compl + " python", compl1, None)), ('first', 'current', (compl, compl1, None)), # current task is to halt ('first', 'next', (compl + " horse", compl1 + " " + compl1, None)), ('first', 'first', (compl + " " + compl1, compl1, None)), )) def test_51_workflow_restart_after_halt(self, obj, task, results): self.wfe.callbacks.add_many([ m('mouse'), [ m('dog'), [m('cat'), m('puppy')], [m('python'), halt_processing()], m('horse'), ] ], self.key) with pytest.raises(HaltProcessing): self.wfe.process(self.tokens) assert get_xth(self.tokens, 0) == 'mouse dog cat puppy python' assert get_xth(self.tokens, 1) is None assert get_xth(self.tokens, 2) is None # this should pick up from the point where we stopped with pytest.raises(HaltProcessing): self.wfe.restart('current', 'next') assert get_xth(self.tokens, 0) == 'mouse dog cat puppy python horse' assert get_xth(self.tokens, 1) == 'mouse dog cat puppy python' assert get_xth(self.tokens, 2) is None with pytest.raises(HaltProcessing): self.wfe.restart(obj, task) assert (get_xth(self.tokens, 0), get_xth(self.tokens, 1), get_xth(self.tokens, 2)) == results def test_restart_accepts_new_objects(self): workflow = [m('test')] self.wfe.callbacks.replace(workflow, self.key) self.wfe.process(self.tokens) new_data = ['a', 'b', 'c', 'd', 'e'] new_tokens = [FakeToken(x, type='*') for x in new_data] self.wfe.restart('first', 'first', objects=new_tokens) assert self.wfe.objects == new_tokens def test_has_completed(self): self.wfe.callbacks.replace([ m('mouse'), halt_processing(), m('horse'), ]) assert self.wfe.has_completed is False with pytest.raises(HaltProcessing): self.wfe.process([self.tokens[0]]) assert self.wfe.has_completed is False self.wfe.restart('current', 'next') assert self.wfe.has_completed is True def test_nested_workflow_halt(self): other_wfe = GenericWorkflowEngine() wfe = self.wfe other_wfe.callbacks.add_many([ m('mouse'), [ m('dog'), [m('cat'), m('puppy')], [m('python'), halt_processing()], m('horse'), ] ], self.key) wfe.callbacks.add_many([ m('mouse'), [ m('dog'), [m('cat'), m('puppy')], [m('python'), lambda o, e: other_wfe.process(self.tokens)], m('horse'), ] ], self.key) with pytest.raises(HaltProcessing): wfe.process(self.tokens) t = get_first(self.tokens) assert get_xth(self.tokens, 0) == 'mouse dog cat puppy python mouse dog cat puppy python' assert get_xth(self.tokens, 1) is None assert get_xth(self.tokens, 2) is None @pytest.mark.parametrize("callbacks,kwargs,result,exception", ( ( [ m('mouse'), [ m('dog'), [m('cat'), m('puppy')], [m('python'), workflow_error()], m('horse'), ] ], {}, 'mouse dog cat puppy python', WorkflowError, ), ( [ m('mouse'), [ m('dog'), [m('cat'), m('puppy')], [m('python'), workflow_error()], m('horse'), ] ], { 'stop_on_error': False, }, 'mouse dog cat puppy python', None, ), ( [ m('mouse'), [ m('dog'), [m('cat'), m('puppy')], [m('python'), halt_processing()], m('horse'), ] ], { 'stop_on_halt': False, }, 'mouse dog cat puppy python', None, ), )) def test_process_smash_through(self, callbacks, kwargs, result, exception): self.wfe.callbacks.add_many(callbacks, self.key) if exception: with pytest.raises(exception): self.wfe.process(self.tokens, **kwargs) else: self.wfe.process(self.tokens, **kwargs) for idx, dummy in enumerate(self.tokens): assert get_xth(self.tokens, idx) == result