def test_init(self): d1 = self.getDoc() d2 = self.getDoc() d3 = self.getDoc() # init with empty to full parameters we1 = GenericWorkflowEngine() we2 = GenericWorkflowEngine(callback_chooser=asterisk_chooser) try: we3 = GenericWorkflowEngine(processing_factory='x', callback_chooser='x', before_processing='x', after_processing='x') except Exception, msg: assert 'must be a callable' not in msg
def run_workflow(message): jsonObj = json.loads(message) print "Workflow input header message %s" % (jsonObj['Header']) data = jsonObj['Body'] my_engine = GenericWorkflowEngine() my_engine.setWorkflow(workflow_model_1()) my_engine.process(data)
def test_nested_workflow_halt(self): other_wfe = GenericWorkflowEngine() wfe = self.wfe other_wfe.callbacks.add_many([ m('mouse'), [ m('dog'), [m('cat'), m('puppy')], [m('python'), halt_processing()], m('horse'), ] ], self.key) wfe.callbacks.add_many([ m('mouse'), [ m('dog'), [m('cat'), m('puppy')], [m('python'), lambda o, e: other_wfe.process(self.tokens)], m('horse'), ] ], self.key) with pytest.raises(HaltProcessing): wfe.process(self.tokens) t = get_first(self.tokens) assert get_xth( self.tokens, 0) == 'mouse dog cat puppy python mouse dog cat puppy python' assert get_xth(self.tokens, 1) is None assert get_xth(self.tokens, 2) is None
def test_PARALLEL_SPLIT01(self): we = GenericWorkflowEngine() doc = self.getDoc() we.setWorkflow([ i('start'), cf.PARALLEL_SPLIT(printer('p1'), printer('p2'), printer('p3'), printer('p4'), printer('p5')), lambda o, e: time.sleep(.1), a('end') ]) we.process(doc) r = [' '.join(doc[x]) for x in range(len(doc))] assert doc[0][0] == 'start' assert doc[0][1] == 'one' assert doc[1][0] == 'start' assert doc[1][1] == 'two' # end must have been inserted while printers were running # mixed together with them all_pos = set() for x in range(len(doc)): pos = doc[x].index('end') assert pos > 2 assert pos < len(doc[x]) all_pos.add(pos)
def test_CHOICE03(self): we = GenericWorkflowEngine() doc = self.getDoc()[0:1] def arbiter(obj, eng): return obj[-1] we.setWorkflow([ i('start'), cf.CHOICE( arbiter, ('bam', lambda obj, eng: obj.append('bom')), ('end', lambda obj, eng: obj.append('error')), ('bim', lambda obj, eng: obj.append('bam')), bom=(lambda obj, eng: obj.append('bum')), one=(lambda obj, eng: obj.append('bim')), bum=cf.STOP(), ), cf.TASK_JUMP_BWD(-1) ]) we.process(doc) d = ' '.join(doc[0]) assert 'bim bam bom bum' in d assert 'error' not in d assert len(doc[0]) == 6
def test_configure(self): d1 = self.getDoc() d2 = self.getDoc() d3 = self.getDoc() we = GenericWorkflowEngine() we.addManyCallbacks('*', [ m('mouse'), [m('dog'), call_forward(1), m('cat'), m('puppy')], m('horse'), ]) # process using defaults we.process(d1) r = 'one mouse dog cat puppy horse'.split() # pass our own callback chooser we.configure(callback_chooser=asterisk_chooser) we.process(d2) assert d1[0] == r assert d2[0] == r assert d1 == d2 # configure it wrongly we.configure(callback_chooser='') self.failUnlessRaises(Exception, we.process, d3) assert d3 == self.getDoc()
def test_IF_ELSE03(self): we = GenericWorkflowEngine() doc = self.getDoc() doc[3].append('4') def test(v): return lambda o, e: v in o we.setWorkflow([ i('add'), cf.IF_ELSE(test('three'), [ a('xxx'), cf.IF_ELSE(test('xxx'), [ a('6'), cf.IF_ELSE(test('6'), a('six'), (a('only-3s'), a('error'))) ], a('ok')) ], [ cf.IF_ELSE( test('4'), cf.IF_ELSE(test('four'), [a('44'), [[[a('forty')]]]], a('error')), a('not-four')) ]), a('end'), cf.IF_ELSE(test('error'), a('gosh!'), a('OK')) ]) we.process(doc) r = [' '.join(doc[x]) for x in range(len(doc))] assert r[0] == 'add one not-four end OK' assert r[1] == 'add two not-four end OK' assert r[2] == 'add three xxx 6 six end OK' assert r[3] == 'add four 4 44 forty end OK' assert r[4] == 'add five not-four end OK'
def test_workflow01(self): class GenericWEWithXChooser(GenericWorkflowEngine): def callback_chooser(self, obj): return self.callbacks.get('x') we0 = GenericWorkflowEngine() we1 = GenericWorkflowEngine() we2 = GenericWEWithXChooser() we0.addManyCallbacks('*', [ obj_append('mouse'), [ obj_append('dog'), jump_call(1), obj_append('cat'), obj_append('puppy') ], obj_append('horse'), ]) we1.setWorkflow([ obj_append('mouse'), [ obj_append('dog'), jump_call(1), obj_append('cat'), obj_append('puppy') ], obj_append('horse'), ]) we2.addManyCallbacks('x', [ obj_append('mouse'), [ obj_append('dog'), jump_call(1), obj_append('cat'), obj_append('puppy') ], obj_append('horse'), ]) we0.process(self.d0) we1.process(self.d1) we2.process(self.d2) assert self.d0 == self.d1 assert self.d0 == self.d2
def run_workflow_1(): my_object0 = MyObject(0) my_object1 = MyObject(1) my_engine_1 = GenericWorkflowEngine() my_engine_1.setWorkflow(my_workflow_1) my_engine_1.process([my_object0, my_object1])
def automated_process(target): # pylint: disable=unused-argument """ Work out the current point in the automated workflow and process the next step. """ my_engine = GenericWorkflowEngine() my_engine.callbacks.replace( [task_add_repo, task_collect_nonwords, task_submit, task_cleanup]) my_engine.process([State(target)])
def test_IF_ELSE02(self): we = GenericWorkflowEngine() doc = self.getDoc() we.setWorkflow([ i('add'), cf.IF_ELSE(lambda o, e: o[1] == 'three', a('3'), a('other')) ]) we.process(doc) r = [' '.join(doc[x]) for x in range(len(doc))] assert r[0] == 'add one other' assert r[1] == 'add two other' assert r[2] == 'add three 3' assert r[3] == 'add four other' assert r[4] == 'add five other'
def test_PARALLEL_SPLIT02(self): """TODO: this test is failing, but that is because sometimes it does not take into accounts threads being executed in random mannger""" we = GenericWorkflowEngine() doc = self.getDoc()[0:1] we.setWorkflow([ i('start'), cf.PARALLEL_SPLIT( [ cf.PARALLEL_SPLIT( printer('p0'), printer('p0a'), cf.PARALLEL_SPLIT(printer('p0b'), printer('p0c')), ), printer('xx') ], [a('AAA'), printer('p2b')], printer('p3'), [a('p4a'), printer('p4b'), printer('p4c')], [ printer('p5'), cf.PARALLEL_SPLIT( printer('p6'), printer('p7'), [printer('p8a'), printer('p8b')], ) ]), a('end') ]) we.process(doc) # give threads time to finish time.sleep(2) assert doc[0][0] == 'start' assert doc[0][1] == 'one' # at least the fist object should have them all # print doc[0] for x in [ 'p0', 'p0a', 'p0b', 'p0c', 'xx', 'AAA', 'p2b', 'p3', 'p4a', 'p4b', 'p4c', 'p5', 'p6', 'p8a', 'p8b' ]: doc[0].index(x) # will fail if not present
def test_init(self): # init with empty to full parameters we1 = GenericWorkflowEngine() callbacks = [ obj_append('mouse'), [ obj_append('dog'), jump_call(1), obj_append('cat'), obj_append('puppy') ], obj_append('horse'), ] we1.addManyCallbacks('*', deepcopy(callbacks)) we1.process(self.d1)
def test_RUN_WF01(self): """Test wfe is reinit=False, eng must remember previous invocations""" we = GenericWorkflowEngine() doc = self.getDoc()[0:1] we.setWorkflow([ i('start'), ut.RUN_WF( [ lambda obj, eng: obj.append('bom'), lambda obj, eng: obj.append('bam'), lambda obj, eng: obj.append('bum'), lambda obj, eng: obj.append('end'), lambda obj, eng: obj. append(eng.store.setdefault('eng-end', '')), e('eng-end', 'eng-end') ], data_connector=lambda obj, eng: [obj], outkey='#wfe', ), ]) we.process(doc) d = ' '.join(doc[0]) assert 'start' in d assert 'bom' in d assert 'bam' in d assert 'bum' in d assert 'end' in d assert 'eng-end' not in d # run the same thing again we.process(doc) d = ' '.join(doc[0]) assert 'start' in d assert d.count('bom') == 2 assert d.count('bam') == 2 assert d.count('bum') == 2 assert 'end' in d assert 'eng-end' in d # now it must be present
def test_PARALLEL_SPLIT03(self): we = GenericWorkflowEngine() doc = self.getDoc() we.setWorkflow([ i('start'), cf.PARALLEL_SPLIT([ cf.IF(lambda obj, eng: 'jump-verified' in obj, a('error')), cf.PARALLEL_SPLIT([ cf.IF(lambda obj, eng: 'nasty-jump' in obj, [ a('jump-ok'), lambda obj, eng: ('nasty-jump' in obj and obj.append('jump-verified')) ]), cf.PARALLEL_SPLIT( a('ok-1'), a('ok-2'), cf.IF( lambda obj, eng: 'ok-3' not in obj, lambda obj, eng: (obj.append('ok-3') and eng.breakFromThisLoop())), a('ok-4')), a('xx'), lambda obj, eng: ('jump-verified' in obj and eng.breakFromThisLoop()), a('nasty-jump'), cf.TASK_JUMP_IF( lambda obj, eng: 'jump-verified' not in obj, -100) ]), ], [a('AAA'), a('p2b')]), a('end') ]) we.process(doc) # give threads time to finish time.sleep(.5) d = doc[0] # at least the fist object should have them all # print doc[0] for x in ['nasty-jump', 'jump-verified', 'ok-3']: d.index(x) # will fail if not present assert d.count('ok-1') > 1
class TestGenericWorkflowEngine(unittest.TestCase): """Tests of the WE interface""" def setUp(self): self.key = '*' def tearDown(self): pass def getDoc(self, val=None): if val: return [[x] for x in val.split()] return [[x] for x in u"one two three four five".split()] def addTestCallbacks(self, no, eng): if type == 1: eng.addManyCallbacks() # --------- initialization --------------- def test_init(self): d1 = self.getDoc() d2 = self.getDoc() d3 = self.getDoc() # init with empty to full parameters we1 = GenericWorkflowEngine() we2 = GenericWorkflowEngine(callback_chooser=asterisk_chooser) try: we3 = GenericWorkflowEngine(processing_factory='x', callback_chooser='x', before_processing='x', after_processing='x') except Exception, msg: assert 'must be a callable' not in msg try: we3 = GenericWorkflowEngine(callback_chooser=asterisk_chooser, after_processing='x') except Exception, msg: assert 'must be a callable' not in msg
def interactive_old_word(state, jsonobj, word): """ Single word processing """ my_engine = GenericWorkflowEngine() my_engine.callbacks.replace( [check_websearch, is_nonword, is_typo, what_now]) newstate = NonwordState( interaction=state.interaction, target=state.target, word=word, details=jsonobj[word], repopath=state.repopath, nonword_delegate=state.nonword_delegate, ) try: my_engine.process([newstate]) except HaltProcessing: if newstate.done: return True return False
def test_SIMPLE_MERGE03(self): we = GenericWorkflowEngine() doc = self.getDoc()[0:1] we.setWorkflow([i('start'), cf.SIMPLE_MERGE( lambda obj, eng: obj.append('bom'), lambda obj, eng: obj.append('error'), lambda obj, eng: obj.append('bam'), lambda obj, eng: obj.append('bum'), lambda obj, eng: obj.append('end'), ), ]) we.process(doc) d = ' '.join(doc[0]) assert 'start' in d assert 'bom' in d assert 'error' not in d assert 'end' in d
def test_RUN_WF02(self): """Test wfe is reinit=True - eng must not remember""" we = GenericWorkflowEngine() doc = self.getDoc()[0:1] we.callbacks.replace([ i('start'), ut.RUN_WF([ lambda obj, eng: obj.append('bom'), lambda obj, eng: obj.append('bam'), lambda obj, eng: obj.append('bum'), lambda obj, eng: obj.append('end'), lambda obj, eng: obj. append(eng.store.setdefault('eng-end', '')), e('eng-end', 'eng-end') ], data_connector=lambda obj, eng: [obj], outkey='#wfe', reinit=True), ]) we.process(doc) d = ' '.join(doc[0]) assert 'start' in d assert 'bom' in d assert 'bam' in d assert 'bum' in d assert 'end' in d assert 'eng-end' not in d # run the same thing again we.process(doc) d = ' '.join(doc[0]) assert 'start' in d assert d.count('bom') == 2 assert d.count('bam') == 2 assert d.count('bum') == 2 assert 'end' in d assert 'eng-end' not in d # it must not be present if reinit=True
def interactive_task_collect_nonwords( # pylint: disable=unused-argument reponame, target, nonword_delegate=None, nonstop=False): """ Saves nonwords until a typo is found """ if nonword_delegate is None: nonword_delegate = interactive_nonword_delegate(target) key = "repository_map" repository_map = get_json_value(key, {}) repodir = repository_map[reponame] repodirpath = Path(repodir) jsonpath = repodirpath / "spelling.json" if not jsonpath.is_file(): print(f"Unable to locate spelling at {jsonpath}", file=sys.stderr) return with io.open(jsonpath, "r", encoding="utf-8") as fobj: jsonobj = json.load(fobj) words = get_sorted_words(jsonobj) my_engine = GenericWorkflowEngine() my_engine.callbacks.replace( [check_websearch, is_nonword, is_typo, what_now]) for word in words: state = NonwordState( target=target, word=word, details=jsonobj[word], repopath=repodirpath, nonword_delegate=nonword_delegate, ) try: my_engine.process([state]) except HaltProcessing: if state.done and not nonstop: return print( f"{Fore.YELLOW}Completed checking all words for {reponame}!{Style.RESET_ALL}" )
def run_workflow_3(): my_engine_3 = GenericWorkflowEngine() my_engine_3.setWorkflow(my_workflow_3) my_engine_3.process([[]])
from workflow.engine import GenericWorkflowEngine def flow1(object, eng): print(object + 1) def flow2(object, eng): print(object + 2) flowEngine = GenericWorkflowEngine() taskFlow = [flow1, flow2] flowEngine.callbacks.replace(taskFlow) flowEngine.process([1])
def setup_method(self, method): self.key = '*' self.wfe = GenericWorkflowEngine() self.data = ['one', 'two', 'three', 'four', 'five'] self.tokens = [FakeToken(x, type='*') for x in self.data]
def runWorkflow(data): from workflow.engine import GenericWorkflowEngine wfe = GenericWorkflowEngine() wfe.setWorkflow(flow) wfe.process(data)