def test_applicable(): data = yadage.workflow_loader.workflow('workflow.yml', 'tests/testspecs/local-helloworld') wflow = YadageWorkflow.createFromJSON( data, LocalFSProvider(LocalFSState(['/workdir']), ensure=False)) wflow.view().init({'par': 'value'}) assert wflow.rules[-1].applicable(wflow) == True
def singlestage_cases(tmpdir, localfs_state_provider): """a workflow object with horizontally scalable map stage scheduling sub-workflows""" data = yadage.workflow_loader.workflow( "workflow.yml", "tests/testspecs/casesflow/single" ) wflow = YadageWorkflow.createFromJSON(data, localfs_state_provider) return wflow
def test_serialize_deserialize(): data = yadage.workflow_loader.workflow('workflow.yml','tests/testspecs/local-helloworld') wflow = YadageWorkflow.createFromJSON(data,LocalFSProvider(LocalFSState(['/workdir']), ensure = False)) wflow.view().init({'hello':'world'}) assert wflow.rules[0].rule.json() assert wflow.rules[1].rule.json() assert JsonStage.fromJSON(wflow.rules[0].rule.json()).json() == wflow.rules[0].rule.json()
def test_getRule(): data = yadage.workflow_loader.workflow('workflow.yml','tests/testspecs/nestedmapreduce') wflow = YadageWorkflow.createFromJSON(data,LocalFSProvider(LocalFSState(['/workdir']), ensure = False)) wflow.view().init({'input':[1,2,3]}) assert wflow.view().getRule(identifier = wflow.rules[0].identifier) == wflow.rules[0] assert wflow.view().getRule(name = wflow.rules[0].rule.name, offset = '') == wflow.rules[0] assert wflow.view().getRule(name = 'nonexistent') == None
def test_serialize_offsetstage(): data = yadage.workflow_loader.workflow("workflow.yml", "tests/testspecs/nestedmapreduce") wflow = YadageWorkflow.createFromJSON( data, LocalFSProvider(LocalFSState(["/workdir"]), ensure=False)) wflow.view().init({"input": [1, 2, 3]}) for x in wflow.rules: assert OffsetStage.fromJSON(x.json()).json() == x.json()
def local_helloworld_wflow_w_init(tmpdir, localfs_state_provider): """a workflow object with horizontally scalable map stage scheduling sub-workflows""" data = yadage.workflow_loader.workflow( "workflow.yml", "tests/testspecs/local-helloworld" ) wflow = YadageWorkflow.createFromJSON(data, localfs_state_provider) wflow.view().init({"par": "hello"}) return wflow
def test_apply(): data = yadage.workflow_loader.workflow('workflow.yml','tests/testspecs/local-helloworld') wflow = YadageWorkflow.createFromJSON(data,LocalFSProvider(LocalFSState(['/workdir']), ensure = False)) wflow.view().init({'par': 'value'}) assert wflow.rules[-1].applicable(wflow) == True wflow.rules[-1].apply(wflow) frommodel_controller('',{},wflow).sync_backend() assert wflow.rules[0].applicable(wflow) == True wflow.rules[0].apply(wflow)
def test_applicable(): data = yadage.workflow_loader.workflow( "workflow.yml", "tests/testspecs/local-helloworld" ) wflow = YadageWorkflow.createFromJSON( data, LocalFSProvider(LocalFSState(["/workdir"]), ensure=False) ) wflow.view().init({"par": "value"}) assert wflow.rules[-1].applicable(wflow) == True
def test_serialize_deserialize(): data = yadage.workflow_loader.workflow('workflow.yml', 'tests/testspecs/local-helloworld') wflow = YadageWorkflow.createFromJSON( data, LocalFSProvider(LocalFSState(['/workdir']), ensure=False)) wflow.view().init({'hello': 'world'}) assert wflow.rules[0].rule.json() assert wflow.rules[1].rule.json() assert JsonStage.fromJSON( wflow.rules[0].rule.json()).json() == wflow.rules[0].rule.json()
def test_deserialize(): data = { "rules": [], "applied": [], "dag": {"nodes": [], "edges": []}, "stepsbystage": {}, "bookkeeping": {}, "values": {}, } wflow = YadageWorkflow.fromJSON(data) assert data == wflow.json()
def test_getRule(): data = yadage.workflow_loader.workflow("workflow.yml", "tests/testspecs/nestedmapreduce") wflow = YadageWorkflow.createFromJSON( data, LocalFSProvider(LocalFSState(["/workdir"]), ensure=False)) wflow.view().init({"input": [1, 2, 3]}) assert wflow.view().getRule( identifier=wflow.rules[0].identifier) == wflow.rules[0] assert (wflow.view().getRule(name=wflow.rules[0].rule.name, offset="") == wflow.rules[0]) assert wflow.view().getRule(name="nonexistent") == None
def test_init(): data = yadage.workflow_loader.workflow('workflow.yml','tests/testspecs/nestedmapreduce') wflow = YadageWorkflow.createFromJSON(data,LocalFSProvider(LocalFSState(['/workdir']), ensure = False)) view = wflow.view() nrules_before = len(wflow.rules) view.init({'hello':'world'}) nrules_after = len(wflow.rules) assert nrules_before + 1 == nrules_after view.rules[-1].apply(wflow) matches = view.query('init',view.steps) assert len(matches) == 1
def test_serialize_deserialize(): data = yadage.workflow_loader.workflow( "workflow.yml", "tests/testspecs/local-helloworld" ) wflow = YadageWorkflow.createFromJSON( data, LocalFSProvider(LocalFSState(["/workdir"]), ensure=False) ) wflow.view().init({"hello": "world"}) assert wflow.rules[0].rule.json() assert wflow.rules[1].rule.json() assert ( JsonStage.fromJSON(wflow.rules[0].rule.json()).json() == wflow.rules[0].rule.json() )
def test_init(): data = yadage.workflow_loader.workflow("workflow.yml", "tests/testspecs/nestedmapreduce") wflow = YadageWorkflow.createFromJSON( data, LocalFSProvider(LocalFSState(["/workdir"]), ensure=False)) view = wflow.view() nrules_before = len(wflow.rules) view.init({"hello": "world"}) nrules_after = len(wflow.rules) assert nrules_before + 1 == nrules_after view.rules[-1].apply(wflow) matches = view.query("init", view.steps) assert len(matches) == 1
def test_multistepstage_schedule_wflows(localfs_state_provider): data = yadage.workflow_loader.workflow('workflow.yml', 'tests/testspecs/nestedmapreduce') wflow = YadageWorkflow.createFromJSON(data, localfs_state_provider) inputdata = [1, 2, 3] wflow.view().init({'input': inputdata}) assert wflow.view().getRule(name='map').applicable(wflow) == False wflow.view().getRule(name='init').apply(wflow) assert len(wflow.dag.nodes()) == 1 frommodel_controller('', {}, wflow).sync_backend() assert wflow.view().getRule(name='map').applicable(wflow) == True wflow.view().getRule(name='map').apply(wflow) assert wflow.view().getRule(name='reduce').applicable(wflow) == False
def test_multistepstage_schedule_wflows(localfs_state_provider): data = yadage.workflow_loader.workflow('workflow.yml','tests/testspecs/nestedmapreduce') wflow = YadageWorkflow.createFromJSON(data,localfs_state_provider) inputdata = [1,2,3] wflow.view().init({'input':inputdata}) assert wflow.view().getRule(name = 'map').applicable(wflow) == False wflow.view().getRule(name = 'init').apply(wflow) assert len(wflow.dag.nodes()) == 1 frommodel_controller('',{},wflow).sync_backend() assert wflow.view().getRule(name = 'map').applicable(wflow) == True wflow.view().getRule(name = 'map').apply(wflow) assert wflow.view().getRule(name = 'reduce').applicable(wflow) == False
def test_view(): wflow = YadageWorkflow() wflow.view()
def test_serialize_deserialize(local_helloworld_wflow): wflow = local_helloworld_wflow assert YadageWorkflow.fromJSON(wflow.json()).json() == wflow.json()
def jqnodestruct(tmpdir, localfs_state_provider): '''a workflow object with horizontally scalable map stage scheduling sub-workflows''' data = yadage.workflow_loader.workflow('workflow.yml', 'tests/testspecs/jqnodestruct') wflow = YadageWorkflow.createFromJSON(data, localfs_state_provider) return wflow
def singlestage_cases(tmpdir,localfs_state_provider): '''a workflow object with horizontally scalable map stage scheduling sub-workflows''' data = yadage.workflow_loader.workflow('workflow.yml','tests/testspecs/casesflow/single') wflow = YadageWorkflow.createFromJSON(data,localfs_state_provider) return wflow
def value_registering_workflow(tmpdir,localfs_state_provider): '''a workflow object with horizontally scalable map stage scheduling sub-workflows''' data = yadage.workflow_loader.workflow('workflow.yml','tests/testspecs/registering_values') wflow = YadageWorkflow.createFromJSON(data,localfs_state_provider) return wflow
def local_helloworld_wflow_w_init(tmpdir,localfs_state_provider): '''a workflow object with horizontally scalable map stage scheduling sub-workflows''' data = yadage.workflow_loader.workflow('workflow.yml','tests/testspecs/local-helloworld') wflow = YadageWorkflow.createFromJSON(data,localfs_state_provider) wflow.view().init({'par': 'hello'}) return wflow
def batched_zip_mapreduce(tmpdir,localfs_state_provider): '''a workflow object with horizontally scalable map stage scheduling sub-workflows''' data = yadage.workflow_loader.workflow('workflow.yml','tests/testspecs/mapreduce_batched') wflow = YadageWorkflow.createFromJSON(data,localfs_state_provider) return wflow
def test_serialize(): wflow = YadageWorkflow() data = wflow.json() json.dumps(data)
def test_deserialize(): data = {'rules': [], 'applied': [], 'dag': {'nodes': [], 'edges': []}, 'stepsbystage': {}, 'bookkeeping': {}, 'values': {}} wflow = YadageWorkflow.fromJSON(data) assert data == wflow.json()
def test_serialize_offsetstage(): data = yadage.workflow_loader.workflow('workflow.yml','tests/testspecs/nestedmapreduce') wflow = YadageWorkflow.createFromJSON(data,LocalFSProvider(LocalFSState(['/workdir']), ensure = False)) wflow.view().init({'input':[1,2,3]}) for x in wflow.rules: assert OffsetStage.fromJSON(x.json()).json() == x.json()
def test_view(): wflow = YadageWorkflow() wflow.view()
def test_create(): wflow = YadageWorkflow()
def test_serialize_deserialize(local_helloworld_wflow): wflow = local_helloworld_wflow assert YadageWorkflow.fromJSON(wflow.json()).json() == wflow.json()
def test_serialize(): wflow = YadageWorkflow() data = wflow.json() json.dumps(data)