def batched_zip_mapreduce(tmpdir, localfs_state_provider):
"""a workflow object with horizontally scalable map stage scheduling sub-workflows"""
data = yadage.workflow_loader.workflow(
"workflow.yml", "tests/testspecs/mapreduce_batched"
)
wflow = YadageWorkflow.createFromJSON(data, localfs_state_provider)
return wflow
def local_helloworld_wflow_w_init(tmpdir, localfs_state_provider):
'''a workflow object with horizontally scalable map stage scheduling sub-workflows'''
data = yadage.workflow_loader.workflow('workflow.yml',
'tests/testspecs/local-helloworld')
wflow = YadageWorkflow.createFromJSON(data, localfs_state_provider)
wflow.view().init({'par': 'hello'})
return wflow
def value_registering_workflow(tmpdir, localfs_state_provider):
"""a workflow object with horizontally scalable map stage scheduling sub-workflows"""
data = yadage.workflow_loader.workflow(
"workflow.yml", "tests/testspecs/registering_values"
)
wflow = YadageWorkflow.createFromJSON(data, localfs_state_provider)
return wflow
def singlestage_cases(tmpdir, localfs_state_provider):
"""a workflow object with horizontally scalable map stage scheduling sub-workflows"""
data = yadage.workflow_loader.workflow(
"workflow.yml", "tests/testspecs/casesflow/single"
)
wflow = YadageWorkflow.createFromJSON(data, localfs_state_provider)
return wflow
def test_applicable():
data = yadage.workflow_loader.workflow('workflow.yml',
'tests/testspecs/local-helloworld')
wflow = YadageWorkflow.createFromJSON(
data, LocalFSProvider(LocalFSState(['/workdir']), ensure=False))
wflow.view().init({'par': 'value'})
assert wflow.rules[-1].applicable(wflow) == True
def test_getRule():
data = yadage.workflow_loader.workflow('workflow.yml','tests/testspecs/nestedmapreduce')
wflow = YadageWorkflow.createFromJSON(data,LocalFSProvider(LocalFSState(['/workdir']), ensure = False))
wflow.view().init({'input':[1,2,3]})
assert wflow.view().getRule(identifier = wflow.rules[0].identifier) == wflow.rules[0]
assert wflow.view().getRule(name = wflow.rules[0].rule.name, offset = '') == wflow.rules[0]
assert wflow.view().getRule(name = 'nonexistent') == None
def test_serialize_offsetstage():
data = yadage.workflow_loader.workflow("workflow.yml",
"tests/testspecs/nestedmapreduce")
wflow = YadageWorkflow.createFromJSON(
data, LocalFSProvider(LocalFSState(["/workdir"]), ensure=False))
wflow.view().init({"input": [1, 2, 3]})
for x in wflow.rules:
assert OffsetStage.fromJSON(x.json()).json() == x.json()
def local_helloworld_wflow_w_init(tmpdir, localfs_state_provider):
"""a workflow object with horizontally scalable map stage scheduling sub-workflows"""
data = yadage.workflow_loader.workflow(
"workflow.yml", "tests/testspecs/local-helloworld"
)
wflow = YadageWorkflow.createFromJSON(data, localfs_state_provider)
wflow.view().init({"par": "hello"})
return wflow
def test_serialize_deserialize():
data = yadage.workflow_loader.workflow('workflow.yml','tests/testspecs/local-helloworld')
wflow = YadageWorkflow.createFromJSON(data,LocalFSProvider(LocalFSState(['/workdir']), ensure = False))
wflow.view().init({'hello':'world'})
assert wflow.rules[0].rule.json()
assert wflow.rules[1].rule.json()
assert JsonStage.fromJSON(wflow.rules[0].rule.json()).json() == wflow.rules[0].rule.json()
def test_apply():
data = yadage.workflow_loader.workflow('workflow.yml','tests/testspecs/local-helloworld')
wflow = YadageWorkflow.createFromJSON(data,LocalFSProvider(LocalFSState(['/workdir']), ensure = False))
wflow.view().init({'par': 'value'})
assert wflow.rules[-1].applicable(wflow) == True
wflow.rules[-1].apply(wflow)
frommodel_controller('',{},wflow).sync_backend()
assert wflow.rules[0].applicable(wflow) == True
wflow.rules[0].apply(wflow)
def test_applicable():
data = yadage.workflow_loader.workflow(
"workflow.yml", "tests/testspecs/local-helloworld"
)
wflow = YadageWorkflow.createFromJSON(
data, LocalFSProvider(LocalFSState(["/workdir"]), ensure=False)
)
wflow.view().init({"par": "value"})
assert wflow.rules[-1].applicable(wflow) == True
def test_serialize_deserialize():
data = yadage.workflow_loader.workflow('workflow.yml',
'tests/testspecs/local-helloworld')
wflow = YadageWorkflow.createFromJSON(
data, LocalFSProvider(LocalFSState(['/workdir']), ensure=False))
wflow.view().init({'hello': 'world'})
assert wflow.rules[0].rule.json()
assert wflow.rules[1].rule.json()
assert JsonStage.fromJSON(
wflow.rules[0].rule.json()).json() == wflow.rules[0].rule.json()
def test_getRule():
data = yadage.workflow_loader.workflow("workflow.yml",
"tests/testspecs/nestedmapreduce")
wflow = YadageWorkflow.createFromJSON(
data, LocalFSProvider(LocalFSState(["/workdir"]), ensure=False))
wflow.view().init({"input": [1, 2, 3]})
assert wflow.view().getRule(
identifier=wflow.rules[0].identifier) == wflow.rules[0]
assert (wflow.view().getRule(name=wflow.rules[0].rule.name,
offset="") == wflow.rules[0])
assert wflow.view().getRule(name="nonexistent") == None
def test_init():
data = yadage.workflow_loader.workflow('workflow.yml','tests/testspecs/nestedmapreduce')
wflow = YadageWorkflow.createFromJSON(data,LocalFSProvider(LocalFSState(['/workdir']), ensure = False))
view = wflow.view()
nrules_before = len(wflow.rules)
view.init({'hello':'world'})
nrules_after = len(wflow.rules)
assert nrules_before + 1 == nrules_after
view.rules[-1].apply(wflow)
matches = view.query('init',view.steps)
assert len(matches) == 1
def test_serialize_deserialize():
data = yadage.workflow_loader.workflow(
"workflow.yml", "tests/testspecs/local-helloworld"
)
wflow = YadageWorkflow.createFromJSON(
data, LocalFSProvider(LocalFSState(["/workdir"]), ensure=False)
)
wflow.view().init({"hello": "world"})
assert wflow.rules[0].rule.json()
assert wflow.rules[1].rule.json()
assert (
JsonStage.fromJSON(wflow.rules[0].rule.json()).json()
== wflow.rules[0].rule.json()
)
def test_init():
data = yadage.workflow_loader.workflow("workflow.yml",
"tests/testspecs/nestedmapreduce")
wflow = YadageWorkflow.createFromJSON(
data, LocalFSProvider(LocalFSState(["/workdir"]), ensure=False))
view = wflow.view()
nrules_before = len(wflow.rules)
view.init({"hello": "world"})
nrules_after = len(wflow.rules)
assert nrules_before + 1 == nrules_after
view.rules[-1].apply(wflow)
matches = view.query("init", view.steps)
assert len(matches) == 1
def test_multistepstage_schedule_wflows(localfs_state_provider):
data = yadage.workflow_loader.workflow('workflow.yml',
'tests/testspecs/nestedmapreduce')
wflow = YadageWorkflow.createFromJSON(data, localfs_state_provider)
inputdata = [1, 2, 3]
wflow.view().init({'input': inputdata})
assert wflow.view().getRule(name='map').applicable(wflow) == False
wflow.view().getRule(name='init').apply(wflow)
assert len(wflow.dag.nodes()) == 1
frommodel_controller('', {}, wflow).sync_backend()
assert wflow.view().getRule(name='map').applicable(wflow) == True
wflow.view().getRule(name='map').apply(wflow)
assert wflow.view().getRule(name='reduce').applicable(wflow) == False
def test_multistepstage_schedule_wflows(localfs_state_provider):
data = yadage.workflow_loader.workflow('workflow.yml','tests/testspecs/nestedmapreduce')
wflow = YadageWorkflow.createFromJSON(data,localfs_state_provider)
inputdata = [1,2,3]
wflow.view().init({'input':inputdata})
assert wflow.view().getRule(name = 'map').applicable(wflow) == False
wflow.view().getRule(name = 'init').apply(wflow)
assert len(wflow.dag.nodes()) == 1
frommodel_controller('',{},wflow).sync_backend()
assert wflow.view().getRule(name = 'map').applicable(wflow) == True
wflow.view().getRule(name = 'map').apply(wflow)
assert wflow.view().getRule(name = 'reduce').applicable(wflow) == False
def test_serialize_offsetstage():
data = yadage.workflow_loader.workflow('workflow.yml','tests/testspecs/nestedmapreduce')
wflow = YadageWorkflow.createFromJSON(data,LocalFSProvider(LocalFSState(['/workdir']), ensure = False))
wflow.view().init({'input':[1,2,3]})
for x in wflow.rules:
assert OffsetStage.fromJSON(x.json()).json() == x.json()
def value_registering_workflow(tmpdir,localfs_state_provider):
'''a workflow object with horizontally scalable map stage scheduling sub-workflows'''
data = yadage.workflow_loader.workflow('workflow.yml','tests/testspecs/registering_values')
wflow = YadageWorkflow.createFromJSON(data,localfs_state_provider)
return wflow
def singlestage_cases(tmpdir,localfs_state_provider):
'''a workflow object with horizontally scalable map stage scheduling sub-workflows'''
data = yadage.workflow_loader.workflow('workflow.yml','tests/testspecs/casesflow/single')
wflow = YadageWorkflow.createFromJSON(data,localfs_state_provider)
return wflow
def batched_zip_mapreduce(tmpdir,localfs_state_provider):
'''a workflow object with horizontally scalable map stage scheduling sub-workflows'''
data = yadage.workflow_loader.workflow('workflow.yml','tests/testspecs/mapreduce_batched')
wflow = YadageWorkflow.createFromJSON(data,localfs_state_provider)
return wflow
def local_helloworld_wflow_w_init(tmpdir,localfs_state_provider):
'''a workflow object with horizontally scalable map stage scheduling sub-workflows'''
data = yadage.workflow_loader.workflow('workflow.yml','tests/testspecs/local-helloworld')
wflow = YadageWorkflow.createFromJSON(data,localfs_state_provider)
wflow.view().init({'par': 'hello'})
return wflow
def jqnodestruct(tmpdir, localfs_state_provider):
'''a workflow object with horizontally scalable map stage scheduling sub-workflows'''
data = yadage.workflow_loader.workflow('workflow.yml',
'tests/testspecs/jqnodestruct')
wflow = YadageWorkflow.createFromJSON(data, localfs_state_provider)
return wflow