Python YadageWorkflow Examples

Example #1

def test_applicable():
    data = yadage.workflow_loader.workflow('workflow.yml',
                                           'tests/testspecs/local-helloworld')
    wflow = YadageWorkflow.createFromJSON(
        data, LocalFSProvider(LocalFSState(['/workdir']), ensure=False))
    wflow.view().init({'par': 'value'})
    assert wflow.rules[-1].applicable(wflow) == True

Example #2

File: conftest.py Project: matthewfeickert/yadage

def singlestage_cases(tmpdir, localfs_state_provider):
    """a workflow object with horizontally scalable map stage scheduling sub-workflows"""
    data = yadage.workflow_loader.workflow(
        "workflow.yml", "tests/testspecs/casesflow/single"
    )
    wflow = YadageWorkflow.createFromJSON(data, localfs_state_provider)
    return wflow

Example #3

File: test_stage.py Project: lukasheinrich/yadage

def test_serialize_deserialize():
    data  = yadage.workflow_loader.workflow('workflow.yml','tests/testspecs/local-helloworld')
    wflow = YadageWorkflow.createFromJSON(data,LocalFSProvider(LocalFSState(['/workdir']), ensure = False))
    wflow.view().init({'hello':'world'})
    assert wflow.rules[0].rule.json()
    assert wflow.rules[1].rule.json()

    assert JsonStage.fromJSON(wflow.rules[0].rule.json()).json() == wflow.rules[0].rule.json()

Example #4

def test_getRule():
    data  = yadage.workflow_loader.workflow('workflow.yml','tests/testspecs/nestedmapreduce')
    wflow = YadageWorkflow.createFromJSON(data,LocalFSProvider(LocalFSState(['/workdir']), ensure = False))
    wflow.view().init({'input':[1,2,3]})

    assert wflow.view().getRule(identifier = wflow.rules[0].identifier) == wflow.rules[0]
    assert wflow.view().getRule(name = wflow.rules[0].rule.name, offset = '') == wflow.rules[0]
    assert wflow.view().getRule(name = 'nonexistent') == None

Example #5

def test_serialize_offsetstage():
    data = yadage.workflow_loader.workflow("workflow.yml",
                                           "tests/testspecs/nestedmapreduce")
    wflow = YadageWorkflow.createFromJSON(
        data, LocalFSProvider(LocalFSState(["/workdir"]), ensure=False))
    wflow.view().init({"input": [1, 2, 3]})
    for x in wflow.rules:
        assert OffsetStage.fromJSON(x.json()).json() == x.json()

Example #6

File: conftest.py Project: matthewfeickert/yadage

def local_helloworld_wflow_w_init(tmpdir, localfs_state_provider):
    """a workflow object with horizontally scalable map stage scheduling sub-workflows"""
    data = yadage.workflow_loader.workflow(
        "workflow.yml", "tests/testspecs/local-helloworld"
    )
    wflow = YadageWorkflow.createFromJSON(data, localfs_state_provider)
    wflow.view().init({"par": "hello"})
    return wflow

Example #7

File: test_stage.py Project: lukasheinrich/yadage

def test_apply():
    data  = yadage.workflow_loader.workflow('workflow.yml','tests/testspecs/local-helloworld')
    wflow = YadageWorkflow.createFromJSON(data,LocalFSProvider(LocalFSState(['/workdir']), ensure = False))
    wflow.view().init({'par': 'value'})
    assert wflow.rules[-1].applicable(wflow) == True
    wflow.rules[-1].apply(wflow)
    frommodel_controller('',{},wflow).sync_backend()
    assert wflow.rules[0].applicable(wflow) == True
    wflow.rules[0].apply(wflow)

Example #8

File: test_stage.py Project: matthewfeickert/yadage

def test_applicable():
    data = yadage.workflow_loader.workflow(
        "workflow.yml", "tests/testspecs/local-helloworld"
    )
    wflow = YadageWorkflow.createFromJSON(
        data, LocalFSProvider(LocalFSState(["/workdir"]), ensure=False)
    )
    wflow.view().init({"par": "value"})
    assert wflow.rules[-1].applicable(wflow) == True

Example #9

def test_serialize_deserialize():
    data = yadage.workflow_loader.workflow('workflow.yml',
                                           'tests/testspecs/local-helloworld')
    wflow = YadageWorkflow.createFromJSON(
        data, LocalFSProvider(LocalFSState(['/workdir']), ensure=False))
    wflow.view().init({'hello': 'world'})
    assert wflow.rules[0].rule.json()
    assert wflow.rules[1].rule.json()

    assert JsonStage.fromJSON(
        wflow.rules[0].rule.json()).json() == wflow.rules[0].rule.json()

Example #10

def test_deserialize():
    data = {
        "rules": [],
        "applied": [],
        "dag": {"nodes": [], "edges": []},
        "stepsbystage": {},
        "bookkeeping": {},
        "values": {},
    }
    wflow = YadageWorkflow.fromJSON(data)
    assert data == wflow.json()

Example #11

def test_getRule():
    data = yadage.workflow_loader.workflow("workflow.yml",
                                           "tests/testspecs/nestedmapreduce")
    wflow = YadageWorkflow.createFromJSON(
        data, LocalFSProvider(LocalFSState(["/workdir"]), ensure=False))
    wflow.view().init({"input": [1, 2, 3]})

    assert wflow.view().getRule(
        identifier=wflow.rules[0].identifier) == wflow.rules[0]
    assert (wflow.view().getRule(name=wflow.rules[0].rule.name,
                                 offset="") == wflow.rules[0])
    assert wflow.view().getRule(name="nonexistent") == None

Example #12

def test_init():
    data  = yadage.workflow_loader.workflow('workflow.yml','tests/testspecs/nestedmapreduce')
    wflow = YadageWorkflow.createFromJSON(data,LocalFSProvider(LocalFSState(['/workdir']), ensure = False))
    view  = wflow.view()

    nrules_before = len(wflow.rules)
    view.init({'hello':'world'})
    nrules_after = len(wflow.rules)
    assert nrules_before + 1 == nrules_after
    view.rules[-1].apply(wflow)

    matches = view.query('init',view.steps)
    assert len(matches) == 1

Example #13

File: test_stage.py Project: matthewfeickert/yadage

def test_serialize_deserialize():
    data = yadage.workflow_loader.workflow(
        "workflow.yml", "tests/testspecs/local-helloworld"
    )
    wflow = YadageWorkflow.createFromJSON(
        data, LocalFSProvider(LocalFSState(["/workdir"]), ensure=False)
    )
    wflow.view().init({"hello": "world"})
    assert wflow.rules[0].rule.json()
    assert wflow.rules[1].rule.json()

    assert (
        JsonStage.fromJSON(wflow.rules[0].rule.json()).json()
        == wflow.rules[0].rule.json()
    )

Example #14

def test_init():
    data = yadage.workflow_loader.workflow("workflow.yml",
                                           "tests/testspecs/nestedmapreduce")
    wflow = YadageWorkflow.createFromJSON(
        data, LocalFSProvider(LocalFSState(["/workdir"]), ensure=False))
    view = wflow.view()

    nrules_before = len(wflow.rules)
    view.init({"hello": "world"})
    nrules_after = len(wflow.rules)
    assert nrules_before + 1 == nrules_after
    view.rules[-1].apply(wflow)

    matches = view.query("init", view.steps)
    assert len(matches) == 1

Example #15

File: test_predicates.py Project: vvolkl/yadage

def test_multistepstage_schedule_wflows(localfs_state_provider):
    data = yadage.workflow_loader.workflow('workflow.yml',
                                           'tests/testspecs/nestedmapreduce')
    wflow = YadageWorkflow.createFromJSON(data, localfs_state_provider)

    inputdata = [1, 2, 3]
    wflow.view().init({'input': inputdata})
    assert wflow.view().getRule(name='map').applicable(wflow) == False

    wflow.view().getRule(name='init').apply(wflow)
    assert len(wflow.dag.nodes()) == 1

    frommodel_controller('', {}, wflow).sync_backend()
    assert wflow.view().getRule(name='map').applicable(wflow) == True

    wflow.view().getRule(name='map').apply(wflow)
    assert wflow.view().getRule(name='reduce').applicable(wflow) == False

Example #16

File: test_predicates.py Project: lukasheinrich/yadage

def test_multistepstage_schedule_wflows(localfs_state_provider):
    data  = yadage.workflow_loader.workflow('workflow.yml','tests/testspecs/nestedmapreduce')
    wflow = YadageWorkflow.createFromJSON(data,localfs_state_provider)


    inputdata = [1,2,3]
    wflow.view().init({'input':inputdata})
    assert wflow.view().getRule(name = 'map').applicable(wflow) == False

    wflow.view().getRule(name = 'init').apply(wflow)
    assert len(wflow.dag.nodes()) == 1

    frommodel_controller('',{},wflow).sync_backend()
    assert wflow.view().getRule(name = 'map').applicable(wflow) == True


    wflow.view().getRule(name = 'map').apply(wflow)
    assert wflow.view().getRule(name = 'reduce').applicable(wflow) == False

Example #17

File: test_wflow.py Project: lukasheinrich/yadage

def test_view():
    wflow = YadageWorkflow()
    wflow.view()

Example #18

File: test_wflow.py Project: lukasheinrich/yadage

def test_serialize_deserialize(local_helloworld_wflow):
    wflow = local_helloworld_wflow
    assert YadageWorkflow.fromJSON(wflow.json()).json() == wflow.json()

Example #19

File: conftest.py Project: vvolkl/yadage

def jqnodestruct(tmpdir, localfs_state_provider):
    '''a workflow object with horizontally scalable map stage scheduling sub-workflows'''
    data = yadage.workflow_loader.workflow('workflow.yml',
                                           'tests/testspecs/jqnodestruct')
    wflow = YadageWorkflow.createFromJSON(data, localfs_state_provider)
    return wflow

Example #20

File: conftest.py Project: lukasheinrich/yadage

def singlestage_cases(tmpdir,localfs_state_provider):
    '''a workflow object with horizontally scalable map stage scheduling sub-workflows'''
    data  = yadage.workflow_loader.workflow('workflow.yml','tests/testspecs/casesflow/single')
    wflow = YadageWorkflow.createFromJSON(data,localfs_state_provider)
    return wflow

Example #21

File: conftest.py Project: lukasheinrich/yadage

def value_registering_workflow(tmpdir,localfs_state_provider):
    '''a workflow object with horizontally scalable map stage scheduling sub-workflows'''
    data  = yadage.workflow_loader.workflow('workflow.yml','tests/testspecs/registering_values')
    wflow = YadageWorkflow.createFromJSON(data,localfs_state_provider)
    return wflow

Example #22

File: conftest.py Project: lukasheinrich/yadage

def local_helloworld_wflow_w_init(tmpdir,localfs_state_provider):
    '''a workflow object with horizontally scalable map stage scheduling sub-workflows'''
    data  = yadage.workflow_loader.workflow('workflow.yml','tests/testspecs/local-helloworld')
    wflow = YadageWorkflow.createFromJSON(data,localfs_state_provider)
    wflow.view().init({'par': 'hello'})
    return wflow

Example #23

File: conftest.py Project: lukasheinrich/yadage

def batched_zip_mapreduce(tmpdir,localfs_state_provider):
    '''a workflow object with horizontally scalable map stage scheduling sub-workflows'''
    data  = yadage.workflow_loader.workflow('workflow.yml','tests/testspecs/mapreduce_batched')
    wflow = YadageWorkflow.createFromJSON(data,localfs_state_provider)
    return wflow

Example #24

def test_serialize():
    wflow = YadageWorkflow()
    data = wflow.json()
    json.dumps(data)

Example #25

File: test_wflow.py Project: vvolkl/yadage

def test_deserialize():
    data = {'rules': [], 'applied': [], 'dag': {'nodes': [], 'edges': []}, 'stepsbystage': {}, 'bookkeeping': {}, 'values': {}}
    wflow = YadageWorkflow.fromJSON(data)
    assert data == wflow.json()

Example #26

File: test_view.py Project: lukasheinrich/yadage

def test_serialize_offsetstage():
    data  = yadage.workflow_loader.workflow('workflow.yml','tests/testspecs/nestedmapreduce')
    wflow = YadageWorkflow.createFromJSON(data,LocalFSProvider(LocalFSState(['/workdir']), ensure = False))
    wflow.view().init({'input':[1,2,3]})
    for x in wflow.rules:
        assert OffsetStage.fromJSON(x.json()).json() == x.json()

Example #27

def test_view():
    wflow = YadageWorkflow()
    wflow.view()

Example #28

def test_create():
    wflow = YadageWorkflow()

Example #29

def test_serialize_deserialize(local_helloworld_wflow):
    wflow = local_helloworld_wflow
    assert YadageWorkflow.fromJSON(wflow.json()).json() == wflow.json()

Example #30

File: test_wflow.py Project: lukasheinrich/yadage

def test_serialize():
    wflow = YadageWorkflow()
    data = wflow.json()
    json.dumps(data)