def test_stage_output_selector(nested_mapreduce_wflow): wflow = nested_mapreduce_wflow inputdata = {'here':[1,2,3],'deeply':{'nested':{'this':['h','e','r','e']}},'nested_list':[[1,2,3]]} view = wflow.view() view.init({'input':inputdata}) view.getRule(name = 'init').apply(wflow) frommodel_controller('',{},wflow).sync_backend() result = exh.handlers['stage-output-selector'](view,{'stages': 'init', 'output': 'input.here'}) values = process_refs(result,wflow.dag) assert values == [inputdata['here']] result = exh.handlers['stage-output-selector'](view,{'stages': 'init', 'output': 'input.here', 'unwrap': True}) values = process_refs(result,wflow.dag) assert values == inputdata['here'] result = exh.handlers['stage-output-selector'](view,{'stages': 'init', 'output': 'input.deeply', 'unwrap': True}) values = process_refs(result,wflow.dag) assert values == inputdata['deeply'] result = exh.handlers['stage-output-selector'](view,{'stages': 'init', 'output': 'input.nested_list', 'unwrap': True, 'flatten': True}) values = process_refs(result,wflow.dag) assert values == [1,2,3] with pytest.raises(RuntimeError): result = exh.handlers['stage-output-selector'](view,{'stages': 'init', 'output': 'nonexist'})
def test_apply(): data = yadage.workflow_loader.workflow('workflow.yml','tests/testspecs/local-helloworld') wflow = YadageWorkflow.createFromJSON(data,LocalFSProvider(LocalFSState(['/workdir']), ensure = False)) wflow.view().init({'par': 'value'}) assert wflow.rules[-1].applicable(wflow) == True wflow.rules[-1].apply(wflow) frommodel_controller('',{},wflow).sync_backend() assert wflow.rules[0].applicable(wflow) == True wflow.rules[0].apply(wflow)
def test_stage_output_selector(nested_mapreduce_wflow): wflow = nested_mapreduce_wflow inputdata = { "here": [1, 2, 3], "deeply": { "nested": { "this": ["h", "e", "r", "e"] } }, "nested_list": [[1, 2, 3]], } view = wflow.view() view.init({"input": inputdata}) view.getRule(name="init").apply(wflow) frommodel_controller("", {}, wflow).sync_backend() result = exh.handlers["stage-output-selector"](view, { "stages": "init", "output": "input.here" }) values = process_refs(result, wflow.dag) assert values == [inputdata["here"]] result = exh.handlers["stage-output-selector"](view, { "stages": "init", "output": "input.here", "unwrap": True }) values = process_refs(result, wflow.dag) assert values == inputdata["here"] result = exh.handlers["stage-output-selector"](view, { "stages": "init", "output": "input.deeply", "unwrap": True }) values = process_refs(result, wflow.dag) assert values == inputdata["deeply"] result = exh.handlers["stage-output-selector"]( view, { "stages": "init", "output": "input.nested_list", "unwrap": True, "flatten": True, }, ) values = process_refs(result, wflow.dag) assert values == [1, 2, 3] with pytest.raises(RuntimeError): result = exh.handlers["stage-output-selector"](view, { "stages": "init", "output": "nonexist" })
def test_apply(): data = yadage.workflow_loader.workflow('workflow.yml', 'tests/testspecs/local-helloworld') wflow = YadageWorkflow.createFromJSON( data, LocalFSProvider(LocalFSState(['/workdir']), ensure=False)) wflow.view().init({'par': 'value'}) assert wflow.rules[-1].applicable(wflow) == True wflow.rules[-1].apply(wflow) frommodel_controller('', {}, wflow).sync_backend() assert wflow.rules[0].applicable(wflow) == True wflow.rules[0].apply(wflow)
def test_stage_output_selector(nested_mapreduce_wflow): wflow = nested_mapreduce_wflow inputdata = { 'here': [1, 2, 3], 'deeply': { 'nested': { 'this': ['h', 'e', 'r', 'e'] } }, 'nested_list': [[1, 2, 3]] } view = wflow.view() view.init({'input': inputdata}) view.getRule(name='init').apply(wflow) frommodel_controller('', {}, wflow).sync_backend() result = exh.handlers['stage-output-selector'](view, { 'stages': 'init', 'output': 'input.here' }) values = process_refs(result, wflow.dag) assert values == [inputdata['here']] result = exh.handlers['stage-output-selector'](view, { 'stages': 'init', 'output': 'input.here', 'unwrap': True }) values = process_refs(result, wflow.dag) assert values == inputdata['here'] result = exh.handlers['stage-output-selector'](view, { 'stages': 'init', 'output': 'input.deeply', 'unwrap': True }) values = process_refs(result, wflow.dag) assert values == inputdata['deeply'] result = exh.handlers['stage-output-selector'](view, { 'stages': 'init', 'output': 'input.nested_list', 'unwrap': True, 'flatten': True }) values = process_refs(result, wflow.dag) assert values == [1, 2, 3] with pytest.raises(RuntimeError): result = exh.handlers['stage-output-selector'](view, { 'stages': 'init', 'output': 'nonexist' })
def test_apply(): data = yadage.workflow_loader.workflow( "workflow.yml", "tests/testspecs/local-helloworld" ) wflow = YadageWorkflow.createFromJSON( data, LocalFSProvider(LocalFSState(["/workdir"]), ensure=False) ) wflow.view().init({"par": "value"}) assert wflow.rules[-1].applicable(wflow) == True wflow.rules[-1].apply(wflow) frommodel_controller("", {}, wflow).sync_backend() assert wflow.rules[0].applicable(wflow) == True wflow.rules[0].apply(wflow)
def test_multistepstage_schedule_wflows(localfs_state_provider): data = yadage.workflow_loader.workflow('workflow.yml', 'tests/testspecs/nestedmapreduce') wflow = YadageWorkflow.createFromJSON(data, localfs_state_provider) inputdata = [1, 2, 3] wflow.view().init({'input': inputdata}) assert wflow.view().getRule(name='map').applicable(wflow) == False wflow.view().getRule(name='init').apply(wflow) assert len(wflow.dag.nodes()) == 1 frommodel_controller('', {}, wflow).sync_backend() assert wflow.view().getRule(name='map').applicable(wflow) == True wflow.view().getRule(name='map').apply(wflow) assert wflow.view().getRule(name='reduce').applicable(wflow) == False
def test_multistepstage_schedule_wflows(localfs_state_provider): data = yadage.workflow_loader.workflow('workflow.yml','tests/testspecs/nestedmapreduce') wflow = YadageWorkflow.createFromJSON(data,localfs_state_provider) inputdata = [1,2,3] wflow.view().init({'input':inputdata}) assert wflow.view().getRule(name = 'map').applicable(wflow) == False wflow.view().getRule(name = 'init').apply(wflow) assert len(wflow.dag.nodes()) == 1 frommodel_controller('',{},wflow).sync_backend() assert wflow.view().getRule(name = 'map').applicable(wflow) == True wflow.view().getRule(name = 'map').apply(wflow) assert wflow.view().getRule(name = 'reduce').applicable(wflow) == False
def test_jqnodestruct_stage(jqnodestruct): wflow = jqnodestruct inputdata = {"scatterone": ["a", "b", "c"], "scatertwo": [1, 2, 3]} wflow.view().init(inputdata) assert wflow.rules[0].applicable(wflow) == False wflow.view().rules[-1].apply(wflow) assert len(wflow.dag.nodes()) == 1 c = frommodel_controller("", {}, wflow) c.sync_backend() assert wflow.rules[0].applicable(wflow) == True assert wflow.rules[1].applicable(wflow) == True # assert wflow.rules[0].rule.stagespec['scheduler_type'] == 'jq-stage' wflow.rules[0].apply(wflow) wflow.rules[1].apply(wflow) assert len(wflow.dag.nodes()) == 7 c = frommodel_controller("", {}, wflow) c.sync_backend() assert wflow.rules[2].applicable(wflow) == True wflow.rules[2].apply(wflow) assert len(wflow.dag.nodes()) == 8 pars = wflow.dag.getNodeByName("reduce_complex_0").task.parameters assert list(pars.json().keys()) == ["grouped_inputs"] assert set(pars["grouped_inputs"].keys()) == {"one", "two"} assert pars["grouped_inputs"]["one"] == [ wflow.dag.getNodeByName("map_one_{}".format(i)).result["outputfile"] for i in range(3) ] assert pars["grouped_inputs"]["two"] == [ wflow.dag.getNodeByName("map_two_{}".format(i)).result["outputfile"] for i in range(3) ]
def test_jqnodestruct_stage(jqnodestruct): wflow = jqnodestruct inputdata = {'scatterone': ['a', 'b', 'c'], 'scatertwo': [1, 2, 3]} wflow.view().init(inputdata) assert wflow.rules[0].applicable(wflow) == False wflow.view().rules[-1].apply(wflow) assert len(wflow.dag.nodes()) == 1 c = frommodel_controller('', {}, wflow) c.sync_backend() assert wflow.rules[0].applicable(wflow) == True assert wflow.rules[1].applicable(wflow) == True # assert wflow.rules[0].rule.stagespec['scheduler_type'] == 'jq-stage' wflow.rules[0].apply(wflow) wflow.rules[1].apply(wflow) assert len(wflow.dag.nodes()) == 7 c = frommodel_controller('', {}, wflow) c.sync_backend() assert wflow.rules[2].applicable(wflow) == True wflow.rules[2].apply(wflow) assert len(wflow.dag.nodes()) == 8 pars = wflow.dag.getNodeByName('reduce_complex_0').task.parameters assert list(pars.json().keys()) == ['grouped_inputs'] assert set(pars['grouped_inputs'].keys()) == {'one', 'two'} assert pars['grouped_inputs']['one'] == [ wflow.dag.getNodeByName('map_one_{}'.format(i)).result['outputfile'] for i in range(3) ] assert pars['grouped_inputs']['two'] == [ wflow.dag.getNodeByName('map_two_{}'.format(i)).result['outputfile'] for i in range(3) ]
def test_multistepstage_zip_schedule_steps(simple_mapreduce): wflow = simple_mapreduce inputdata = [1, 2, 3] wflow.view().init({"input": inputdata}) assert wflow.rules[0].applicable(wflow) == False wflow.view().rules[-1].apply(wflow) assert len(wflow.dag.nodes()) == 1 c = frommodel_controller("", {}, wflow) c.sync_backend() assert wflow.rules[0].applicable(wflow) == True assert wflow.rules[0].rule.stagespec["scheduler_type"] == "multistep-stage" wflow.rules[0].apply(wflow) assert len(wflow.dag.nodes()) == 1 + len(inputdata)
def test_jq_stage(jqworkflow): wflow = jqworkflow inputdata = [1, 2, 3] wflow.view().init({"parone": inputdata}) assert wflow.rules[0].applicable(wflow) == False wflow.view().rules[-1].apply(wflow) assert len(wflow.dag.nodes()) == 1 c = frommodel_controller("", {}, wflow) c.sync_backend() assert wflow.rules[0].applicable(wflow) == True assert wflow.rules[0].rule.stagespec["scheduler_type"] == "jq-stage" wflow.rules[0].apply(wflow) assert len(wflow.dag.nodes()) == 1 + len(inputdata)
def test_nested_wflow(nested_wflow): wflow = nested_wflow inputdata = "world" wflow.view().init({"input": inputdata}) assert wflow.rules[0].applicable(wflow) == False wflow.view().rules[-1].apply(wflow) assert len(wflow.dag.nodes()) == 1 c = frommodel_controller("", {}, wflow) c.sync_backend() assert wflow.rules[0].applicable(wflow) == True assert wflow.rules[0].rule.stagespec[ "scheduler_type"] == "singlestep-stage" assert len(wflow.view("").rules) == 3 # init nested higherscope wflow.rules[0].apply(wflow)
def test_multistepstage_cartesian_schedule_steps(cartesian_mapreduce): wflow = cartesian_mapreduce factor_one = [1, 2, 3] factor_two = [4, 5] wflow.view().init({"factor_one": factor_one, "factor_two": factor_two}) assert wflow.rules[0].applicable(wflow) == False wflow.view().rules[-1].apply(wflow) assert len(wflow.dag.nodes()) == 1 c = frommodel_controller("", {}, wflow) c.sync_backend() assert wflow.rules[0].applicable(wflow) == True assert wflow.rules[0].rule.stagespec["scheduler_type"] == "multistep-stage" wflow.rules[0].apply(wflow) assert len(wflow.dag.nodes()) == 1 + len(factor_one) * len(factor_two)
def test_nested_wflow(nested_wflow): wflow = nested_wflow inputdata = 'world' wflow.view().init({'input': inputdata}) assert wflow.rules[0].applicable(wflow) == False wflow.view().rules[-1].apply(wflow) assert len(wflow.dag.nodes()) == 1 c = frommodel_controller('', {}, wflow) c.sync_backend() assert wflow.rules[0].applicable(wflow) == True assert wflow.rules[0].rule.stagespec[ 'scheduler_type'] == 'singlestep-stage' assert len(wflow.view('').rules) == 3 # init nested higherscope wflow.rules[0].apply(wflow)
def test_multistepstage_schedule_wflows(nested_mapreduce_wflow): wflow = nested_mapreduce_wflow inputdata = [1, 2, 3] wflow.view().init({"input": inputdata}) assert wflow.rules[0].applicable(wflow) == False wflow.view().rules[-1].apply(wflow) assert len(wflow.dag.nodes()) == 1 c = frommodel_controller("", {}, wflow) c.sync_backend() assert wflow.rules[0].applicable(wflow) == True assert wflow.rules[0].rule.stagespec["scheduler_type"] == "multistep-stage" assert len(wflow.view("").rules) == 3 # init map reduce wflow.rules[0].apply(wflow) assert len(wflow.view( "/map/0").rules) == 3 + 2 * len(inputdata) # 3x init and stage rules
def test_singlestepstage_schedule_steps(local_helloworld_wflow): wflow = local_helloworld_wflow assert wflow.rules[0].rule.stagespec[ "scheduler_type"] == "singlestep-stage" assert wflow.rules[0].applicable(wflow) == False wflow.view().init({"par": "value"}) # apply init assert wflow.rules[-1].applicable(wflow) wflow.rules[-1].apply(wflow) assert len(wflow.dag.nodes()) == 1 # get init result c = frommodel_controller("", {}, wflow) c.sync_backend() # recheck assert wflow.rules[0].applicable(wflow) == True wflow.rules[0].apply(wflow)
def test_singlestep_cases_nochoice(singlestage_cases): wflow = singlestage_cases inputdata = {"par": -1} wflow.view().init(inputdata) right_taskspec = wflow.rules[0].rule.stagespec["cases"][1]["step"] assert wflow.rules[0].applicable(wflow) == False wflow.view().rules[-1].apply(wflow) assert len(wflow.dag.nodes()) == 1 c = frommodel_controller("", {}, wflow) c.sync_backend() assert wflow.rules[0].applicable(wflow) == True wflow.rules[0].apply(wflow) assert len( wflow.dag.nodes()) == 1 # we scheduled is b/c it hits a good case assert wflow.dag.getNodeByName("hello_world") == None
def test_singlestep_cases_first(singlestage_cases, foregroundasync_backend): wflow = singlestage_cases inputdata = {"par": 0} wflow.view().init(inputdata) right_taskspec = wflow.rules[0].rule.stagespec["cases"][0]["step"] assert wflow.rules[0].applicable(wflow) == False wflow.view().rules[-1].apply(wflow) c = frommodel_controller("", {}, wflow) c.sync_backend() assert len(wflow.dag.nodes()) == 1 assert wflow.rules[0].applicable(wflow) == True wflow.rules[0].apply(wflow) assert len( wflow.dag.nodes()) == 2 # we scheduled is b/c it hits a good case assert wflow.dag.getNodeByName("hello_world").task.spec == right_taskspec
def test_singlestep_cases_second(singlestage_cases): wflow = singlestage_cases inputdata = {'par': 1} wflow.view().init(inputdata) right_taskspec = wflow.rules[0].rule.stagespec['cases'][1]['step'] assert wflow.rules[0].applicable(wflow) == False wflow.view().rules[-1].apply(wflow) assert len(wflow.dag.nodes()) == 1 c = frommodel_controller('', {}, wflow) c.sync_backend() assert wflow.rules[0].applicable(wflow) == True wflow.rules[0].apply(wflow) assert len( wflow.dag.nodes()) == 2 # we scheduled is b/c it hits a good case assert wflow.dag.getNodeByName('hello_world').task.spec == right_taskspec
def test_value_registration(value_registering_workflow, foregroundasync_backend): wflow = value_registering_workflow inputdata = {"msg": "Hello World"} wflow.view().init(inputdata) assert wflow.rules[0].applicable(wflow) == False wflow.view().rules[-1].apply(wflow) # apply init assert len(wflow.dag.nodes()) == 1 # init applied c = frommodel_controller("", {}, wflow) c.sync_backend() assert wflow.rules[0].applicable(wflow) == True # first stage is good wflow.rules[0].apply(wflow) # apply first stage value = wflow.view().getValue("ascopedfile") assert value assert type(value) == dict assert value["expression_type"] == "stage-output-selector" assert value["stages"] == "hello" assert value["output"] == "output" assert value["unwrap"] == True
def test_value_registration(value_registering_workflow, foregroundasync_backend): wflow = value_registering_workflow inputdata = {'msg': 'Hello World'} wflow.view().init(inputdata) assert wflow.rules[0].applicable(wflow) == False wflow.view().rules[-1].apply(wflow) #apply init assert len(wflow.dag.nodes()) == 1 #init applied c = frommodel_controller('', {}, wflow) c.sync_backend() assert wflow.rules[0].applicable(wflow) == True #first stage is good wflow.rules[0].apply(wflow) #apply first stage value = wflow.view().getValue('ascopedfile') assert value assert type(value) == dict assert value['expression_type'] == 'stage-output-selector' assert value['stages'] == 'hello' assert value['output'] == 'output' assert value['unwrap'] == True