def test_replace_and_detour_flow(lpad, mongo_jobstore, fw_dir,
replace_and_detour_flow, capsys):
from fireworks.core.rocket_launcher import rapidfire
from jobflow.managers.fireworks import flow_to_workflow
flow = replace_and_detour_flow()
uuid1 = flow.jobs[0].uuid
uuid3 = flow.jobs[1].uuid
wf = flow_to_workflow(flow, mongo_jobstore)
fw_ids = lpad.add_wf(wf)
# run the workflow
rapidfire(lpad)
# check workflow completed
fw_id = list(fw_ids.values())[0]
wf = lpad.get_wf_by_fw_id(fw_id)
uuids = [fw.tasks[0]["job"].uuid for fw in wf.fws]
uuid2 = [u for u in uuids if u != uuid1 and u != uuid3][0]
assert all([s == "COMPLETED" for s in wf.fw_states.values()])
# check store has the activity output
result1 = mongo_jobstore.query_one({"uuid": uuid1, "index": 1})
result2 = mongo_jobstore.query_one({"uuid": uuid1, "index": 2})
result3 = mongo_jobstore.query_one({"uuid": uuid2, "index": 1})
result4 = mongo_jobstore.query_one({"uuid": uuid3, "index": 1})
assert result1["output"] == 11
assert result2["output"] == "11_end"
assert result3["output"] == "xyz_end"
assert result4["output"] == "12345_end"
def test_simple_flow(lpad, mongo_jobstore, fw_dir, simple_flow, capsys):
from fireworks.core.rocket_launcher import rapidfire
from jobflow.managers.fireworks import flow_to_workflow
flow = simple_flow()
uuid = flow.jobs[0].uuid
wf = flow_to_workflow(flow, mongo_jobstore)
fw_ids = lpad.add_wf(wf)
# run the workflow
rapidfire(lpad)
# check workflow completed
fw_id = list(fw_ids.values())[0]
wf = lpad.get_wf_by_fw_id(fw_id)
assert all([s == "COMPLETED" for s in wf.fw_states.values()])
# check store has the activity output
result = mongo_jobstore.query_one({"uuid": uuid})
assert result["output"] == "12345_end"
# check logs printed
captured = capsys.readouterr()
assert "INFO Starting job - func" in captured.out
assert "INFO Finished job - func" in captured.out
def test_stop_children_flow(lpad, mongo_jobstore, fw_dir, stop_children_flow,
capsys):
from collections import Counter
from fireworks.core.rocket_launcher import rapidfire
from jobflow.managers.fireworks import flow_to_workflow
flow = stop_children_flow()
uuid1 = flow.jobs[0].uuid
uuid2 = flow.jobs[1].uuid
uuid3 = flow.jobs[2].uuid
wf = flow_to_workflow(flow, mongo_jobstore)
fw_ids = lpad.add_wf(wf)
# run the workflow
rapidfire(lpad)
# check workflow completed
fw_id = list(fw_ids.values())[0]
wf = lpad.get_wf_by_fw_id(fw_id)
states = Counter(wf.fw_states.values())
assert states["COMPLETED"] == 2
assert states["DEFUSED"] == 1
# check store has the activity output
result1 = mongo_jobstore.query_one({"uuid": uuid1})
result2 = mongo_jobstore.query_one({"uuid": uuid2})
result3 = mongo_jobstore.query_one({"uuid": uuid3})
assert result1["output"] == "1234"
assert result2 is None
assert result3["output"] == "12345_end"
def test_detour_stop_flow(lpad, mongo_jobstore, fw_dir, detour_stop_flow,
capsys):
from fireworks.core.rocket_launcher import rapidfire
from jobflow.managers.fireworks import flow_to_workflow
flow = detour_stop_flow()
uuid1 = flow.jobs[0].uuid
uuid3 = flow.jobs[1].uuid
wf = flow_to_workflow(flow, mongo_jobstore)
fw_ids = lpad.add_wf(wf)
# run the workflow
rapidfire(lpad)
# check workflow completed
fw_id = list(fw_ids.values())[0]
wf = lpad.get_wf_by_fw_id(fw_id)
uuids = [fw.tasks[0]["job"].uuid for fw in wf.fws]
uuid2 = [u for u in uuids if u != uuid1 and u != uuid3][0]
assert list(wf.fw_states.values()) == ["DEFUSED", "COMPLETED", "COMPLETED"]
# check store has the activity output
result1 = mongo_jobstore.query_one({"uuid": uuid1})
result2 = mongo_jobstore.query_one({"uuid": uuid2})
result3 = mongo_jobstore.query_one({"uuid": uuid3})
assert result1["output"] == 11
assert result2["output"] == "1234"
assert result3 is None
def test_stop_jobflow_job(lpad, mongo_jobstore, fw_dir, stop_jobflow_job,
capsys):
from fireworks.core.rocket_launcher import rapidfire
from jobflow.managers.fireworks import flow_to_workflow
job = stop_jobflow_job()
uuid1 = job.uuid
wf = flow_to_workflow(job, mongo_jobstore)
fw_ids = lpad.add_wf(wf)
# run the workflow
rapidfire(lpad)
# check workflow completed
fw_id = list(fw_ids.values())[0]
wf = lpad.get_wf_by_fw_id(fw_id)
assert list(wf.fw_states.values()) == ["COMPLETED"]
# check store has the activity output
result1 = mongo_jobstore.query_one({"uuid": uuid1})
assert result1["output"] == "1234"
def test_replace_flow(lpad, mongo_jobstore, fw_dir, replace_flow, capsys):
from fireworks.core.rocket_launcher import rapidfire
from jobflow.managers.fireworks import flow_to_workflow
flow = replace_flow()
uuid1 = flow.jobs[0].uuid
uuid2 = flow.jobs[1].uuid
wf = flow_to_workflow(flow, mongo_jobstore)
fw_ids = lpad.add_wf(wf)
# run the workflow
rapidfire(lpad)
# check workflow completed
fw_id = list(fw_ids.values())[0]
wf = lpad.get_wf_by_fw_id(fw_id)
assert all([s == "COMPLETED" for s in wf.fw_states.values()])
# check store has the activity output
result1 = mongo_jobstore.query_one({"uuid": uuid1, "index": 1})
result2 = mongo_jobstore.query_one({"uuid": uuid1, "index": 2})
result3 = mongo_jobstore.query_one({"uuid": uuid2, "index": 1})
assert result1["output"] == 11
assert result2["output"] == "11_end"
assert result3["output"] == "12345_end"
# assert job2 (replaced job) ran before job3
assert result2["completed_at"] < result3["completed_at"]
def test_nested_flow(lpad, mongo_jobstore, fw_dir, nested_flow, capsys):
from fireworks.core.rocket_launcher import rapidfire
from jobflow.managers.fireworks import flow_to_workflow
flow = nested_flow()
uuid1 = flow.jobs[0].jobs[0].uuid
uuid2 = flow.jobs[0].jobs[1].uuid
uuid3 = flow.jobs[1].jobs[0].uuid
uuid4 = flow.jobs[1].jobs[1].uuid
wf = flow_to_workflow(flow, mongo_jobstore)
fw_ids = lpad.add_wf(wf)
# run the workflow
rapidfire(lpad)
# check workflow completed
fw_id = list(fw_ids.values())[0]
wf = lpad.get_wf_by_fw_id(fw_id)
assert all([s == "COMPLETED" for s in wf.fw_states.values()])
# check store has the activity output
result1 = mongo_jobstore.query_one({"uuid": uuid1})
result2 = mongo_jobstore.query_one({"uuid": uuid2})
result3 = mongo_jobstore.query_one({"uuid": uuid3})
result4 = mongo_jobstore.query_one({"uuid": uuid4})
assert result1["output"] == "12345_end"
assert result2["output"] == "12345_end_end"
assert result3["output"] == "12345_end_end_end"
assert result4["output"] == "12345_end_end_end_end"
def test_simple_flow_metadata(lpad, mongo_jobstore, fw_dir, simple_flow,
capsys):
from fireworks.core.rocket_launcher import rapidfire
from jobflow.managers.fireworks import flow_to_workflow
flow = simple_flow()
uuid = flow.jobs[0].uuid
flow.jobs[0].metadata = {"tags": ["my_flow"]}
wf = flow_to_workflow(flow, mongo_jobstore)
fw_ids = lpad.add_wf(wf)
# run the workflow
rapidfire(lpad)
# check workflow completed
fw_id = list(fw_ids.values())[0]
wf = lpad.get_wf_by_fw_id(fw_id)
assert all([s == "COMPLETED" for s in wf.fw_states.values()])
assert wf.fws[0].spec["tags"] == ["my_flow"]
# check store has the activity output
result = mongo_jobstore.query_one({"uuid": uuid})
assert result["output"] == "12345_end"
assert result["metadata"]["fw_id"] == fw_id
# test override
flow = simple_flow()
flow.jobs[0].config.manager_config = {"_add_launchpad_and_fw_id": False}
uuid = flow.jobs[0].uuid
wf = flow_to_workflow(flow, mongo_jobstore)
lpad.add_wf(wf)
# run the workflow
rapidfire(lpad)
result = mongo_jobstore.query_one({"uuid": uuid})
assert result["metadata"] == {}
def test_error_flow(lpad, mongo_jobstore, fw_dir, error_flow):
from fireworks.core.rocket_launcher import rapidfire
from jobflow.managers.fireworks import flow_to_workflow
flow = error_flow()
uuid = flow.jobs[0].uuid
wf = flow_to_workflow(flow, mongo_jobstore)
fw_ids = lpad.add_wf(wf)
# run the workflow
rapidfire(lpad)
# check workflow completed
fw_id = list(fw_ids.values())[0]
wf = lpad.get_wf_by_fw_id(fw_id)
assert set(wf.fw_states.values()) == {"WAITING", "FIZZLED"}
result1 = mongo_jobstore.query_one({"uuid": uuid})
assert result1 is None
def test_stored_data_flow(lpad, mongo_jobstore, fw_dir, stored_data_flow,
capsys):
from fireworks.core.rocket_launcher import rapidfire
from jobflow.managers.fireworks import flow_to_workflow
flow = stored_data_flow()
flow.jobs[0].uuid
wf = flow_to_workflow(flow, mongo_jobstore)
fw_ids = lpad.add_wf(wf)
# run the workflow
rapidfire(lpad)
# check workflow completed
fw_id = list(fw_ids.values())[0]
wf = lpad.get_wf_by_fw_id(fw_id)
assert list(wf.fw_states.values()) == ["COMPLETED"]
result = lpad.db.launches.find_one({"fw_id": fw_id})
assert result["action"]["stored_data"] == {"a": "message"}
def test_flow_to_workflow(memory_jobstore, simple_job, simple_flow,
connected_flow, nested_flow):
from fireworks import Workflow
from jobflow import OnMissing
from jobflow.managers.fireworks import flow_to_workflow
# test simple job
flow = simple_job()
wf = flow_to_workflow(flow, memory_jobstore)
assert type(wf) == Workflow
assert wf.name == "Flow"
assert len(wf.fws) == 1
assert wf.fws[0].name == "func"
# test simple job no store
flow = simple_job()
wf = flow_to_workflow(flow)
assert type(wf) == Workflow
assert wf.name == "Flow"
assert len(wf.fws) == 1
assert wf.fws[0].name == "func"
# test simple flow
flow = simple_flow()
wf = flow_to_workflow(flow, memory_jobstore)
assert type(wf) == Workflow
assert wf.name == "Flow"
assert len(wf.fws) == 1
assert wf.fws[0].name == "func"
# test connected flow
flow = connected_flow()
wf = flow_to_workflow(flow, memory_jobstore)
assert type(wf) == Workflow
assert wf.name == "Connected Flow"
assert len(wf.fws) == 2
assert wf.fws[0].name == "func"
assert wf.fws[1].name == "func"
# test nested flow
flow = nested_flow()
wf = flow_to_workflow(flow, memory_jobstore)
assert len(wf.fws) == 4
assert wf.fws[0].name == "func"
assert wf.fws[1].name == "func"
assert wf.fws[2].name == "func"
assert wf.fws[3].name == "func"
# test on missing causes allow fizzled parents
flow = connected_flow()
flow.jobs[0].config.on_missing_references = OnMissing.NONE
wf = flow_to_workflow(flow, memory_jobstore)
assert wf.fws[0].spec["_allow_fizzled_parents"] is True
# test manager config
flow = connected_flow()
flow.jobs[0].config.manager_config = {"metadata": 5}
wf = flow_to_workflow(flow, memory_jobstore)
assert wf.fws[0].spec["metadata"] == 5
