def test_packed_workflow_execution(wf_path, job_path, namespaced, tmpdir):
loadingContext = LoadingContext()
loadingContext.resolver = tool_resolver
loadingContext, workflowobj, uri = fetch_document(
get_data(wf_path), loadingContext)
loadingContext.do_update = False
loadingContext, uri = resolve_and_validate_document(
loadingContext, workflowobj, uri)
processobj = loadingContext.loader.resolve_ref(uri)[0]
packed = json.loads(print_pack(loadingContext.loader, processobj, uri, loadingContext.metadata))
assert not namespaced or "$namespaces" in packed
wf_packed_handle, wf_packed_path = tempfile.mkstemp()
with open(wf_packed_path, 'w') as temp_file:
json.dump(packed, temp_file)
normal_output = StringIO()
packed_output = StringIO()
normal_params = ['--outdir', str(tmpdir), get_data(wf_path), get_data(job_path)]
packed_params = ['--outdir', str(tmpdir), '--debug', wf_packed_path, get_data(job_path)]
assert main(normal_params, stdout=normal_output) == 0
assert main(packed_params, stdout=packed_output) == 0
assert json.loads(packed_output.getvalue()) == json.loads(normal_output.getvalue())
os.close(wf_packed_handle)
os.remove(wf_packed_path)
def test_get_subgraph() -> None:
"""Compare known correct subgraphs to generated subgraphs."""
loading_context = LoadingContext({"construct_tool_object": default_make_tool})
wf = Path(get_data("tests/subgraph/count-lines1-wf.cwl")).as_uri()
loading_context.do_update = False
tool = load_tool(wf, loading_context)
sg = Path(get_data("tests/subgraph")).as_uri()
for a in (
"file1",
"file2",
"file3",
"count_output",
"output3",
"output4",
"output5",
"step1",
"step2",
"step3",
"step4",
"step5",
):
assert isinstance(tool, Workflow)
extracted = get_subgraph([wf + "#" + a], tool, loading_context)
with open(get_data("tests/subgraph/extract_" + a + ".json")) as f:
assert json.load(f) == clean(convert_to_dict(extracted), sg)
def test_get_subgraph():
loadingContext = LoadingContext({"construct_tool_object": default_make_tool})
wf = norm(Path(get_data("tests/subgraph/count-lines1-wf.cwl")).as_uri())
loadingContext.do_update = False
tool = load_tool(wf, loadingContext)
sg = norm(Path(get_data("tests/subgraph")).as_uri())
def clean(val):
if isinstance(val, str):
if val.startswith(sg):
return val[len(sg) + 1 :]
if isinstance(val, dict):
return {k: clean(v) for k, v in val.items()}
if isinstance(val, list):
return [clean(v) for v in val]
return val
for a in (
"file1",
"file2",
"file3",
"count_output",
"output3",
"output4",
"output5",
"step1",
"step2",
"step3",
"step4",
"step5",
):
extracted = get_subgraph([wf + "#" + a], tool)
with open(get_data("tests/subgraph/extract_" + a + ".json")) as f:
assert json.load(f) == clean(convert_to_dict(extracted))
def test_get_step() -> None:
loadingContext = LoadingContext(
{"construct_tool_object": default_make_tool})
wf = Path(get_data("tests/subgraph/count-lines1-wf.cwl")).as_uri()
loadingContext.do_update = False
tool = load_tool(wf, loadingContext)
assert isinstance(tool, Workflow)
sg = Path(get_data("tests/subgraph")).as_uri()
def clean(val: Any) -> Any:
if isinstance(val, str):
if val.startswith(sg):
return val[len(sg) + 1:]
if isinstance(val, dict):
return {k: clean(v) for k, v in val.items()}
if isinstance(val, list):
return [clean(v) for v in val]
return val
for a in (
"step1",
"step2",
"step3",
"step4",
"step5",
):
extracted = get_step(tool, wf + "#" + a)
with open(get_data("tests/subgraph/single_" + a + ".json")) as f:
assert json.load(f) == clean(convert_to_dict(extracted))
def test_packed_workflow_execution(wf_path, job_path, namespaced, tmpdir):
loadingContext = LoadingContext()
loadingContext.resolver = tool_resolver
loadingContext, workflowobj, uri = fetch_document(
get_data(wf_path), loadingContext)
loadingContext.do_update = False
loadingContext, uri = resolve_and_validate_document(
loadingContext, workflowobj, uri)
processobj = loadingContext.loader.resolve_ref(uri)[0]
packed = json.loads(print_pack(loadingContext.loader, processobj, uri, loadingContext.metadata))
assert not namespaced or "$namespaces" in packed
wf_packed_handle, wf_packed_path = tempfile.mkstemp()
with open(wf_packed_path, 'w') as temp_file:
json.dump(packed, temp_file)
normal_output = StringIO()
packed_output = StringIO()
normal_params = ['--outdir', str(tmpdir), get_data(wf_path), get_data(job_path)]
packed_params = ['--outdir', str(tmpdir), '--debug', get_data(wf_packed_path), get_data(job_path)]
assert main(normal_params, stdout=normal_output) == 0
assert main(packed_params, stdout=packed_output) == 0
assert json.loads(packed_output.getvalue()) == json.loads(normal_output.getvalue())
os.close(wf_packed_handle)
os.remove(wf_packed_path)
def test_get_subgraph_long_out_form() -> None:
"""Compare subgraphs generatation when 'out' is in the long form."""
loading_context = LoadingContext({"construct_tool_object": default_make_tool})
wf = Path(get_data("tests/subgraph/1432.cwl")).as_uri()
loading_context.do_update = False
tool = load_tool(wf, loading_context)
sg = Path(get_data("tests/")).as_uri()
assert isinstance(tool, Workflow)
extracted = get_subgraph([wf + "#step2"], tool, loading_context)
with open(get_data("tests/subgraph/extract_step2_1432.json")) as f:
assert json.load(f) == clean(convert_to_dict(extracted), sg)
def test_packed_workflow_execution(wf_path: str, job_path: str,
namespaced: bool,
tmpdir: py.path.local) -> None:
loadingContext = LoadingContext()
loadingContext.resolver = tool_resolver
loadingContext, workflowobj, uri = fetch_document(get_data(wf_path),
loadingContext)
loadingContext.do_update = False
loadingContext, uri = resolve_and_validate_document(
loadingContext, workflowobj, uri)
loader = loadingContext.loader
assert loader
loader.resolve_ref(uri)[0]
packed = json.loads(print_pack(loadingContext, uri))
assert not namespaced or "$namespaces" in packed
wf_packed_handle, wf_packed_path = tempfile.mkstemp()
with open(wf_packed_path, "w") as temp_file:
json.dump(packed, temp_file)
normal_output = StringIO()
packed_output = StringIO()
normal_params = [
"--outdir",
str(tmpdir),
get_data(wf_path),
get_data(job_path)
]
packed_params = [
"--outdir",
str(tmpdir),
"--debug",
wf_packed_path,
get_data(job_path),
]
assert main(normal_params, stdout=normal_output) == 0
assert main(packed_params, stdout=packed_output) == 0
assert json.loads(packed_output.getvalue()) == json.loads(
normal_output.getvalue())
os.close(wf_packed_handle)
os.remove(wf_packed_path)
def test_get_step() -> None:
loading_context = LoadingContext({"construct_tool_object": default_make_tool})
wf = Path(get_data("tests/subgraph/count-lines1-wf.cwl")).as_uri()
loading_context.do_update = False
tool = load_tool(wf, loading_context)
assert isinstance(tool, Workflow)
sg = Path(get_data("tests/subgraph")).as_uri()
for a in (
"step1",
"step2",
"step3",
"step4",
"step5",
):
extracted = get_step(tool, wf + "#" + a, loading_context)
with open(get_data("tests/subgraph/single_" + a + ".json")) as f:
assert json.load(f) == clean(convert_to_dict(extracted), sg)
def test_get_subgraph():
loadingContext = LoadingContext({"construct_tool_object": default_make_tool})
wf = norm(Path(get_data("tests/subgraph/count-lines1-wf.cwl")).as_uri())
loadingContext.do_update = False
tool = load_tool(wf, loadingContext)
sg = norm(Path(get_data("tests/subgraph")).as_uri())
def clean(val):
if isinstance(val, string_types):
if val.startswith(sg):
return val[len(sg)+1:]
if isinstance(val, dict):
return {k: clean(v) for k,v in val.items()}
if isinstance(val, list):
return [clean(v) for v in val]
return val
for a in ("file1", "file2", "file3", "count_output",
"output3", "output4", "output5",
"step1", "step2", "step3", "step4", "step5"):
extracted = get_subgraph([wf+"#"+a], tool)
with open(get_data("tests/subgraph/extract_"+a+".json")) as f:
assert json.load(f) == clean(convert_to_dict(extracted))
def run(args: argparse.Namespace) -> int:
"""Primary processing loop."""
imports: Set[str] = set()
for document in args.inputs:
_logger.info("Processing %s.", document)
with open(document) as doc_handle:
result = yaml.main.round_trip_load(doc_handle,
preserve_quotes=True)
add_lc_filename(result, document)
version = result.get("cwlVersion", None)
if version in ("draft-3", "cwl:draft-3", "v1.0", "v1.1"):
result = cwlupgrader.upgrade_document(result, False, False,
args.dir, imports)
else:
_logger.error(
"Sorry, %s in %s is not a supported CWL version by this tool.",
(version, document),
)
return -1
uri = Path(document).resolve().as_uri()
if not args.no_expression_refactoring:
refactored, _ = cwl_v1_2_expression_refactor.traverse(
load_document_by_yaml(result, uri),
not args.etools,
False,
args.skip_some1,
args.skip_some2,
)
if not isinstance(refactored, MutableSequence):
result = save(
refactored,
base_url=refactored.loadingOptions.fileuri
if refactored.loadingOptions.fileuri else "",
)
# ^^ Setting the base_url and keeping the default value
# for relative_uris=True means that the IDs in the generated
# JSON/YAML are kept clean of the path to the input document
else:
result = [
save(result_item,
base_url=result_item.loadingOptions.fileuri)
for result_item in refactored
]
if "$graph" in result:
packed = result
else:
with tempfile.TemporaryDirectory() as tmpdirname:
path = Path(tmpdirname) / Path(document).name
with open(path, "w") as handle:
yaml.main.round_trip_dump(result, handle)
# TODO replace the cwltool based packing with a parser_v1_2 based packer
runtimeContext = RuntimeContext()
loadingContext = LoadingContext()
use_standard_schema("v1.2")
# loadingContext.construct_tool_object = workflow.default_make_tool
# loadingContext.resolver = tool_resolver
loadingContext.do_update = False
uri, tool_file_uri = resolve_tool_uri(
str(path),
resolver=loadingContext.resolver,
fetcher_constructor=loadingContext.fetcher_constructor,
)
loadingContext, workflowobj, uri = fetch_document(
uri, loadingContext)
loadingContext, uri = resolve_and_validate_document(
loadingContext,
workflowobj,
uri,
preprocess_only=True,
skip_schemas=True,
)
packed = print_pack(loadingContext, uri)
output = Path(args.dir) / Path(document).name
with open(output, "w", encoding="utf-8") as output_filehandle:
output_filehandle.write(packed)
return 0