def execute(self) -> Tuple[UUID, Dict, Optional[Exception]]:
"""
:return: Run ID, dict with new files, exception if there is any
"""
run_id = uuid4()
runtime_context = RuntimeContext()
runtime_context.outdir = self.file_manager.ROOT_DIRECTORY
runtime_context.basedir = self.file_manager.ROOT_DIRECTORY
runtime_context.default_stdin = subprocess.DEVNULL
runtime_context.default_stdout = subprocess.DEVNULL
runtime_context.default_stderr = subprocess.DEVNULL
os.chdir(self.file_manager.ROOT_DIRECTORY)
factory = Factory(runtime_context=runtime_context)
executable = factory.make(self._workflow_path)
data = {}
for data_file in self._data_paths:
with open(data_file) as f:
new_data = yaml.load(f, Loader=yaml.Loader)
data = {**new_data, **data}
try:
result: Dict = executable(**data)
return run_id, result, None
except Exception as e:
traceback.print_exc(file=sys.stderr)
return run_id, {}, e
def test_value_from_two_concatenated_expressions() -> None:
sandboxjs.have_node_slim = False
sandboxjs.localdata = threading.local()
factory = Factory()
echo = factory.make(get_data("tests/wf/vf-concat.cwl"))
file = {"class": "File", "location": get_data("tests/wf/whale.txt")}
assert echo(file1=file) == {"out": "a string\n"}
def test_scattered_workflow() -> None:
test_file = "tests/wf/scatter-wf4.cwl"
job_file = "tests/wf/scatter-job2.json"
factory = Factory(MultithreadedJobExecutor())
echo = factory.make(get_data(test_file))
with open(get_data(job_file)) as job:
assert echo(**json.load(job)) == {
"out": ["foo one three", "foo two four"]
}
def test_sequential_workflow(tmp_path: Path) -> None:
test_file = "tests/wf/count-lines1-wf.cwl"
executor = MultithreadedJobExecutor()
runtime_context = RuntimeContext()
runtime_context.outdir = str(tmp_path)
runtime_context.select_resources = executor.select_resources
factory = Factory(executor, None, runtime_context)
echo = factory.make(get_data(test_file))
file_contents = {
"class": "File",
"location": get_data("tests/wf/whale.txt")
}
assert echo(file1=file_contents) == {"count_output": 16}
def test_value_from_two_concatenated_expressions_singularity(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
"""Javascript test using Singularity."""
sandboxjs.have_node_slim = False
sandboxjs.localdata = threading.local()
new_paths = hide_nodejs(tmp_path)
factory = Factory()
factory.loading_context.singularity = True
factory.loading_context.debug = True
factory.runtime_context.debug = True
with monkeypatch.context() as m:
m.setenv("PATH", new_paths)
echo = factory.make(get_data("tests/wf/vf-concat.cwl"))
file = {"class": "File", "location": get_data("tests/wf/whale.txt")}
assert echo(file1=file) == {"out": "a string\n"}
def get_windows_safe_factory(**execkwargs):
if onWindows():
makekwargs = {'find_default_container': functools.partial(
force_default_container, windows_default_container_id),
'use_container': True}
execkwargs['default_container'] = windows_default_container_id
else:
makekwargs = {}
return Factory(makekwargs=makekwargs, **execkwargs)
def get_windows_safe_factory(runtime_context=None, # type: RuntimeContext
loading_context=None, # type: LoadingContext
executor=None # type: Any
): # type: (...) -> Factory
if onWindows():
if not runtime_context:
runtime_context = RuntimeContext()
runtime_context.find_default_container = functools.partial(
force_default_container, windows_default_container_id)
runtime_context.use_container = True
runtime_context.default_container = windows_default_container_id
return Factory(executor, loading_context, runtime_context)
def get_windows_safe_factory(
runtime_context: Optional[RuntimeContext] = None,
loading_context: Optional[LoadingContext] = None,
executor: Optional[JobExecutor] = None,
) -> Factory:
if onWindows():
if not runtime_context:
runtime_context = RuntimeContext()
runtime_context.find_default_container = functools.partial(
force_default_container, windows_default_container_id)
runtime_context.use_container = True
runtime_context.default_container = windows_default_container_id
return Factory(executor, loading_context, runtime_context)
def __init__(
self,
tool:dict,
inputs:dict,
workdir = None, # contains tmp and out dir
# if tmp not specified separately
tmpdir = None,
use_container = True,
user_space_docker_cmd = "",
force_docker_pull = False,
singularity = False,
debug = False,
default_container = ""
):
random_string = "".join([random_choice(ascii_letters + digits) for c in range(0,14)])
self.tool = tool
self.inputs = inputs
if workdir is None:
workdir = os.getcwd()
assert (not os.path.exists(workdir)) or os.path.isdir(workdir), \
error_message(
"task_executor",
f"Message workdir exists but is not a directory: {workdir}",
is_known=True
)
self.workdir = workdir if not os.path.exists(workdir) \
else os.path.join(workdir, random_string)
os.makedirs(self.workdir)
self.inputs_file = os.path.join(self.workdir, "inputs.json")
with open(self.inputs_file, "w") as inp_file:
inp_file.write(json.dumps(self.inputs, indent=2))
self.cwl_file = os.path.join(self.workdir, "tool.cwl")
with open(self.cwl_file, "w") as cwl_file:
cwl_file.write(json.dumps(self.tool, indent=2))
self.tmpdir = tmpdir if tmpdir \
else os.path.join(self.workdir, "tmp")
os.mkdir(self.tmpdir)
self.outdir = os.path.join(self.workdir, "out")
os.mkdir(self.outdir)
self.loading_context = LoadingContext()
self.runtime_context = RuntimeContext()
# self.runtime_context.tmpdir = self.tmpdir
self.runtime_context.outdir = self.outdir
# self.runtime_context.use_container = use_container
# self.runtime_context.user_space_docker_cmd = user_space_docker_cmd
# self.runtime_context.force_docker_pull = force_docker_pull
# self.runtime_context.singularity = singularity
print(self.runtime_context.outdir)
fac = Factory(
# loading_context=self.loading_context,
runtime_context=self.runtime_context
)
self.callable_tool = fac.make(self.cwl_file)
self.success = None
self.out = {}
def test_newline_in_entry() -> None:
"""Files in a InitialWorkingDirectory are created with a newline character."""
factory = Factory()
echo = factory.make(get_data("tests/wf/iwdr-entry.cwl"))
assert echo(message="hello") == {"out": "CONFIGVAR=hello\n"}
def test_value_from_two_concatenated_expressions() -> None:
factory = Factory()
echo = factory.make(get_data("tests/wf/vf-concat.cwl"))
file = {"class": "File", "location": get_data("tests/wf/whale.txt")}
assert echo(file1=file) == {"out": "a string\n"}