def get(self, type_name, tag: Optional[str]) -> Optional[T]:
if type_name not in self.registry:
return None
tagged_objs = self.registry[type_name]
versions_without_default = set(tagged_objs.keys())
if self.default_tag in versions_without_default:
versions_without_default.remove(self.default_tag)
Logger.debug(
f"'{type_name}' has {len(versions_without_default)} versions ({', '.join(versions_without_default)})"
)
if tag is None or tag == self.default_tag:
if self.default_tag in tagged_objs:
Logger.info(
f"Using the default tag for '{type_name}' from {len(versions_without_default)} version(s): {', '.join(versions_without_default)}"
)
return tagged_objs.get(self.default_tag)[0]
return None
if tag not in tagged_objs:
Logger.log(
"Found collection '{tool}' in registry, but couldn't find tag '{tag}'"
.format(tool=type_name, tag=tag))
return None
return tagged_objs[tag]
def stop_engine(self):
if self._logger:
self._logger.terminate()
self.should_stop = True
if self._timer_thread:
self._timer_thread.set()
if self._logfp:
self._logfp.flush()
os.fsync(self._logfp.fileno())
self._logfp.close()
if not self.process_id:
Logger.warn("Could not find a cromwell process to end, SKIPPING")
return
Logger.info("Stopping cromwell")
if self.process_id:
try:
process = os.getpgid(int(self.process_id))
os.killpg(process, signal.SIGTERM)
Logger.info("Stopped cromwell")
except Exception as e:
# can't do
Logger.warn("Couldn't stop Cromwell process: " + str(e))
pass
else:
Logger.warn(
"Couldn't stop Cromwell process as Janis wasn't managing it")
self.is_started = False
def rm_dir(self, directory):
Logger.info(f"Removing local directory '{directory}'")
try:
return shutil.rmtree(directory)
except Exception as e:
Logger.critical(f"Error removing directory '{directory}': {e}")
return False
def run_delete_database_script(self, execution_dir: str):
try:
import subprocess, os
from janis_assistant.management.envvariables import EnvVariables
file_path = os.getenv(EnvVariables.db_script_generator_cleanup)
if file_path is None:
raise Exception(
f"Couldn't delete generated database credentials as couldn't find value in env var '{EnvVariables.db_script_generator_cleanup}'"
)
Logger.debug(
f"Found path '{EnvVariables.db_script_generator_cleanup}' to delete database credentials"
)
# if not os.path.exists(file_path):
# raise Exception(f"Couldn't locate script '{file_path}' to execute")
val = collect_output_from_command(f"{file_path} {execution_dir}",
stderr=Logger.guess_log,
shell=True)
if val is not None and len(val) > 0:
Logger.info(
f"Successfully deleted DB credentials and received message: {val}"
)
else:
Logger.info("Deleted credentials with rc=0")
except Exception as e:
Logger.warn(
f"Failed to delete database configuration details for execution directory '{execution_dir}': "
+ repr(e))
def rm_dir(self, directory):
import urllib.request
Logger.info(f"Issuing HTTP.DELETE request for directory '{directory}'")
req = urllib.request.Request(directory)
req.get_method = lambda: "DELETE"
return urllib.request.urlopen(req)
def terminate_task(self, identifier) -> TaskStatus:
from time import sleep
try:
url = self.url_abort(identifier)
data = parse.urlencode({}).encode()
req = request.Request(
url, data=data) # this will make the method "POST"
r = request.urlopen(req)
data = r.read()
Logger.debug("Janis has issued abort request to Cromwell: " +
str(data))
taskstatus = self.poll_task(identifier)
while taskstatus not in TaskStatus.final_states():
Logger.debug(
f"Task status ({taskstatus}) has not moved to final state after aborting..."
)
sleep(1)
taskstatus = self.poll_task(identifier)
Logger.info(
f"Workflow with Cromwell identifier ({identifier} has been terminated ({taskstatus})."
)
self.progress_callbacks.pop(identifier)
except Exception as e:
raise Exception(
f"Failed to abort workflow with id = {identifier} :: {e}")
return TaskStatus.ABORTED
def do_init(args):
stream = sys.stdout if args.stdout else None
init_template(
args.template,
stream=stream,
unparsed_init_args=args.init_params,
output_location=args.output,
force=args.force,
)
if args.ensure_cromwell:
cromwell_loc = Cromwell.resolve_jar(None)
Logger.info("Located Cromwell at: " + str(cromwell_loc))
def start_from_paths(self, wid, source_path: str, input_path: str,
deps_path: str):
from janis_assistant.data.models.preparedjob import PreparedJob
jobfile = PreparedJob.instance()
self.taskmeta = {
"start": DateUtil.now(),
"status": TaskStatus.PROCESSING,
"jobs": {},
}
config: CWLToolConfiguration = self.config
if Logger.CONSOLE_LEVEL == LogLevel.VERBOSE:
config.debug = True
config.disable_color = True
# more options
if not config.tmpdir_prefix:
config.outdir = self.execution_dir + "/"
config.tmpdir_prefix = self.execution_dir + "/"
config.leave_tmpdir = True
if jobfile.call_caching_enabled:
config.cachedir = os.path.join(self.execution_dir, "cached/")
cmd = config.build_command_line(source_path, input_path)
Logger.debug("Running command: '" + " ".join(cmd) + "'")
process = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
preexec_fn=os.setsid,
stderr=subprocess.PIPE)
self.taskmeta["status"] = TaskStatus.RUNNING
Logger.info("CWLTool has started with pid=" + str(process.pid))
self.process_id = process.pid
self._logger = CWLToolLogger(
wid,
process,
logfp=open(self.logfile, "a+"),
metadata_callback=self.task_did_update,
exit_function=self.task_did_exit,
)
return wid
def cp_to(
self,
source,
dest,
force=False,
report_progress: Optional[Callable[[float], None]] = None,
):
if force:
Logger.critical("SSHFileScheme does not support the 'force' flag")
Logger.info(
f"Secure copying (SCP) from local:{source} to {self.connectionstring}:{dest}"
)
args = ["scp", source, self.connectionstring + ":" + dest]
subprocess.call(args)
def do_prepare(args):
job, wf = prepare_from_args(args, run_prepare_processing=True)
d = job.to_dict()
WorkflowManager.write_prepared_submission_file(prepared_job=job,
output_dir=job.output_dir,
force_write=True)
script_location = os.path.join(job.output_dir, "run.sh")
Logger.info("Job prepared successfully, you can run your workflow with:")
Logger.info(f"\tsh {script_location}")
print(dict_to_yaml_string(d))
def do_init(args):
stream = sys.stdout if args.stdout else None
init_template(
args.template,
stream=stream,
unparsed_init_args=args.init_params,
output_location=args.output,
force=args.force,
)
if args.ensure_cromwell:
cromwell_loc = Cromwell.resolve_jar(
cromwelljar=None,
janiscromwellconf=None,
configdir=EnvVariables.config_dir.resolve(),
)
Logger.info("Located Cromwell at: " + str(cromwell_loc))
def stop_engine(self):
# we're going to abort!
if self.process_id:
Logger.info("Received stop_engine request for CWLTool")
try:
import signal
os.kill(self.process_id, signal.SIGTERM)
except Exception as e:
Logger.critical("Couldn't terminate CWLTool as " + str(e))
else:
Logger.critical(
"Couldn't terminate CWLTool as there was no process ID")
return self
def build_resources_input(cls,
tool,
hints,
max_cores=None,
max_mem=None,
inputs=None,
prefix=""):
from janis_core.workflow.workflow import Workflow
inputs = inputs or {}
if not isinstance(tool, Workflow):
cpus = inputs.get(f"{prefix}runtime_cpu", tool.cpus(hints) or 1)
mem = inputs.get(f"{prefix}runtime_memory", tool.memory(hints))
disk = inputs.get(f"{prefix}runtime_disks", "local-disk 60 SSD")
if max_cores and cpus > max_cores:
Logger.info(
f"Tool '{tool.tool()}' exceeded ({cpus}) max number of cores ({max_cores}), "
"this was dropped to the new maximum")
cpus = max_cores
if mem and max_mem and mem > max_mem:
Logger.info(
f"Tool '{tool.tool()}' exceeded ({mem} GB) max amount of memory ({max_mem} GB), "
"this was dropped to the new maximum")
mem = max_mem
return {
prefix + "runtime_memory": mem,
prefix + "runtime_cpu": cpus,
prefix + "runtime_disks": disk,
}
new_inputs = {}
for s in tool.step_nodes.values():
new_inputs.update(
cls.build_resources_input(
s.tool,
hints=hints,
max_cores=max_cores,
max_mem=max_mem,
prefix=prefix + s.id() + "_",
inputs=inputs,
))
return new_inputs
def generate_resources_table(
self,
hints: Dict[str, Any],
to_console=True,
to_disk=False,
output_type: str = "tsv",
):
delim = "\t" if output_type == "tsv" else ","
tools = self.get_tools()
header = ["name", "cpu", "memory (GB)"]
data = []
for t in sorted(tools.keys()):
tool = tools[t]
data.append([tool.id(), tool.cpus(hints), tool.memory(hints)])
data.sort(key=lambda a: a[0].lower())
data.insert(0, header)
if to_console:
import tabulate
print(tabulate.tabulate(data, headers="firstrow"))
if to_disk:
import csv
d = ExportPathKeywords.resolve(
ExportPathKeywords.default_no_spec, None, self.id()
)
path = d + f"resources.{output_type}"
if not os.path.isdir(d):
os.makedirs(d)
Logger.info(f"Writing resources {output_type} to '{path}'")
with open(path, "w+") as mf:
writer = csv.writer(mf, delimiter=delim)
for row in data:
writer.writerow(row)
return data
def start_from_paths(self, wid, source_path: str, input_path: str, deps_path: str):
print("TMP: " + os.getenv("TMPDIR"))
scale = ["--scale", str(self.scale)] if self.scale else []
loglevel = ["--logLevel=" + self.loglevel] if self.loglevel else []
cmd = ["toil-cwl-runner", "--stats", *loglevel, *scale, source_path, input_path]
Logger.debug("Running command: '" + " ".join(cmd) + "'")
process = subprocess.Popen(
cmd, stdout=subprocess.PIPE, preexec_fn=os.setsid, stderr=subprocess.PIPE
)
Logger.info("CWLTool has started with pid=" + str(process.pid))
for line in read_stdout(process):
if "Path to job store directory is" in line:
idx = line.index("Path to job store directory is")
Logger.critical("JOBSTORE DIR: " + line[idx + 1 :])
Logger.debug("toil: " + line)
print("finished")
def stop_engine(self):
if not self.is_started:
return Logger.debug(
"Cromwell has already shut down, skipping shut down request")
if self._logger:
self._logger.terminate()
self.should_stop = True
if self._timer_thread:
self._timer_thread.set()
if not self.process_id:
self.is_started = False
Logger.info("Janis isn't managing Cromwell, skipping the shutdown")
return
Logger.info("Stopping cromwell")
if self.process_id:
try:
process = os.getpgid(int(self.process_id))
os.killpg(process, signal.SIGTERM)
Logger.info("Stopped cromwell")
except Exception as e:
# can't do
Logger.warn("Couldn't stop Cromwell process: " + str(e))
else:
Logger.warn(
"Couldn't stop Cromwell process as Janis wasn't managing it")
Logger.debug("Setting 'cromwell.is_started' to False")
self.is_started = False
def cp_from(
self,
source,
dest,
force=False,
report_progress: Optional[Callable[[float], None]] = None,
):
if force:
Logger.critical("SSHFileScheme does not support the 'force' flag")
args = ["scp", self.connectionstring + ":" + source, dest]
if dest.endswith("bam"):
return Logger.warn(
"Manually skipped BAM file, as they're usually too big")
if os.path.exists(dest):
return Logger.log(f"Skipping as exists ({source} -> {dest}")
Logger.info(
f"Secure copying (SCP) from {self.connectionstring}:{source} to local:{dest}"
)
subprocess.call(args)
def do_run(args):
if args.job:
from os import getcwd
workflow, workflow_ref = resolve_tool(
tool=args.workflow,
name=args.name,
from_toolshed=True,
only_toolbox=args.toolbox,
force=args.no_cache,
)
# parse and load the job file
Logger.info("Specified job file, ignoring all other parameters")
d = parse_dict(get_file_from_searchname(args.job, getcwd()))
job = PreparedJob(**d, workflow_reference=workflow_ref)
else:
job, workflow = prepare_from_args(args, run_prepare_processing=False)
jobfile = run_from_jobfile(workflow, jobfile=job, wait=args.wait)
Logger.info("Exiting")
raise SystemExit
def cp_from(
self,
source,
dest,
force=False,
report_progress: Optional[Callable[[float], None]] = None,
):
import urllib.request
if os.path.exists(dest):
if not force:
return Logger.info(
f"File already exists, skipping download ('{dest}')")
os.remove(dest)
return urllib.request.urlretrieve(url=source, filename=dest)
def link_copy_or_fail(source, dest, force=False):
"""
Eventually move this to some generic util class
:param s: Source to link from
:param d: Place to link to
:return:
"""
try:
to_copy = [(source, dest)]
while len(to_copy) > 0:
s, d = to_copy.pop(0)
if os.path.exists(d) and force:
Logger.info(f"Destination exists, overwriting '{d}'")
if os.path.isdir(d):
rmtree(d)
else:
os.remove(d)
Logger.info(f"Hard linking {s} → {d}")
if os.path.isdir(s):
os.makedirs(d, exist_ok=True)
for f in os.listdir(s):
to_copy.append((os.path.join(s, f), os.path.join(d,
f)))
continue
try:
os.link(s, d)
except FileExistsError:
Logger.critical(
"The file 'd' already exists. The force flag is required to overwrite."
)
except Exception as e:
Logger.warn("Couldn't link file: " + str(e))
# if this fails, it should error
Logger.info(f"Copying file {s} → {d}")
copyfile(s, d)
except Exception as e:
Logger.critical(
f"An unexpected error occurred when link/copying {s}: {e}")
def do_wait(args):
wids = args.wid
statuses = {}
for wid in wids:
wm = ConfigManager.get_from_path_or_submission_lazy(wid, readonly=True)
Logger.info(f"Waiting for '{wid}' to finish")
status = wm.database.get_uncached_status()
while not status.is_in_final_state():
sleep(2)
status = wm.database.get_uncached_status()
statuses[wid] = (wm.submission_id, status)
Logger.info(
f"Workflow {wid} finished with status: {status.to_string()}")
collapsed_status = TaskStatus.collapse_states(
[s[1] for s in statuses.values()])
rc = collapsed_status.get_exit_code()
Logger.info(
f"All workflows finished with collapsed status {collapsed_status.to_string()}, exiting with rc={rc}"
)
sys.exit(rc)
def stop_engine(self):
Logger.info(
"Toil doesn't run in a server mode, an instance will "
"be automatically terminated when a task is finished"
)
def start_engine(self):
Logger.info(
"Toil doesn't run in a server mode, an instance will "
"automatically be started when a task is created"
)
return self
def mkdirs(self, directory):
args = ["ssh", self.connectionstring, "-p", directory]
Logger.info(
f"Securely making directory {self.connectionstring}:{directory} through ssh"
)
subprocess.call(args)
def rm_dir(self, directory):
args = ["ssh", self.connectionstring, "rm -r " + directory]
Logger.info(
f"Secure deleting directory {self.connectionstring}:{directory} through ssh"
)
subprocess.call(args)
def translate(
self,
workflow,
to_console=True,
tool_to_console=False,
with_docker=True,
with_resource_overrides=False,
to_disk=False,
write_inputs_file=True,
export_path=ExportPathKeywords.default,
should_validate=False,
should_zip=True,
merge_resources=False,
hints=None,
allow_null_if_not_optional=True,
additional_inputs: Dict = None,
max_cores=None,
max_mem=None,
):
# self.validate_inputs(workflow._inputs, allow_null_if_not_optional)
tr_wf, tr_tools = self.translate_workflow(
workflow,
with_docker=with_docker,
with_resource_overrides=with_resource_overrides,
)
tr_inp = self.build_inputs_file(
workflow,
recursive=False,
merge_resources=merge_resources,
hints=hints,
additional_inputs=additional_inputs,
max_cores=max_cores,
max_mem=max_mem,
)
tr_res = self.build_resources_input(workflow, hints)
str_wf = self.stringify_translated_workflow(tr_wf)
str_inp = self.stringify_translated_inputs(tr_inp)
str_tools = [(
"tools/" + self.tool_filename(t),
self.stringify_translated_workflow(tr_tools[t]),
) for t in tr_tools]
str_resources = self.stringify_translated_inputs(tr_res)
if to_console:
print("=== WORKFLOW ===")
print(str_wf)
if tool_to_console:
print("\n=== TOOLS ===")
[print(f":: {t[0]} ::\n" + t[1]) for t in str_tools]
print("\n=== INPUTS ===")
print(str_inp)
if not merge_resources and with_resource_overrides:
print("\n=== RESOURCES ===")
print(str_resources)
d = ExportPathKeywords.resolve(export_path,
workflow_spec=self.name,
workflow_name=workflow.id())
fn_workflow = self.workflow_filename(workflow)
fn_inputs = self.inputs_filename(workflow)
fn_resources = self.resources_filename(workflow)
if to_disk and write_inputs_file:
if not os.path.isdir(d):
os.makedirs(d)
with open(os.path.join(d, fn_inputs), "w+") as f:
Logger.log(f"Writing {fn_inputs} to disk")
f.write(str_inp)
Logger.log(f"Written {fn_inputs} to disk")
else:
Logger.log("Skipping writing input (yaml) job file")
if to_disk:
toolsdir = os.path.join(d, "tools")
if not os.path.isdir(toolsdir):
os.makedirs(toolsdir)
Logger.info(f"Exporting workflow files to '{d}'")
with open(os.path.join(d, fn_workflow), "w+") as wf:
Logger.log(f"Writing {fn_workflow} to disk")
wf.write(str_wf)
Logger.log(f"Wrote {fn_workflow} to disk")
for (fn_tool, str_tool) in str_tools:
with open(os.path.join(d, fn_tool), "w+") as toolfp:
Logger.log(f"Writing {fn_tool} to disk")
toolfp.write(str_tool)
Logger.log(f"Written {fn_tool} to disk")
if not merge_resources and with_resource_overrides:
print("\n=== RESOURCES ===")
with open(os.path.join(d, fn_resources), "w+") as wf:
Logger.log(f"Writing {fn_resources} to disk")
wf.write(str_wf)
Logger.log(f"Wrote {fn_resources} to disk")
print(str_resources)
import subprocess
if should_zip:
Logger.info("Zipping tools")
with Path(d):
FNULL = open(os.devnull, "w")
zip_result = subprocess.run(
["zip", "-r", "tools.zip", "tools/"], stdout=FNULL)
if zip_result.returncode == 0:
Logger.info("Zipped tools")
else:
Logger.critical(zip_result.stderr)
if should_validate:
with Path(d):
Logger.info(f"Validating outputted {self.name}")
enved_vcs = [
(os.getenv(x[1:]) if x.startswith("$") else x)
for x in self.validate_command_for(
fn_workflow, fn_inputs, "tools/", "tools.zip")
]
cwltool_result = subprocess.run(enved_vcs)
if cwltool_result.returncode == 0:
Logger.info("Exported workflow was validated by: " +
" ".join(enved_vcs))
else:
Logger.critical(cwltool_result.stderr)
return str_wf, str_inp, str_tools
def resolve_jar(cromwelljar, janiscromwellconf, configdir):
if cromwelljar:
if not os.path.exists(cromwelljar):
raise Exception(
f"Specified cromwell jar '{cromwelljar}' but this didn't exist"
)
return cromwelljar
if (janiscromwellconf and janiscromwellconf.jar
and os.path.exists(janiscromwellconf.jar)):
return janiscromwellconf.jar
fromenv = EnvVariables.cromwelljar.resolve(False)
if fromenv and os.path.exists(fromenv):
return fromenv
potentials = list(
reversed(sorted(glob(os.path.join(configdir, "cromwell-*.jar")))))
valid_paths = [p for p in potentials if os.path.exists(p)]
if len(valid_paths) > 0:
if len(valid_paths) == 0:
raise Exception(
"Couldn't find cromwelljar at any of the required paths: "
+ ", ".join(potentials))
cromwelljar = valid_paths[0]
if not cromwelljar:
progress_is_loaded = False
try:
import progressbar
progress_is_loaded = True
except:
Logger.critical("Couldn't find progressbar module")
pbar = None
def show_progress(block_num, block_size, total_size):
nonlocal pbar
if pbar is None and progress_is_loaded:
pbar = progressbar.ProgressBar(maxval=total_size)
downloaded = block_num * block_size
if downloaded < total_size:
if pbar:
pbar.update(downloaded)
else:
print(
f"\rProgress: {round(downloaded * 100 / total_size)}%",
end="",
file=sys.stderr,
)
else:
if pbar:
pbar.finish()
pbar = None
else:
print("\rCompleted download of cromwell",
file=sys.stderr)
cromwellurl, cromwellfilename = Cromwell.get_latest_cromwell_url()
Logger.info(
f"Couldn't find cromwell at any of the usual spots, downloading '{cromwellfilename}' now"
)
cromwelljar = os.path.join(configdir, cromwellfilename)
request.urlretrieve(cromwellurl, cromwelljar, show_progress)
Logger.info(f"Downloaded {cromwellfilename}")
return cromwelljar
def build_resources_input(cls,
workflow,
hints,
max_cores=None,
max_mem=None,
inputs=None,
prefix=""):
from janis_core.workflow.workflow import Workflow, Tool, CommandTool
# returns a list of key, value pairs
steps: Dict[str, Optional[Any]] = {}
inputs = inputs or {}
# prefix = ""
# if not prefix:
# prefix = ""
# else:
# prefix += ""
for s in workflow.step_nodes.values():
tool: Tool = s.tool
if isinstance(tool, CommandTool):
tool_pre = prefix + s.id() + "_"
cpus = inputs.get(f"{s.id()}_runtime_cpu",
tool.cpus(hints) or 1)
mem = inputs.get(f"{s.id()}_runtime_memory",
tool.memory(hints))
disk = inputs.get(f"{s.id()}_runtime_disks",
"local-disk 60 SSD")
if max_cores and cpus > max_cores:
Logger.info(
f"Tool '{tool.tool()}' exceeded ({cpus}) max number of cores ({max_cores}), "
"this was dropped to the new maximum")
cpus = max_cores
if max_mem and mem > max_mem:
Logger.info(
f"Tool '{tool.tool()}' exceeded ({mem} GB) max amount of memory({max_mem} GB), "
"this was dropped to the new maximum")
mem = max_mem
steps.update({
tool_pre + "runtime_memory": mem,
tool_pre + "runtime_cpu": cpus,
tool_pre + "runtime_disks": disk,
})
elif isinstance(tool, Workflow):
tool_pre = prefix + s.id() + "_"
steps.update(
cls.build_resources_input(
tool,
hints,
max_cores=max_cores,
max_mem=max_mem,
prefix=tool_pre,
inputs={
k[len(s.id()) + 1:]: v
for k, v in inputs.items() if k.startswith(s.id())
},
))
return steps
def start_engine(self, additional_cromwell_options: List[str] = None):
from ...data.models.preparedjob import PreparedJob
job = PreparedJob.instance()
self._start_time = DateUtil.now()
self.timeout = job.cromwell.timeout or 10
if self.test_connection():
Logger.info("Engine has already been started")
return self
if self.connect_to_instance:
self.is_started = True
Logger.info(
"Cromwell environment discovered, skipping local instance")
return self
if self._process:
self.is_started = True
Logger.info(
f"Discovered Cromwell instance (pid={self._process}), skipping start"
)
return self
if self.config:
with open(self.config_path, "w+") as f:
f.writelines(self.config.output())
Logger.log("Finding cromwell jar")
cromwell_loc = self.resolve_jar(self.cromwelljar, job.cromwell,
job.config_dir)
Logger.info(f"Starting cromwell ({cromwell_loc})...")
cmd = ["java", "-DLOG_MODE=standard"]
if job.cromwell and job.cromwell.memory_mb:
cmd.extend([
f"-Xmx{job.cromwell.memory_mb}M",
f"-Xms{max(job.cromwell.memory_mb//2, 1)}M",
])
# if Logger.CONSOLE_LEVEL == LogLevel.VERBOSE:
# cmd.append("-DLOG_LEVEL=DEBUG")
if additional_cromwell_options:
cmd.extend(additional_cromwell_options)
self.port = find_free_port()
self.host = f"127.0.0.1:{self.port}"
cmd.append(f"-Dwebservice.port={self.port}")
cmd.append(f"-Dwebservice.interface=127.0.0.1")
if self.config_path and os.path.exists(self.config_path):
Logger.debug("Using configuration file for Cromwell: " +
self.config_path)
cmd.append("-Dconfig.file=" + self.config_path)
cmd.extend(["-jar", cromwell_loc, "server"])
Logger.debug(f"Starting Cromwell with command: '{' '.join(cmd)}'")
self._process = subprocess.Popen(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
# preexec_fn creates a process group https://stackoverflow.com/a/4791612/
preexec_fn=os.setsid,
)
Logger.info("Cromwell is starting with pid=" + str(self._process.pid))
Logger.debug(
"Cromwell will start the HTTP server, reading logs to determine when this occurs"
)
self._logfp = open(self.logfile, "a+")
Logger.info(f"Will log Cromwell output to the file: {self.logfile}"
if bool(self._logfp) else
"Janis is NOT logging Cromwell output to a file")
for c in iter(self._process.stdout.readline,
"b"): # replace '' with b'' for Python 3
line = None
if c:
line = c.decode("utf-8").rstrip()
if not line:
rc = self._process.poll()
if rc is not None:
critical_suffix = f"Last received message '{line}'. "
Logger.critical(
f"Cromwell has exited with rc={rc}. {critical_suffix}The last lines of the logfile ({self.logfile}):"
)
Logger.critical("".join(tail(self._logfp, 10)))
return False
continue
if self._logfp and not self._logfp.closed:
self._logfp.write(line + "\n")
self._logfp.flush()
os.fsync(self._logfp.fileno())
Logger.debug("Cromwell: " + line)
# self.stdout.append(str(c))
if "service started on" in line:
self.process_id = self._process.pid
Logger.info("Service successfully started with pid=" +
str(self._process.pid))
break
# elif ansi_escape.match():
# raise Exception(cd)
self.is_started = True
if self._process:
self._logger = ProcessLogger(
process=self._process,
prefix="Cromwell: ",
logfp=self._logfp,
# exit_function=self.something_has_happened_to_cromwell,
)
return self
def run(self):
finalstatus = None
iserroring = False
try:
for c in iter(self.process.stderr.readline, "b"):
if self.should_terminate:
return
line = None
if c:
line = c.decode("utf-8").rstrip()
if not line:
if self.process.poll() is not None:
finalstatus = TaskStatus.ABORTED
Logger.warn(
f"CWLTool finished with rc={self.process.returncode} but janis "
f"was unable to capture the workflow status. Marking as aborted"
)
break
continue
if self.logfp and not self.logfp.closed:
self.logfp.write(line + "\n")
self.logfp.flush()
os.fsync(self.logfp.fileno())
lowline = line.lower().lstrip()
if lowline.startswith("error"):
Logger.critical("cwltool: " + line)
iserroring = True
elif lowline.startswith("warn"):
iserroring = False
Logger.warn("cwltool: " + line)
elif lowline.startswith("info"):
iserroring = False
Logger.info("cwltool: " + line)
self.process_metadataupdate_if_match(line)
else:
Logger.debug("cwltool: " + line)
if iserroring:
self.error = (self.error or "") + "\n" + line
if "final process status is" in lowline:
if "fail" in line.lower():
finalstatus = TaskStatus.FAILED
elif "success" in line.lower():
finalstatus = TaskStatus.COMPLETED
else:
finalstatus = TaskStatus.ABORTED
break
j = ""
Logger.info("Process has completed")
if finalstatus == TaskStatus.COMPLETED:
for c in iter(self.process.stdout.readline, "s"):
if not c:
continue
line = c.decode("utf-8").rstrip()
Logger.debug(line)
if self.logfp and not self.logfp.closed:
self.logfp.write(line + "\n")
self.logfp.flush()
os.fsync(self.logfp.fileno())
j += line
try:
self.outputs = json.loads(j)
break
except:
continue
if self.error:
Logger.critical("Janis detected a CWLTool error: " +
self.error)
Logger.info("CWLTool detected transition to terminal status: " +
str(finalstatus))
self.terminate()
if self.exit_function:
self.exit_function(self, finalstatus)
except KeyboardInterrupt:
self.should_terminate = True
print("Detected keyboard interrupt")
# raise
except Exception as e:
print("Detected another error")
raise e