def get(self, type_name, tag: Optional[str]) -> Optional[T]: if type_name not in self.registry: return None tagged_objs = self.registry[type_name] versions_without_default = set(tagged_objs.keys()) if self.default_tag in versions_without_default: versions_without_default.remove(self.default_tag) Logger.debug( f"'{type_name}' has {len(versions_without_default)} versions ({', '.join(versions_without_default)})" ) if tag is None or tag == self.default_tag: if self.default_tag in tagged_objs: Logger.info( f"Using the default tag for '{type_name}' from {len(versions_without_default)} version(s): {', '.join(versions_without_default)}" ) return tagged_objs.get(self.default_tag)[0] return None if tag not in tagged_objs: Logger.log( "Found collection '{tool}' in registry, but couldn't find tag '{tag}'" .format(tool=type_name, tag=tag)) return None return tagged_objs[tag]
def stop_engine(self): if self._logger: self._logger.terminate() self.should_stop = True if self._timer_thread: self._timer_thread.set() if self._logfp: self._logfp.flush() os.fsync(self._logfp.fileno()) self._logfp.close() if not self.process_id: Logger.warn("Could not find a cromwell process to end, SKIPPING") return Logger.info("Stopping cromwell") if self.process_id: try: process = os.getpgid(int(self.process_id)) os.killpg(process, signal.SIGTERM) Logger.info("Stopped cromwell") except Exception as e: # can't do Logger.warn("Couldn't stop Cromwell process: " + str(e)) pass else: Logger.warn( "Couldn't stop Cromwell process as Janis wasn't managing it") self.is_started = False
def rm_dir(self, directory): Logger.info(f"Removing local directory '{directory}'") try: return shutil.rmtree(directory) except Exception as e: Logger.critical(f"Error removing directory '{directory}': {e}") return False
def run_delete_database_script(self, execution_dir: str): try: import subprocess, os from janis_assistant.management.envvariables import EnvVariables file_path = os.getenv(EnvVariables.db_script_generator_cleanup) if file_path is None: raise Exception( f"Couldn't delete generated database credentials as couldn't find value in env var '{EnvVariables.db_script_generator_cleanup}'" ) Logger.debug( f"Found path '{EnvVariables.db_script_generator_cleanup}' to delete database credentials" ) # if not os.path.exists(file_path): # raise Exception(f"Couldn't locate script '{file_path}' to execute") val = collect_output_from_command(f"{file_path} {execution_dir}", stderr=Logger.guess_log, shell=True) if val is not None and len(val) > 0: Logger.info( f"Successfully deleted DB credentials and received message: {val}" ) else: Logger.info("Deleted credentials with rc=0") except Exception as e: Logger.warn( f"Failed to delete database configuration details for execution directory '{execution_dir}': " + repr(e))
def rm_dir(self, directory): import urllib.request Logger.info(f"Issuing HTTP.DELETE request for directory '{directory}'") req = urllib.request.Request(directory) req.get_method = lambda: "DELETE" return urllib.request.urlopen(req)
def terminate_task(self, identifier) -> TaskStatus: from time import sleep try: url = self.url_abort(identifier) data = parse.urlencode({}).encode() req = request.Request( url, data=data) # this will make the method "POST" r = request.urlopen(req) data = r.read() Logger.debug("Janis has issued abort request to Cromwell: " + str(data)) taskstatus = self.poll_task(identifier) while taskstatus not in TaskStatus.final_states(): Logger.debug( f"Task status ({taskstatus}) has not moved to final state after aborting..." ) sleep(1) taskstatus = self.poll_task(identifier) Logger.info( f"Workflow with Cromwell identifier ({identifier} has been terminated ({taskstatus})." ) self.progress_callbacks.pop(identifier) except Exception as e: raise Exception( f"Failed to abort workflow with id = {identifier} :: {e}") return TaskStatus.ABORTED
def do_init(args): stream = sys.stdout if args.stdout else None init_template( args.template, stream=stream, unparsed_init_args=args.init_params, output_location=args.output, force=args.force, ) if args.ensure_cromwell: cromwell_loc = Cromwell.resolve_jar(None) Logger.info("Located Cromwell at: " + str(cromwell_loc))
def start_from_paths(self, wid, source_path: str, input_path: str, deps_path: str): from janis_assistant.data.models.preparedjob import PreparedJob jobfile = PreparedJob.instance() self.taskmeta = { "start": DateUtil.now(), "status": TaskStatus.PROCESSING, "jobs": {}, } config: CWLToolConfiguration = self.config if Logger.CONSOLE_LEVEL == LogLevel.VERBOSE: config.debug = True config.disable_color = True # more options if not config.tmpdir_prefix: config.outdir = self.execution_dir + "/" config.tmpdir_prefix = self.execution_dir + "/" config.leave_tmpdir = True if jobfile.call_caching_enabled: config.cachedir = os.path.join(self.execution_dir, "cached/") cmd = config.build_command_line(source_path, input_path) Logger.debug("Running command: '" + " ".join(cmd) + "'") process = subprocess.Popen(cmd, stdout=subprocess.PIPE, preexec_fn=os.setsid, stderr=subprocess.PIPE) self.taskmeta["status"] = TaskStatus.RUNNING Logger.info("CWLTool has started with pid=" + str(process.pid)) self.process_id = process.pid self._logger = CWLToolLogger( wid, process, logfp=open(self.logfile, "a+"), metadata_callback=self.task_did_update, exit_function=self.task_did_exit, ) return wid
def cp_to( self, source, dest, force=False, report_progress: Optional[Callable[[float], None]] = None, ): if force: Logger.critical("SSHFileScheme does not support the 'force' flag") Logger.info( f"Secure copying (SCP) from local:{source} to {self.connectionstring}:{dest}" ) args = ["scp", source, self.connectionstring + ":" + dest] subprocess.call(args)
def do_prepare(args): job, wf = prepare_from_args(args, run_prepare_processing=True) d = job.to_dict() WorkflowManager.write_prepared_submission_file(prepared_job=job, output_dir=job.output_dir, force_write=True) script_location = os.path.join(job.output_dir, "run.sh") Logger.info("Job prepared successfully, you can run your workflow with:") Logger.info(f"\tsh {script_location}") print(dict_to_yaml_string(d))
def do_init(args): stream = sys.stdout if args.stdout else None init_template( args.template, stream=stream, unparsed_init_args=args.init_params, output_location=args.output, force=args.force, ) if args.ensure_cromwell: cromwell_loc = Cromwell.resolve_jar( cromwelljar=None, janiscromwellconf=None, configdir=EnvVariables.config_dir.resolve(), ) Logger.info("Located Cromwell at: " + str(cromwell_loc))
def stop_engine(self): # we're going to abort! if self.process_id: Logger.info("Received stop_engine request for CWLTool") try: import signal os.kill(self.process_id, signal.SIGTERM) except Exception as e: Logger.critical("Couldn't terminate CWLTool as " + str(e)) else: Logger.critical( "Couldn't terminate CWLTool as there was no process ID") return self
def build_resources_input(cls, tool, hints, max_cores=None, max_mem=None, inputs=None, prefix=""): from janis_core.workflow.workflow import Workflow inputs = inputs or {} if not isinstance(tool, Workflow): cpus = inputs.get(f"{prefix}runtime_cpu", tool.cpus(hints) or 1) mem = inputs.get(f"{prefix}runtime_memory", tool.memory(hints)) disk = inputs.get(f"{prefix}runtime_disks", "local-disk 60 SSD") if max_cores and cpus > max_cores: Logger.info( f"Tool '{tool.tool()}' exceeded ({cpus}) max number of cores ({max_cores}), " "this was dropped to the new maximum") cpus = max_cores if mem and max_mem and mem > max_mem: Logger.info( f"Tool '{tool.tool()}' exceeded ({mem} GB) max amount of memory ({max_mem} GB), " "this was dropped to the new maximum") mem = max_mem return { prefix + "runtime_memory": mem, prefix + "runtime_cpu": cpus, prefix + "runtime_disks": disk, } new_inputs = {} for s in tool.step_nodes.values(): new_inputs.update( cls.build_resources_input( s.tool, hints=hints, max_cores=max_cores, max_mem=max_mem, prefix=prefix + s.id() + "_", inputs=inputs, )) return new_inputs
def generate_resources_table( self, hints: Dict[str, Any], to_console=True, to_disk=False, output_type: str = "tsv", ): delim = "\t" if output_type == "tsv" else "," tools = self.get_tools() header = ["name", "cpu", "memory (GB)"] data = [] for t in sorted(tools.keys()): tool = tools[t] data.append([tool.id(), tool.cpus(hints), tool.memory(hints)]) data.sort(key=lambda a: a[0].lower()) data.insert(0, header) if to_console: import tabulate print(tabulate.tabulate(data, headers="firstrow")) if to_disk: import csv d = ExportPathKeywords.resolve( ExportPathKeywords.default_no_spec, None, self.id() ) path = d + f"resources.{output_type}" if not os.path.isdir(d): os.makedirs(d) Logger.info(f"Writing resources {output_type} to '{path}'") with open(path, "w+") as mf: writer = csv.writer(mf, delimiter=delim) for row in data: writer.writerow(row) return data
def start_from_paths(self, wid, source_path: str, input_path: str, deps_path: str): print("TMP: " + os.getenv("TMPDIR")) scale = ["--scale", str(self.scale)] if self.scale else [] loglevel = ["--logLevel=" + self.loglevel] if self.loglevel else [] cmd = ["toil-cwl-runner", "--stats", *loglevel, *scale, source_path, input_path] Logger.debug("Running command: '" + " ".join(cmd) + "'") process = subprocess.Popen( cmd, stdout=subprocess.PIPE, preexec_fn=os.setsid, stderr=subprocess.PIPE ) Logger.info("CWLTool has started with pid=" + str(process.pid)) for line in read_stdout(process): if "Path to job store directory is" in line: idx = line.index("Path to job store directory is") Logger.critical("JOBSTORE DIR: " + line[idx + 1 :]) Logger.debug("toil: " + line) print("finished")
def stop_engine(self): if not self.is_started: return Logger.debug( "Cromwell has already shut down, skipping shut down request") if self._logger: self._logger.terminate() self.should_stop = True if self._timer_thread: self._timer_thread.set() if not self.process_id: self.is_started = False Logger.info("Janis isn't managing Cromwell, skipping the shutdown") return Logger.info("Stopping cromwell") if self.process_id: try: process = os.getpgid(int(self.process_id)) os.killpg(process, signal.SIGTERM) Logger.info("Stopped cromwell") except Exception as e: # can't do Logger.warn("Couldn't stop Cromwell process: " + str(e)) else: Logger.warn( "Couldn't stop Cromwell process as Janis wasn't managing it") Logger.debug("Setting 'cromwell.is_started' to False") self.is_started = False
def cp_from( self, source, dest, force=False, report_progress: Optional[Callable[[float], None]] = None, ): if force: Logger.critical("SSHFileScheme does not support the 'force' flag") args = ["scp", self.connectionstring + ":" + source, dest] if dest.endswith("bam"): return Logger.warn( "Manually skipped BAM file, as they're usually too big") if os.path.exists(dest): return Logger.log(f"Skipping as exists ({source} -> {dest}") Logger.info( f"Secure copying (SCP) from {self.connectionstring}:{source} to local:{dest}" ) subprocess.call(args)
def do_run(args): if args.job: from os import getcwd workflow, workflow_ref = resolve_tool( tool=args.workflow, name=args.name, from_toolshed=True, only_toolbox=args.toolbox, force=args.no_cache, ) # parse and load the job file Logger.info("Specified job file, ignoring all other parameters") d = parse_dict(get_file_from_searchname(args.job, getcwd())) job = PreparedJob(**d, workflow_reference=workflow_ref) else: job, workflow = prepare_from_args(args, run_prepare_processing=False) jobfile = run_from_jobfile(workflow, jobfile=job, wait=args.wait) Logger.info("Exiting") raise SystemExit
def cp_from( self, source, dest, force=False, report_progress: Optional[Callable[[float], None]] = None, ): import urllib.request if os.path.exists(dest): if not force: return Logger.info( f"File already exists, skipping download ('{dest}')") os.remove(dest) return urllib.request.urlretrieve(url=source, filename=dest)
def link_copy_or_fail(source, dest, force=False): """ Eventually move this to some generic util class :param s: Source to link from :param d: Place to link to :return: """ try: to_copy = [(source, dest)] while len(to_copy) > 0: s, d = to_copy.pop(0) if os.path.exists(d) and force: Logger.info(f"Destination exists, overwriting '{d}'") if os.path.isdir(d): rmtree(d) else: os.remove(d) Logger.info(f"Hard linking {s} → {d}") if os.path.isdir(s): os.makedirs(d, exist_ok=True) for f in os.listdir(s): to_copy.append((os.path.join(s, f), os.path.join(d, f))) continue try: os.link(s, d) except FileExistsError: Logger.critical( "The file 'd' already exists. The force flag is required to overwrite." ) except Exception as e: Logger.warn("Couldn't link file: " + str(e)) # if this fails, it should error Logger.info(f"Copying file {s} → {d}") copyfile(s, d) except Exception as e: Logger.critical( f"An unexpected error occurred when link/copying {s}: {e}")
def do_wait(args): wids = args.wid statuses = {} for wid in wids: wm = ConfigManager.get_from_path_or_submission_lazy(wid, readonly=True) Logger.info(f"Waiting for '{wid}' to finish") status = wm.database.get_uncached_status() while not status.is_in_final_state(): sleep(2) status = wm.database.get_uncached_status() statuses[wid] = (wm.submission_id, status) Logger.info( f"Workflow {wid} finished with status: {status.to_string()}") collapsed_status = TaskStatus.collapse_states( [s[1] for s in statuses.values()]) rc = collapsed_status.get_exit_code() Logger.info( f"All workflows finished with collapsed status {collapsed_status.to_string()}, exiting with rc={rc}" ) sys.exit(rc)
def stop_engine(self): Logger.info( "Toil doesn't run in a server mode, an instance will " "be automatically terminated when a task is finished" )
def start_engine(self): Logger.info( "Toil doesn't run in a server mode, an instance will " "automatically be started when a task is created" ) return self
def mkdirs(self, directory): args = ["ssh", self.connectionstring, "-p", directory] Logger.info( f"Securely making directory {self.connectionstring}:{directory} through ssh" ) subprocess.call(args)
def rm_dir(self, directory): args = ["ssh", self.connectionstring, "rm -r " + directory] Logger.info( f"Secure deleting directory {self.connectionstring}:{directory} through ssh" ) subprocess.call(args)
def translate( self, workflow, to_console=True, tool_to_console=False, with_docker=True, with_resource_overrides=False, to_disk=False, write_inputs_file=True, export_path=ExportPathKeywords.default, should_validate=False, should_zip=True, merge_resources=False, hints=None, allow_null_if_not_optional=True, additional_inputs: Dict = None, max_cores=None, max_mem=None, ): # self.validate_inputs(workflow._inputs, allow_null_if_not_optional) tr_wf, tr_tools = self.translate_workflow( workflow, with_docker=with_docker, with_resource_overrides=with_resource_overrides, ) tr_inp = self.build_inputs_file( workflow, recursive=False, merge_resources=merge_resources, hints=hints, additional_inputs=additional_inputs, max_cores=max_cores, max_mem=max_mem, ) tr_res = self.build_resources_input(workflow, hints) str_wf = self.stringify_translated_workflow(tr_wf) str_inp = self.stringify_translated_inputs(tr_inp) str_tools = [( "tools/" + self.tool_filename(t), self.stringify_translated_workflow(tr_tools[t]), ) for t in tr_tools] str_resources = self.stringify_translated_inputs(tr_res) if to_console: print("=== WORKFLOW ===") print(str_wf) if tool_to_console: print("\n=== TOOLS ===") [print(f":: {t[0]} ::\n" + t[1]) for t in str_tools] print("\n=== INPUTS ===") print(str_inp) if not merge_resources and with_resource_overrides: print("\n=== RESOURCES ===") print(str_resources) d = ExportPathKeywords.resolve(export_path, workflow_spec=self.name, workflow_name=workflow.id()) fn_workflow = self.workflow_filename(workflow) fn_inputs = self.inputs_filename(workflow) fn_resources = self.resources_filename(workflow) if to_disk and write_inputs_file: if not os.path.isdir(d): os.makedirs(d) with open(os.path.join(d, fn_inputs), "w+") as f: Logger.log(f"Writing {fn_inputs} to disk") f.write(str_inp) Logger.log(f"Written {fn_inputs} to disk") else: Logger.log("Skipping writing input (yaml) job file") if to_disk: toolsdir = os.path.join(d, "tools") if not os.path.isdir(toolsdir): os.makedirs(toolsdir) Logger.info(f"Exporting workflow files to '{d}'") with open(os.path.join(d, fn_workflow), "w+") as wf: Logger.log(f"Writing {fn_workflow} to disk") wf.write(str_wf) Logger.log(f"Wrote {fn_workflow} to disk") for (fn_tool, str_tool) in str_tools: with open(os.path.join(d, fn_tool), "w+") as toolfp: Logger.log(f"Writing {fn_tool} to disk") toolfp.write(str_tool) Logger.log(f"Written {fn_tool} to disk") if not merge_resources and with_resource_overrides: print("\n=== RESOURCES ===") with open(os.path.join(d, fn_resources), "w+") as wf: Logger.log(f"Writing {fn_resources} to disk") wf.write(str_wf) Logger.log(f"Wrote {fn_resources} to disk") print(str_resources) import subprocess if should_zip: Logger.info("Zipping tools") with Path(d): FNULL = open(os.devnull, "w") zip_result = subprocess.run( ["zip", "-r", "tools.zip", "tools/"], stdout=FNULL) if zip_result.returncode == 0: Logger.info("Zipped tools") else: Logger.critical(zip_result.stderr) if should_validate: with Path(d): Logger.info(f"Validating outputted {self.name}") enved_vcs = [ (os.getenv(x[1:]) if x.startswith("$") else x) for x in self.validate_command_for( fn_workflow, fn_inputs, "tools/", "tools.zip") ] cwltool_result = subprocess.run(enved_vcs) if cwltool_result.returncode == 0: Logger.info("Exported workflow was validated by: " + " ".join(enved_vcs)) else: Logger.critical(cwltool_result.stderr) return str_wf, str_inp, str_tools
def resolve_jar(cromwelljar, janiscromwellconf, configdir): if cromwelljar: if not os.path.exists(cromwelljar): raise Exception( f"Specified cromwell jar '{cromwelljar}' but this didn't exist" ) return cromwelljar if (janiscromwellconf and janiscromwellconf.jar and os.path.exists(janiscromwellconf.jar)): return janiscromwellconf.jar fromenv = EnvVariables.cromwelljar.resolve(False) if fromenv and os.path.exists(fromenv): return fromenv potentials = list( reversed(sorted(glob(os.path.join(configdir, "cromwell-*.jar"))))) valid_paths = [p for p in potentials if os.path.exists(p)] if len(valid_paths) > 0: if len(valid_paths) == 0: raise Exception( "Couldn't find cromwelljar at any of the required paths: " + ", ".join(potentials)) cromwelljar = valid_paths[0] if not cromwelljar: progress_is_loaded = False try: import progressbar progress_is_loaded = True except: Logger.critical("Couldn't find progressbar module") pbar = None def show_progress(block_num, block_size, total_size): nonlocal pbar if pbar is None and progress_is_loaded: pbar = progressbar.ProgressBar(maxval=total_size) downloaded = block_num * block_size if downloaded < total_size: if pbar: pbar.update(downloaded) else: print( f"\rProgress: {round(downloaded * 100 / total_size)}%", end="", file=sys.stderr, ) else: if pbar: pbar.finish() pbar = None else: print("\rCompleted download of cromwell", file=sys.stderr) cromwellurl, cromwellfilename = Cromwell.get_latest_cromwell_url() Logger.info( f"Couldn't find cromwell at any of the usual spots, downloading '{cromwellfilename}' now" ) cromwelljar = os.path.join(configdir, cromwellfilename) request.urlretrieve(cromwellurl, cromwelljar, show_progress) Logger.info(f"Downloaded {cromwellfilename}") return cromwelljar
def build_resources_input(cls, workflow, hints, max_cores=None, max_mem=None, inputs=None, prefix=""): from janis_core.workflow.workflow import Workflow, Tool, CommandTool # returns a list of key, value pairs steps: Dict[str, Optional[Any]] = {} inputs = inputs or {} # prefix = "" # if not prefix: # prefix = "" # else: # prefix += "" for s in workflow.step_nodes.values(): tool: Tool = s.tool if isinstance(tool, CommandTool): tool_pre = prefix + s.id() + "_" cpus = inputs.get(f"{s.id()}_runtime_cpu", tool.cpus(hints) or 1) mem = inputs.get(f"{s.id()}_runtime_memory", tool.memory(hints)) disk = inputs.get(f"{s.id()}_runtime_disks", "local-disk 60 SSD") if max_cores and cpus > max_cores: Logger.info( f"Tool '{tool.tool()}' exceeded ({cpus}) max number of cores ({max_cores}), " "this was dropped to the new maximum") cpus = max_cores if max_mem and mem > max_mem: Logger.info( f"Tool '{tool.tool()}' exceeded ({mem} GB) max amount of memory({max_mem} GB), " "this was dropped to the new maximum") mem = max_mem steps.update({ tool_pre + "runtime_memory": mem, tool_pre + "runtime_cpu": cpus, tool_pre + "runtime_disks": disk, }) elif isinstance(tool, Workflow): tool_pre = prefix + s.id() + "_" steps.update( cls.build_resources_input( tool, hints, max_cores=max_cores, max_mem=max_mem, prefix=tool_pre, inputs={ k[len(s.id()) + 1:]: v for k, v in inputs.items() if k.startswith(s.id()) }, )) return steps
def start_engine(self, additional_cromwell_options: List[str] = None): from ...data.models.preparedjob import PreparedJob job = PreparedJob.instance() self._start_time = DateUtil.now() self.timeout = job.cromwell.timeout or 10 if self.test_connection(): Logger.info("Engine has already been started") return self if self.connect_to_instance: self.is_started = True Logger.info( "Cromwell environment discovered, skipping local instance") return self if self._process: self.is_started = True Logger.info( f"Discovered Cromwell instance (pid={self._process}), skipping start" ) return self if self.config: with open(self.config_path, "w+") as f: f.writelines(self.config.output()) Logger.log("Finding cromwell jar") cromwell_loc = self.resolve_jar(self.cromwelljar, job.cromwell, job.config_dir) Logger.info(f"Starting cromwell ({cromwell_loc})...") cmd = ["java", "-DLOG_MODE=standard"] if job.cromwell and job.cromwell.memory_mb: cmd.extend([ f"-Xmx{job.cromwell.memory_mb}M", f"-Xms{max(job.cromwell.memory_mb//2, 1)}M", ]) # if Logger.CONSOLE_LEVEL == LogLevel.VERBOSE: # cmd.append("-DLOG_LEVEL=DEBUG") if additional_cromwell_options: cmd.extend(additional_cromwell_options) self.port = find_free_port() self.host = f"127.0.0.1:{self.port}" cmd.append(f"-Dwebservice.port={self.port}") cmd.append(f"-Dwebservice.interface=127.0.0.1") if self.config_path and os.path.exists(self.config_path): Logger.debug("Using configuration file for Cromwell: " + self.config_path) cmd.append("-Dconfig.file=" + self.config_path) cmd.extend(["-jar", cromwell_loc, "server"]) Logger.debug(f"Starting Cromwell with command: '{' '.join(cmd)}'") self._process = subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, # preexec_fn creates a process group https://stackoverflow.com/a/4791612/ preexec_fn=os.setsid, ) Logger.info("Cromwell is starting with pid=" + str(self._process.pid)) Logger.debug( "Cromwell will start the HTTP server, reading logs to determine when this occurs" ) self._logfp = open(self.logfile, "a+") Logger.info(f"Will log Cromwell output to the file: {self.logfile}" if bool(self._logfp) else "Janis is NOT logging Cromwell output to a file") for c in iter(self._process.stdout.readline, "b"): # replace '' with b'' for Python 3 line = None if c: line = c.decode("utf-8").rstrip() if not line: rc = self._process.poll() if rc is not None: critical_suffix = f"Last received message '{line}'. " Logger.critical( f"Cromwell has exited with rc={rc}. {critical_suffix}The last lines of the logfile ({self.logfile}):" ) Logger.critical("".join(tail(self._logfp, 10))) return False continue if self._logfp and not self._logfp.closed: self._logfp.write(line + "\n") self._logfp.flush() os.fsync(self._logfp.fileno()) Logger.debug("Cromwell: " + line) # self.stdout.append(str(c)) if "service started on" in line: self.process_id = self._process.pid Logger.info("Service successfully started with pid=" + str(self._process.pid)) break # elif ansi_escape.match(): # raise Exception(cd) self.is_started = True if self._process: self._logger = ProcessLogger( process=self._process, prefix="Cromwell: ", logfp=self._logfp, # exit_function=self.something_has_happened_to_cromwell, ) return self
def run(self): finalstatus = None iserroring = False try: for c in iter(self.process.stderr.readline, "b"): if self.should_terminate: return line = None if c: line = c.decode("utf-8").rstrip() if not line: if self.process.poll() is not None: finalstatus = TaskStatus.ABORTED Logger.warn( f"CWLTool finished with rc={self.process.returncode} but janis " f"was unable to capture the workflow status. Marking as aborted" ) break continue if self.logfp and not self.logfp.closed: self.logfp.write(line + "\n") self.logfp.flush() os.fsync(self.logfp.fileno()) lowline = line.lower().lstrip() if lowline.startswith("error"): Logger.critical("cwltool: " + line) iserroring = True elif lowline.startswith("warn"): iserroring = False Logger.warn("cwltool: " + line) elif lowline.startswith("info"): iserroring = False Logger.info("cwltool: " + line) self.process_metadataupdate_if_match(line) else: Logger.debug("cwltool: " + line) if iserroring: self.error = (self.error or "") + "\n" + line if "final process status is" in lowline: if "fail" in line.lower(): finalstatus = TaskStatus.FAILED elif "success" in line.lower(): finalstatus = TaskStatus.COMPLETED else: finalstatus = TaskStatus.ABORTED break j = "" Logger.info("Process has completed") if finalstatus == TaskStatus.COMPLETED: for c in iter(self.process.stdout.readline, "s"): if not c: continue line = c.decode("utf-8").rstrip() Logger.debug(line) if self.logfp and not self.logfp.closed: self.logfp.write(line + "\n") self.logfp.flush() os.fsync(self.logfp.fileno()) j += line try: self.outputs = json.loads(j) break except: continue if self.error: Logger.critical("Janis detected a CWLTool error: " + self.error) Logger.info("CWLTool detected transition to terminal status: " + str(finalstatus)) self.terminate() if self.exit_function: self.exit_function(self, finalstatus) except KeyboardInterrupt: self.should_terminate = True print("Detected keyboard interrupt") # raise except Exception as e: print("Detected another error") raise e