def __init__(self, name, title = "", desc = "", enabled = True, serializer = None, wsize = None, maxpar = None, conf = None, in_ports = None, out_ports = None, resources = None, params = None): _BasePort.__init__(self, name, title, desc, enabled, serializer, wsize) self.maxpar = maxpar self.conf = conf if in_ports is None: self.in_ports = [] self.in_port_map = {} else: self.in_ports = in_ports for p in in_ports: self.in_port_map[p.name] = p if out_ports is None: self.out_ports = [] self.out_port_map = {} else: self.out_ports = out_ports for p in out_ports: self.out_port_map[p.name] = p if resources is None: resources = Data.element() self.resources = resources if params is None: params = [] self.params = params
def create_case(self, user, case_name, conf_builder, project_name, flow_name, properties=None, start=True): case = Case( owner_id=user.id, name=case_name, project_name=project_name, flow_name=flow_name, conf=conf_builder.get_conf(), properties=Data.element(properties)) session = db.Session() session.add(case) session.commit() engine_case_name = "{}-{}".format(user.nick, case_name) #while self.engine.exists_case(engine_case_name): # engine_case_name = "{}-{}".format(user.nick, uuid4().hex[-6:]) engine_case = self.engine.create_case(engine_case_name, conf_builder, project_name, flow_name, engine_case_name) case.created = engine_case.created case.engine_name = engine_case_name session.commit() if start: engine_case.start() return case
def to_element(self, e=None): if e is None: e = Data.element() e["ns"] = self.namespace e["name"] = self.name e["cname"] = self.cname return e
def to_element(self, e = None): if e is None: e = Data.element() e["ns"] = self.namespace e["name"] = self.name e["cname"] = self.cname return e
def resources(self): if self.parent is None: res = Data.element() else: res = self.parent.resources if self.model.resources is not None: res.merge(self.model.resources) return res
def initialize(conf=None, format=None, datefmt=None, level=None): """ Initialize the logging system. If conf is a dictionary then the parameters considered for configuration are: - format: Logger format - datefmt: Date format - loggers: list of tuples (name, conf) to configure loggers If conf is a list then only the loggers are configured. If conf is an string then the default logger is configured for the logging level. """ global _initialized if conf is None: conf = Data.element() elif not isinstance(conf, basestring): conf = Data.create(conf) if Data.is_list(conf): loggers_conf = conf conf = Data.element() elif Data.is_element(conf): loggers_conf = conf.get("loggers", default=Data.list) elif isinstance(conf, basestring): loggers_conf = Data.list([["", conf]]) conf = Data.element() format = format or conf.get("format", _DEFAULT_FORMAT) datefmt = datefmt or conf.get("datefmt", _DEFAULT_DATEFMT) logging.basicConfig(format=format, datefmt=datefmt) for (log_name, log_conf) in loggers_conf: init_logger(log_name, conf=log_conf) if level is not None: init_logger("", conf=level) _initialized = True
def to_element(self, e=None): if e is None: e = Data.element() e["name"] = self.name e["state"] = str(self.state) e["conf"] = self.conf #FIXME self.root_node.update_tasks_count_by_state() self.root_node.update_component_count_by_state() self.root_node.to_element(e.element("root")) return e
def __init__(self, parent, index, id=None, namespace="", state=runstates.READY, substate=None, partition=None): Node.__init__(self, parent, namespace=namespace) self.id = id self.index = index self.state = state self.substate = substate self.partition = partition or Data.element() self.job_id = None self.job_result = None
def to_element(self, e = None): if e is None: e = Data.element() e["name"] = self.name e["state"] = str(self.state) e["conf"] = self.conf #FIXME self.root_node.update_tasks_count_by_state() self.root_node.update_component_count_by_state() self.root_node.to_element(e.element("root")) return e
def _initialize_conf(self): # project defined config self.conf = conf = self.project.get_conf() # user defined config self.user_conf = self.conf_builder.get_conf() if "wok" in self.user_conf: self.user_conf.delete("wok.work_path", "wok.projects", "wok.platforms", "wok.logging") conf.merge(self.user_conf) # runtime config conf[rtconf.CASE_NAME] = self.name conf[rtconf.FLOW] = Data.element(dict( name=self.root_flow.name, uri=self.flow_uri))
def _initialize_conf(self): # project defined config self.conf = conf = self.project.get_conf() # user defined config self.user_conf = self.conf_builder.get_conf() if "wok" in self.user_conf: self.user_conf.delete("wok.work_path", "wok.projects", "wok.platforms", "wok.logging") conf.merge(self.user_conf) # runtime config conf[rtconf.CASE_NAME] = self.name conf[rtconf.FLOW] = Data.element( dict(name=self.root_flow.name, uri=self.flow_uri))
def _create_platforms(self): """ Creates the platform according to the configuration :return: Platform """ platform_confs = self._conf.get("platforms") if platform_confs is None: platform_confs = Data.list() elif not Data.is_list(platform_confs): self._log.error("Wrong configuration type for 'platforms': {}".format(platform_confs)) platform_confs = Data.list() if len(platform_confs) == 0: platform_confs += [Data.element(dict(type="local"))] platforms = [] names = {} for pidx, platform_conf in enumerate(platform_confs): if isinstance(platform_conf, basestring): if not os.path.isabs(platform_conf) and self._conf_base_path is not None: platform_conf = os.path.join(self._conf_base_path, platform_conf) platform_conf = ConfigLoader(platform_conf).load() if not Data.is_element(platform_conf): raise errors.ConfigTypeError("wok.platforms[{}]".format(pidx, platform_conf)) ptype = platform_conf.get("type", "local") name = platform_conf.get("name", ptype) if name in names: name = "{}-{}".format(name, names[name]) names[name] += 1 else: names[name] = 2 platform_conf["name"] = name if "work_path" not in platform_conf: platform_conf["work_path"] = os.path.join(self._work_path, "platform_{}".format(name)) self._log.info("Creating '{}' platform ...".format(name)) self._log.debug("Platform configuration: {}".format(repr(platform_conf))) platforms += [create_platform(ptype, platform_conf)] return platforms
def _load_project_desc(self, path, base_path=None): if not os.path.isabs(path): if base_path is not None: path = os.path.join(base_path, path) else: path = os.path.abspath(path) if not os.path.exists(path): raise Exception("Project path not found: {}".format(path)) if os.path.isdir(path): path = os.path.join(path, "project.conf") if not os.path.isfile(path): raise Exception("Project configuration not found: {}".format(path)) project = Data.element() project.merge(ConfigLoader(path).load()) base_path = os.path.dirname(path) if "path" not in project: project["path"] = base_path if not os.path.isabs(project["path"]): project["path"] = os.path.normpath( os.path.join(base_path, project["path"])) if "conf" in project and isinstance(project["conf"], basestring): conf_path = os.path.join(base_path, project["conf"]) project["conf"] = ConfigLoader(conf_path).load() if "conf_rules" in project and isinstance(project["conf_rules"], basestring): base_path = os.path.dirname(path) conf_path = os.path.join(base_path, project["conf_rules"]) project["conf_rules"] = ConfigLoader(conf_path).load() if "conf_rules" in project and Data.is_list(project["conf_rules"]): for rule in project["conf_rules"]: if Data.is_element(rule) and "merge" in rule and isinstance( rule["merge"], basestring): rule["merge"] = ConfigLoader( os.path.join(base_path, rule["merge"])).load() return project
def _load_project_desc(self, path, base_path=None): if not os.path.isabs(path): if base_path is not None: path = os.path.join(base_path, path) else: path = os.path.abspath(path) if not os.path.exists(path): raise Exception("Project path not found: {}".format(path)) if os.path.isdir(path): path = os.path.join(path, "project.conf") if not os.path.isfile(path): raise Exception("Project configuration not found: {}".format(path)) project = Data.element() project.merge(ConfigLoader(path).load()) base_path = os.path.dirname(path) if "path" not in project: project["path"] = base_path if not os.path.isabs(project["path"]): project["path"] = os.path.normpath(os.path.join(base_path, project["path"])) if "conf" in project and isinstance(project["conf"], basestring): conf_path = os.path.join(base_path, project["conf"]) project["conf"] = ConfigLoader(conf_path).load() if "conf_rules" in project and isinstance(project["conf_rules"], basestring): base_path = os.path.dirname(path) conf_path = os.path.join(base_path, project["conf_rules"]) project["conf_rules"] = ConfigLoader(conf_path).load() if "conf_rules" in project and Data.is_list(project["conf_rules"]): for rule in project["conf_rules"]: if Data.is_element(rule) and "merge" in rule and isinstance(rule["merge"], basestring): rule["merge"] = ConfigLoader(os.path.join(base_path, rule["merge"])).load() return project
def _start(self): self._log.debug("Creating session ...") self._session = saga.Session() ctxs_conf = self._conf.get("context") if ctxs_conf is not None: if Data.is_element(ctxs_conf): ctxs_conf = Data.list([ctxs_conf]) for ctx_conf in ctxs_conf: try: ctx = saga.Context(ctx_conf["type"]) for key in ctx_conf: if hasattr(ctx, key): setattr(ctx, key, ctx_conf[key]) self._session.add_context(ctx) except Exception as ex: self._log.error("Wrong context configuration: {}".format(repr(ctx_conf))) self._log.exception(ex) self._log.debug("Creating job service ...") url = self._conf.get("service_url", "fork://localhost", dtype=str) self._job_service = saga.job.Service(url, session=self._session) self._remote_dir = saga.filesystem.Directory(self._file_url, session=self._session) # FIXME Use the logging configuration mechanisms of SAGA from wok import logger logger.init_logger("SGEJobService", conf=Data.element(dict(level=self._conf.get("saga_log.level", "error")))) # TODO count the number of previously queued jobs # TODO clean output files ? self._running = True self._run_thread = threading.Thread(target=self._run_handler, name="{}-run".format(self._name)) self._join_thread = threading.Thread(target=self._join_handler, name="{}-join".format(self._name)) self._run_thread.start() self._join_thread.start()
def _override_component(ovr, src): ovr.name = src.name if src.title is not None: ovr.title = src.title if src.desc is not None: ovr.desc = src.desc if src.enabled is not None: ovr.enabled = src.enabled if src.serializer is not None: ovr.serializer = mode_def.serializer if src.wsize is not None: ovr.wsize = mode_def.wsize if src.conf is not None: if ovr.conf is None: ovr.conf = Data.element() ovr.conf.merge(mode_def.conf) ovr.priority = src.priority ovr.depends = src.depends ovr.flow_ref = src.flow_ref
def __init__(self, name, title="", desc="", enabled=True, serializer=None, wsize=None, maxpar=None, conf=None, in_ports=None, out_ports=None, resources=None, params=None): _BasePort.__init__(self, name, title, desc, enabled, serializer, wsize) self.maxpar = maxpar self.conf = conf if in_ports is None: self.in_ports = [] self.in_port_map = {} else: self.in_ports = in_ports for p in in_ports: self.in_port_map[p.name] = p if out_ports is None: self.out_ports = [] self.out_port_map = {} else: self.out_ports = out_ports for p in out_ports: self.out_port_map[p.name] = p if resources is None: resources = Data.element() self.resources = resources if params is None: params = [] self.params = params
def init_logger(logger, conf): """ Initializa a logger from configuration. Configuration can be: - An string referring to the log level - A dictionary with the following parameters: - level: log level - handlers: List of log handlers or just a handler. Each handler have the following parameters: - type - ...: each handler type has a set of parameters Supported handlers: - smtp: Send logs by email. Parameters: - host - port (optional) - user - pass - from - to - subject - level - format: can be a simple string or a list of strings that will be joint with '\n' """ if isinstance(logger, basestring): logger = get_logger(logger) if isinstance(conf, basestring): conf = Data.element(dict(level=conf)) else: conf = Data.create(conf) level = conf.get("level") if level is not None: logger.setLevel(get_level(level)) handlers_conf = conf.get("handlers", default=Data.list) if Data.is_element(handlers_conf): handlers_conf = Data.list([handlers_conf]) for handler_conf in handlers_conf: handler = get_handler(logger, handler_conf) logger.addHandler(handler)
def prepare(self, case, task, index): execution = task.execution exec_conf = execution.conf if exec_conf is None: exec_conf = Data.element() if "script_path" not in exec_conf: raise MissingValueError("script_path") script_path = exec_conf["script_path"] lang = exec_conf.get("language", "python") case_conf = case.conf.clone().expand_vars() # Environment variables env = Data.element() #for k, v in os.environ.items(): # env[k] = v env.merge(task.conf.get(rtconf.TASK_ENV)) env.merge(exec_conf.get("env")) # Default module script path platform_project_path = task.conf.get(rtconf.PROJECT_PATH, case.project.path) flow_path = os.path.abspath(os.path.dirname(task.flow_path)) flow_rel_path = os.path.relpath(flow_path, case.project.path) platform_script_path = os.path.join(platform_project_path, flow_rel_path, script_path) env[ENV_PROJECT_PATH] = platform_project_path env[ENV_FLOW_PATH] = flow_rel_path env[ENV_SCRIPT_PATH] = script_path env[ENV_PLATFORM_SCRIPT_PATH] = platform_script_path script = [] sources = task.conf.get(rtconf.TASK_SOURCES, default=Data.list) if isinstance(sources, basestring): sources = Data.list([sources]) for source in sources: script += ['source "{}"'.format(source)] if lang == "python": virtualenv = task.conf.get(rtconf.TASK_PYTHON_VIRTUALENV) if virtualenv is not None: #script += ["set -x"] #script += ["echo Activating virtualenv {} ...".format(virtualenv)] script += ['source "{}"'.format(os.path.join(virtualenv, "bin", "activate"))] #script += ["set +x"] #script += ["echo Running workitem ..."] cmd = [task.conf.get(rtconf.TASK_PYTHON_BIN, "python")] cmd += ["${}".format(ENV_PLATFORM_SCRIPT_PATH)] lib_path = task.conf.get(rtconf.TASK_PYTHON_LIBS) if lib_path is not None: if Data.is_list(lib_path): lib_path = ":".join(lib_path) if "PYTHONPATH" in env: env["PYTHONPATH"] = lib_path + ":" + env["PYTHONPATH"] else: env["PYTHONPATH"] = lib_path else: raise LanguageError(lang) cmd += ["-D", "case={}".format(case.name), "-D", "task={}".format(task.cname), "-D", "index={}".format(index)] #for key, value in self._storage_conf(workitem.case.engine.storage.basic_conf): # cmd += ["-D", "storage.{}={}".format(key, value)] for key, value in self._plain_conf(Data.create(task.platform.data.context_conf(CTX_EXEC))): cmd += ["-D", "data.{}={}".format(key, value)] for key, value in self._plain_conf(task.platform.storage.context_conf(CTX_EXEC)): cmd += ["-D", "storage.{}={}".format(key, value)] script += [" ".join(cmd)] return "\n".join(script), env.to_native()
def init_project(logger, config, paths, storage, project): project_id = project["id"] results_path = paths.results_path() project_path = paths.project_path(project_id) project_temp_path = paths.project_temp_path(project_path) if config.results.purge_on_start: logger.info(" Purging previous results ...") if os.path.isdir(project_path): logger.info(" {} ...".format(os.path.relpath(project_path, results_path))) shutil.rmtree(project_path) #if os.path.isdir(project_temp_path): # logger.info(" {} ...".format(os.path.relpath(project_temp_path, results_path))) # shutil.rmtree(project_temp_path) for obj_name in storage.list_objects(prefix="results/"): logger.info(" {} ...".format(obj_name)) storage.delete_object("results/{}".format(obj_name)) ensure_path_exists(project_path) ensure_path_exists(project_temp_path) projdb_path = os.path.join(project_path, "project.db") if "annotations" in project: annotations = project["annotations"] if not Data.is_element(annotations): logger.warn("Overriding project annotations field with an empty dictionary") project["annotations"] = annotations = Data.element() else: project["annotations"] = annotations = Data.element() # for backward compatibility for key in project.keys(): if key not in ["id", "assembly", "files", "storage_objects", "annotations", "conf", "oncodriveclust", "oncodrivefm"]: value = project[key] del project[key] annotations[key] = value project["conf"] = pconf = project.get("conf") or Data.element() if not Data.is_element(pconf): logger.warn("Overriding project conf field with an empty dictionary") project["conf"] = pconf = Data.element() # for backward compatibility for key in project.keys(): if key in ["oncodriveclust", "oncodrivefm"]: value = project[key] del project[key] pconf[key] = value project["path"] = project_path project["temp_path"] = project_temp_path project["db"] = projdb_path if "assembly" not in project: project["assembly"] = DEFAULT_ASSEMBLY missing_objects = [] for obj_name in project["storage_objects"]: if not storage.exists_object(obj_name): missing_objects += [obj_name] if len(missing_objects) > 0: raise InternalError("Project {0} references some missing objects:\n{1}".format(project_id, "\n".join(missing_objects))) project["files"] = [str(f) for f in project["files"]] #unicode is not json serializable project["storage_objects"] = [str(f) for f in project["storage_objects"]] #unicode is not json serializable project = project.to_native() # save project.conf projres = ProjectResults(project) projres.save_def() return project
def _join(self): "Joiner thread" set_thread_title() self._num_alive_threads += 1 _log = logger.get_logger("wok.engine.join") _log.debug("Engine join thread ready") session = None num_exc = 0 while self._running: try: set_thread_title("waiting") job_info = self.__queue_adaptative_get(self._join_queue) if job_info is None: continue workitem_id, job_id = job_info with self._lock: session = db.Session() workitem = session.query(db.WorkItem).filter(db.WorkItem.id == workitem_id).one() case = self._cases_by_name[workitem.case.name] task = case.component(workitem.task.cname) set_thread_title(task.cname) #_log.debug("Joining work-item %s ..." % task.cname) jr = task.platform.jobs.join(job_id) wr = Data.element(dict( hostname=jr.hostname, created=jr.created.strftime(_DT_FORMAT) if jr.created is not None else None, started=jr.started.strftime(_DT_FORMAT) if jr.started is not None else None, finished=jr.finished.strftime(_DT_FORMAT) if jr.finished is not None else None, exitcode=jr.exitcode.code if jr.exitcode is not None else None)) r = task.platform.data.load_workitem_result(case.name, task.cname, workitem.index) if r is not None: if r.exception is not None: wr["exception"] = r.exception if r.trace is not None: wr["trace"] = r.trace workitem.substate = None workitem.result = wr case.num_active_workitems -= 1 session.commit() # check if there are still more work-items num_workitems = session.query(func.count(db.WorkItem.id)).filter( ~db.WorkItem.state.in_(runstates.TERMINAL_STATES)).scalar() if self._single_run and num_workitems == 0: stop_engine = True for case in self._cases: stop_engine = stop_engine and (case.state in runstates.TERMINAL_STATES) #self._running = not stop_engine if stop_engine: self._finished_event.set() _log.debug("[{}] Joined work-item {}".format(case.name, workitem.cname)) # check stopping instances if case in self._stopping_cases: job_ids = self._stopping_cases[case] if job_id in job_ids: job_ids.remove(job_id) if len(job_ids) == 0: del self._stopping_cases[case] if case.state == runstates.ABORTING: workitem.case.state = case.state = runstates.ABORTED session.commit() if case.removed: self.__remove_case(session, case) session.commit() else: _log.debug("Still waiting for {} jobs to stop".format(len(job_ids))) if case.state in runstates.TERMINAL_STATES and case.num_active_workitems == 0: _log.info("[{}] Case {}. Total time: {}".format(case.name, case.state.title, str(case.elapsed))) self._lock.release() try: self.case_finished.send(case) finally: self._lock.acquire() except BaseException as ex: num_exc += 1 _log.warn("Exception in join thread ({}): {}".format(num_exc, str(ex))) from traceback import format_exc _log.debug(format_exc()) try: if session is not None: session.rollback() except Exception as ex: _log.warn("Session rollback failed") _log.exception(ex) finally: try: if session is not None: session.close() except Exception as ex: _log.warn("Session close failed") _log.exception(ex) self._num_alive_threads -= 1 _log.debug("Engine join thread finished")
return self.engine.exists_case(engine_case_name) or exists_in_db() def create_case(self, user, case_name, conf_builder, project_name, flow_name, properties=None, start=True): case = Case(owner_id=user.id, name=case_name, project_name=project_name, flow_name=flow_name, conf=conf_builder.get_conf(), properties=Data.element(properties)) session = db.Session() session.add(case) session.commit() engine_case_name = "{}-{}".format(user.nick, case_name) #while self.engine.exists_case(engine_case_name): # engine_case_name = "{}-{}".format(user.nick, uuid4().hex[-6:]) engine_case = self.engine.create_case(engine_case_name, conf_builder, project_name, flow_name) case.created = engine_case.created case.engine_name = engine_case_name session.commit()
def project_conf(self, project_name, platform_name=None): if project_name not in self._projects: return Data.element() project = self._projects[project_name] return project.get_conf(platform_name=platform_name)
class NativeCommmandBuilder(CommmandBuilder): def _plain_conf(self, value, path=None): if path is None: path = [] if not Data.is_element(value): yield (".".join(path), value) else: for key in value.keys(): for k, v in self._plain_conf(value[key], path + [key]): yield (k, v) def prepare(self, case, task, index): execution = task.execution exec_conf = execution.conf if exec_conf is None: exec_conf = Data.element() if "script_path" not in exec_conf: raise MissingValueError("script_path") script_path = exec_conf["script_path"] lang = exec_conf.get("language", "python") case_conf = case.conf.clone().expand_vars() # Environment variables env = Data.element() #for k, v in os.environ.items(): # env[k] = v env.merge(task.conf.get(rtconf.TASK_ENV)) env.merge(exec_conf.get("env")) # Default module script path platform_project_path = task.conf.get(rtconf.PROJECT_PATH, case.project.path) flow_path = os.path.abspath(os.path.dirname(task.flow_path)) flow_rel_path = os.path.relpath(flow_path, case.project.path) platform_script_path = os.path.join(platform_project_path, flow_rel_path, script_path) env[ENV_PROJECT_PATH] = platform_project_path env[ENV_FLOW_PATH] = flow_rel_path env[ENV_SCRIPT_PATH] = script_path env[ENV_PLATFORM_SCRIPT_PATH] = platform_script_path script = [] sources = task.conf.get(rtconf.TASK_SOURCES, default=Data.list) if isinstance(sources, basestring): sources = Data.list([sources]) for source in sources: script += ['source "{}"'.format(source)] if lang == "python": virtualenv = task.conf.get(rtconf.TASK_PYTHON_VIRTUALENV) if virtualenv is not None: #script += ["set -x"] #script += ["echo Activating virtualenv {} ...".format(virtualenv)] script += [ 'source "{}"'.format( os.path.join(virtualenv, "bin", "activate")) ] #script += ["set +x"] #script += ["echo Running workitem ..."] cmd = [task.conf.get(rtconf.TASK_PYTHON_BIN, "python")] cmd += ["${}".format(ENV_PLATFORM_SCRIPT_PATH)] lib_path = task.conf.get(rtconf.TASK_PYTHON_LIBS) if lib_path is not None: if Data.is_list(lib_path): lib_path = ":".join(lib_path) if "PYTHONPATH" in env: env["PYTHONPATH"] = lib_path + ":" + env["PYTHONPATH"] else: env["PYTHONPATH"] = lib_path else: raise LanguageError(lang) cmd += [ "-D", "case={}".format(case.name), "-D", "task={}".format(task.cname), "-D", "index={}".format(index) ] #for key, value in self._storage_conf(workitem.case.engine.storage.basic_conf): # cmd += ["-D", "storage.{}={}".format(key, value)] for key, value in self._plain_conf( Data.create(task.platform.data.context_conf(CTX_EXEC))): cmd += ["-D", "data.{}={}".format(key, value)] for key, value in self._plain_conf( task.platform.storage.context_conf(CTX_EXEC)): cmd += ["-D", "storage.{}={}".format(key, value)] script += [" ".join(cmd)] return "\n".join(script), env.to_native()
def compute(project): log = task.logger config = GlobalConfig(task.conf) paths = PathsConfig(config) projects_out_port = task.ports("projects_out") project_id = project["id"] log.info("--- [{0}] --------------------------------------------".format(project_id)) ofm = Data.element(project["oncodrivefm"]) feature = ofm["feature"] slice_name = ofm["slice"] estimator = ofm.get("estimator") num_cores = ofm.get("num_cores", dtype=str) num_samplings = ofm.get("num_samplings", dtype=str) samples_threshold = ofm.get("samples_threshold", dtype=str) filter_enabled = ofm.get("filter_enabled", dtype=bool) filter_path = ofm.get("filter_path", dtype=str) log.info("feature = {0}".format(feature)) log.info("slice = {0}".format(slice_name)) log.info("estimator = {0}".format(estimator)) log.info("num_cores = {0}".format(num_cores)) log.info("num_samplings = {0}".format(num_samplings)) log.info("samples_threshold = {0}".format(samples_threshold)) log.info("filter_enabled = {0}".format(filter_enabled)) log.info("filter_path = {0}".format(os.path.basename(filter_path))) cmd = [ "oncodrivefm-compute", "-o", project["temp_path"], "-n oncodrivefm-{0}".format(feature), "-N", num_samplings, "--threshold", samples_threshold, "-e {0}".format(estimator), "-j", num_cores, "--slices '{0}'".format(slice_name)] if filter_enabled: cmd += ["--filter", filter_path] if feature == "pathways": cmd += ["-m", paths.data_kegg_path("ensg_kegg.tsv")] cmd += [ofm["data"]] project["oncodrivefm"] = dict( feature=feature, slice=slice_name, results=os.path.join(project["temp_path"], "oncodrivefm-{0}-{1}.tsv".format(feature, slice_name))) cmd = " ".join(cmd) log.debug(cmd) ret_code = subprocess.call(cmd, shell=True) if ret_code != 0: raise Exception("OncodriveFM error while computing {0}:\n{1}".format(feature, cmd)) projects_out_port.send(project)