def __init__(self, initial_conf = None, required = [], args_usage = "", add_options = None, expand_vars = False): DataElement.__init__(self) from optparse import OptionParser parser = OptionParser(usage = "usage: %prog [options] " + args_usage, version = VERSION) parser.add_option("-L", "--log-level", dest="log_level", default=None, choices=["debug", "info", "warn", "error", "critical", "notset"], help="Which log level: debug, info, warn, error, critical, notset") parser.add_option("-c", "--conf", action="append", dest="conf_files", default=[], metavar="FILE", help="Load configuration from a file. Multiple files can be specified") parser.add_option("-D", action="append", dest="data", default=[], metavar="PARAM=VALUE", help="External data value. example -D param1=value") if add_options is not None: add_options(parser) (self.options, self.args) = parser.parse_args() self.builder = ConfigBuilder() if initial_conf is not None: if isinstance(initial_conf, dict): initial_conf = DataFactory.from_native(initial_conf) self.builder.add_element(initial_conf) if self.options.log_level is not None: self.builder.add_value("wok.log.level", self.options.log_level) if len(self.options.conf_files) > 0: files = [] for conf_file in self.options.conf_files: self.builder.add_file(conf_file) files.append(os.path.abspath(conf_file)) self.builder.add_value("__files", DataFactory.from_native(files)) for data in self.options.data: d = data.split("=") if len(d) != 2: raise Exception("Data argument wrong: " + data) self.builder.add_value(d[0], d[1]) self.builder.merge_into(self) if len(required) > 0: self.check_required(required) if expand_vars: self.expand_vars()
def _load_task(self, t_id): path = os.path.join(self._work_path, "tasks", t_id + ".json") try: f = open(path, "r") task = DataFactory.from_native(json.load(f), key_sep = "/") f.close() except: self._log.error("Error reading task file: %s" % path) raise return task
def merge_into(self, conf): try: f = open(self.path, "r") v = json.load(f) cf = DataFactory.from_native(v) conf.merge(cf) f.close() except Exception as e: from wok import logger msg = ["Error loading configuration from ", self.path, ":\n\n", str(e), "\n"] logger.get_logger("config").error("".join(msg)) raise
def load_task_config(self, instance_name, module_id, task_index): "Load a task configuration" mod_path = self.module_path(instance_name, module_id) task_config_path = os.path.join(mod_path, "{}.task".format(self.task_prefix(task_index))) try: f = open(task_config_path, "r") o = json.load(f) e = DataFactory.from_native(o, key_sep = "/") f.close() except: self._log.error("Failed reading task configuration: " + task_config_path) raise return e
def merge_into(self, conf): try: v = json.loads(self.value) except: v = self.value conf[self.key] = DataFactory.from_native(v)
def merge_into(self, conf): f = open(self.path, "r") v = json.load(f) cf = DataFactory.from_native(v) conf.merge(cf) f.close()
def main(): # Initialization task.check_conf(["entities", "repositories", "repositories.data", "bin_paths.gitools"]) conf = task.conf log = task.logger() combinations_port, combination_ids_port = \ task.ports("combinations", "combination_ids") es = EntityServer(conf["entities"]) em = es.manager() rs = RepositoryServer(conf["repositories"]) data_repo = rs.repository("data") overwrite = conf.get("overwrite", False, dtype=bool) results_base_path = types.CNV_COMBINATION.replace(".", "/") conditions = ("gain", "loss") for c_dict in combinations_port: c = DataFactory.from_native(c_dict, key_sep = "/") """ o = em.find(c, types.CNV_ONCODRIVE_GENES) if o is None: log.error("{0} not found: {1}".format(types.CNV_ONCODRIVE_GENES, c)) continue okey = (o["study_id"], o["platform_id"], o["icdo_topography"], o["icdo_morphology"]) """ cid = c["id"] key = (c["icdo_topography"], c["icdo_morphology"], c["id_type"]) log.info("Processing combination for ({}) [{}] ...".format(", ".join(key), cid)) #files = c["files"] #if len(files) == 1: # log.info("No combination required, copyed from {0}".format(files[0])) # c["results_file"] = files[0] #else: results_path = rpath.join(results_base_path, cid + ".tsv.gz") results_url = data_repo.url(results_path) if skip_file(overwrite, data_repo, results_path, c.get("results_file")): log.warn("Skipping {} ({}) [{}] as it already exists".format(types.CNV_COMBINATION, ", ".join(key), cid)) combination_ids_port.write(cid) continue c["results_file"] = results_url combination(log, conf, rs, c, data_repo, results_path, conditions) # save combination results em.persist(c, types.CNV_COMBINATION) combination_ids_port.write(cid) em.close() es.close() data_repo.close() rs.close()