def __init__(self, initial_conf = None, required = [], args_usage = "", add_options = None, expand_vars = False):
DataElement.__init__(self)
from optparse import OptionParser
parser = OptionParser(usage = "usage: %prog [options] " + args_usage, version = VERSION)
parser.add_option("-L", "--log-level", dest="log_level",
default=None, choices=["debug", "info", "warn", "error", "critical", "notset"],
help="Which log level: debug, info, warn, error, critical, notset")
parser.add_option("-c", "--conf", action="append", dest="conf_files", default=[], metavar="FILE",
help="Load configuration from a file. Multiple files can be specified")
parser.add_option("-D", action="append", dest="data", default=[], metavar="PARAM=VALUE",
help="External data value. example -D param1=value")
if add_options is not None:
add_options(parser)
(self.options, self.args) = parser.parse_args()
self.builder = ConfigBuilder()
if initial_conf is not None:
if isinstance(initial_conf, dict):
initial_conf = DataFactory.from_native(initial_conf)
self.builder.add_element(initial_conf)
if self.options.log_level is not None:
self.builder.add_value("wok.log.level", self.options.log_level)
if len(self.options.conf_files) > 0:
files = []
for conf_file in self.options.conf_files:
self.builder.add_file(conf_file)
files.append(os.path.abspath(conf_file))
self.builder.add_value("__files", DataFactory.from_native(files))
for data in self.options.data:
d = data.split("=")
if len(d) != 2:
raise Exception("Data argument wrong: " + data)
self.builder.add_value(d[0], d[1])
self.builder.merge_into(self)
if len(required) > 0:
self.check_required(required)
if expand_vars:
self.expand_vars()
def _load_task(self, t_id):
path = os.path.join(self._work_path, "tasks", t_id + ".json")
try:
f = open(path, "r")
task = DataFactory.from_native(json.load(f), key_sep = "/")
f.close()
except:
self._log.error("Error reading task file: %s" % path)
raise
return task
def _parse_exec(self, xmle): execution = Exec() if "launcher" in xmle.attrib: execution.mode = xmle.attrib["launcher"].lower() if execution.mode == "python": execution.mode = "native" execution.conf = DataFactory.from_xmle(xmle) return execution
def merge_into(self, conf):
try:
f = open(self.path, "r")
v = json.load(f)
cf = DataFactory.from_native(v)
conf.merge(cf)
f.close()
except Exception as e:
from wok import logger
msg = ["Error loading configuration from ", self.path, ":\n\n", str(e), "\n"]
logger.get_logger("config").error("".join(msg))
raise
def load_task_config(self, instance_name, module_id, task_index):
"Load a task configuration"
mod_path = self.module_path(instance_name, module_id)
task_config_path = os.path.join(mod_path,
"{}.task".format(self.task_prefix(task_index)))
try:
f = open(task_config_path, "r")
o = json.load(f)
e = DataFactory.from_native(o, key_sep = "/")
f.close()
except:
self._log.error("Failed reading task configuration: " + task_config_path)
raise
return e
def merge_into(self, conf): try: v = json.loads(self.value) except: v = self.value conf[self.key] = DataFactory.from_native(v)
def merge_into(self, conf): f = open(self.path, "r") v = json.load(f) cf = DataFactory.from_native(v) conf.merge(cf) f.close()
def _parse_conf(self, xmle): return DataFactory.from_xmle(xmle)
def main():
# Initialization
task.check_conf(["entities", "repositories", "repositories.data", "bin_paths.gitools"])
conf = task.conf
log = task.logger()
combinations_port, combination_ids_port = \
task.ports("combinations", "combination_ids")
es = EntityServer(conf["entities"])
em = es.manager()
rs = RepositoryServer(conf["repositories"])
data_repo = rs.repository("data")
overwrite = conf.get("overwrite", False, dtype=bool)
results_base_path = types.CNV_COMBINATION.replace(".", "/")
conditions = ("gain", "loss")
for c_dict in combinations_port:
c = DataFactory.from_native(c_dict, key_sep = "/")
"""
o = em.find(c, types.CNV_ONCODRIVE_GENES)
if o is None:
log.error("{0} not found: {1}".format(types.CNV_ONCODRIVE_GENES, c))
continue
okey = (o["study_id"], o["platform_id"], o["icdo_topography"], o["icdo_morphology"])
"""
cid = c["id"]
key = (c["icdo_topography"], c["icdo_morphology"], c["id_type"])
log.info("Processing combination for ({}) [{}] ...".format(", ".join(key), cid))
#files = c["files"]
#if len(files) == 1:
# log.info("No combination required, copyed from {0}".format(files[0]))
# c["results_file"] = files[0]
#else:
results_path = rpath.join(results_base_path, cid + ".tsv.gz")
results_url = data_repo.url(results_path)
if skip_file(overwrite, data_repo, results_path, c.get("results_file")):
log.warn("Skipping {} ({}) [{}] as it already exists".format(types.CNV_COMBINATION, ", ".join(key), cid))
combination_ids_port.write(cid)
continue
c["results_file"] = results_url
combination(log, conf, rs, c, data_repo, results_path, conditions)
# save combination results
em.persist(c, types.CNV_COMBINATION)
combination_ids_port.write(cid)
em.close()
es.close()
data_repo.close()
rs.close()
