def __init__(self, rule, base_path=None, platform=None):
rule = Data.create(rule)
self.on = rule.get("on", {})
if isinstance(self.on, basestring):
self.on = dict(task=self.on)
if platform is not None:
self.on["platform"] = platform
self.dels = rule.get("del", default=Data.list)
if not Data.is_list(self.dels):
raise Exception(
"Expected a list of strings for del operations of rule: {}".
format(repr(rule)))
for k in self.dels:
if not isinstance(k, basestring):
raise Exception(
"Expected a list of strings for del operations of rule: {}"
.format(repr(rule)))
self.set = rule.get("set", default=Data.list)
if not Data.is_list(self.dels):
raise Exception(
"Expected a list of tuples [key, value] for set operations of rule: {}"
.format(repr(rule)))
for s in self.set:
if not Data.is_list(s) or len(s) != 2:
raise Exception(
"Expected a list of tuples [key, value] for set operations of rule: {}"
.format(repr(rule)))
self.merge = rule.get("merge")
if isinstance(self.merge, basestring):
if not os.path.isabs(self.merge):
if base_path is None:
raise Exception(
"Configuration rule merge path should be absolute path: {}"
.format(self.merge))
else:
self.merge = os.path.join(base_path, self.merge)
if not os.path.isfile(self.merge):
raise Exception(
"Configuration rule merge path not found: {}".format(
self.merge))
self.merge = ConfigLoader(os.path.join(base_path or "",
self.merge)).load()
if self.merge is not None and not Data.is_element(self.merge):
raise Exception(
"Expected a dictionary for merge operation of rule: {}".format(
repr(rule)))
def single_project(projects_port):
log = task.logger
conf = task.conf
log.info("Single project ...")
files = conf["files"]
if isinstance(files, basestring):
files = [files]
elif not Data.is_list(files):
raise InternalError("Unexpected configuration type for 'files', only string and list are allowed")
files = [str(f) for f in files] #unicode is not json serializable
project_id = conf["project.id"]
assembly = conf.get("assembly", "hg19")
annotations = conf.get("annotations")
project = {
"id" : project_id,
"files" : files,
"assembly" : assembly
}
if annotations is not None and isinstance(annotations, dict):
project["annotations"] = annotations
make_project(log, conf, project)
projects_port.send(project)
def get_handler(logger, conf):
if not Data.is_element(conf):
logger.error("Malformed log handler:\n{1}".format(repr(conf)))
return
type = conf.get("type")
if type is None or not isinstance(
type, basestring) or type.lower() not in _HANDLERS.keys():
logger.error("Unknown or unsupported handler type: {0}\n{1}".format(
type, repr(conf)))
return
handler = _HANDLERS[type](conf)
level = conf.get("level")
if level is not None:
handler.setLevel(get_level(level))
format = conf.get("format", _DEFAULT_FORMAT)
if format is not None:
if Data.is_list(format):
format = "".join(format.to_native())
handler.setFormatter(logging.Formatter(format))
return handler
def get_annotations_to_save(self, keys, annotations, names=None, values=None):
assert (names is None and values is None) or (names is not None and values is not None and len(names) == len(values))
if names is None:
names = []
values = []
else:
names = [name for name in names]
values = [value for value in values]
ann_keys = keys
if ann_keys is None:
ann_keys = []
elif Data.is_list(ann_keys):
ann_keys = ann_keys.to_native()
else:
log = logger.get_logger(__name__)
log.warn("Wrong type for 'project.annotations', expecting a list but found:\n{0}".format(repr(ann_keys)))
ann_keys = []
for ann_key in ann_keys:
default = None
if isinstance(ann_key, basestring):
key = name = ann_key
elif isinstance(ann_key, list) and len(ann_key) == 2:
key = ann_key[0]
name = ann_key[1]
value = annotations[key] if key in annotations else default
names += [name]
values += [value]
return names, values
def __init__(self, conf):
JobManager.__init__(self, "saga", conf)
self._file_url = self._conf.get("files_url", "file://")
self._remote_path = self._conf.get("remote_path", self._work_path)
self._output_path = os.path.join(self._work_path, "output")
self._remote_output_path = os.path.join(self._remote_path, "output")
self._pe = self._conf.get("pe")
self._cpu_count = self._conf.get("cpu_count", 1)
self._queue = self._conf.get("queue")
self._project = self._conf.get("project")
self._working_directory = self._conf.get("working_directory")
self._state_check_interval = self._conf.get("state_check_interval", 5)
ctx_conf = self._conf.get("context")
if ctx_conf is not None and not (Data.is_element(ctx_conf) or Data.is_list(ctx_conf)):
raise ConfigTypeError("context", ctx_conf)
self._session = None
self._job_service = None
self._queued_count = 0
self._max_queued = self._conf.get("max_queued", 0)
self._running = False
self._run_thread = None
self._join_thread = None
def scan_projects(projects_port):
log = task.logger
conf = task.conf
config = GlobalConfig(conf)
log.debug("Configuration:")
log.debug(config)
log.info("Creating combination folders ...")
paths = PathsConfig(config)
create_combination_folders(conf)
if "projects" not in conf or not Data.is_list(conf["projects"]):
raise InternalError("Required 'projects' configuration parameter has not been found or it is not well defined")
log.info("Initializing projects ...")
for project in conf["projects"]:
if project is None:
continue
project = validate_project(log, project)
project.expand_vars(conf)
log.info("--- [{}] ---------------------------------------------------".format(project["id"]))
project = init_project(log, config, paths, task.storage, project)
log.info(" assembly: {}, variant_files: {}".format(project["assembly"], len(project["files"])))
projects_port.send(project)
def __init__(self, rule, base_path=None, platform=None):
rule = Data.create(rule)
self.on = rule.get("on", {})
if isinstance(self.on, basestring):
self.on = dict(task=self.on)
if platform is not None:
self.on["platform"] = platform
self.dels = rule.get("del", default=Data.list)
if not Data.is_list(self.dels):
raise Exception("Expected a list of strings for del operations of rule: {}".format(repr(rule)))
for k in self.dels:
if not isinstance(k, basestring):
raise Exception("Expected a list of strings for del operations of rule: {}".format(repr(rule)))
self.set = rule.get("set", default=Data.list)
if not Data.is_list(self.dels):
raise Exception("Expected a list of tuples [key, value] for set operations of rule: {}".format(repr(rule)))
for s in self.set:
if not Data.is_list(s) or len(s) != 2:
raise Exception("Expected a list of tuples [key, value] for set operations of rule: {}".format(repr(rule)))
self.merge = rule.get("merge")
if isinstance(self.merge, basestring):
if not os.path.isabs(self.merge):
if base_path is None:
raise Exception("Configuration rule merge path should be absolute path: {}".format(self.merge))
else:
self.merge = os.path.join(base_path, self.merge)
if not os.path.isfile(self.merge):
raise Exception("Configuration rule merge path not found: {}".format(self.merge))
self.merge = ConfigLoader(os.path.join(base_path or "", self.merge)).load()
if self.merge is not None and not Data.is_element(self.merge):
raise Exception("Expected a dictionary for merge operation of rule: {}".format(repr(rule)))
def _create_platforms(self):
"""
Creates the platform according to the configuration
:return: Platform
"""
platform_confs = self._conf.get("platforms")
if platform_confs is None:
platform_confs = Data.list()
elif not Data.is_list(platform_confs):
self._log.error("Wrong configuration type for 'platforms': {}".format(platform_confs))
platform_confs = Data.list()
if len(platform_confs) == 0:
platform_confs += [Data.element(dict(type="local"))]
platforms = []
names = {}
for pidx, platform_conf in enumerate(platform_confs):
if isinstance(platform_conf, basestring):
if not os.path.isabs(platform_conf) and self._conf_base_path is not None:
platform_conf = os.path.join(self._conf_base_path, platform_conf)
platform_conf = ConfigLoader(platform_conf).load()
if not Data.is_element(platform_conf):
raise errors.ConfigTypeError("wok.platforms[{}]".format(pidx, platform_conf))
ptype = platform_conf.get("type", "local")
name = platform_conf.get("name", ptype)
if name in names:
name = "{}-{}".format(name, names[name])
names[name] += 1
else:
names[name] = 2
platform_conf["name"] = name
if "work_path" not in platform_conf:
platform_conf["work_path"] = os.path.join(self._work_path, "platform_{}".format(name))
self._log.info("Creating '{}' platform ...".format(name))
self._log.debug("Platform configuration: {}".format(repr(platform_conf)))
platforms += [create_platform(ptype, platform_conf)]
return platforms
def initialize(self):
self._log.info("Initializing projects ...")
if Data.is_element(self.conf):
iter_conf = self._iter_dict(self.conf)
elif Data.is_list(self.conf):
iter_conf = self._iter_list(self.conf)
else:
iter_conf = iter([])
for name, pdesc in iter_conf:
if isinstance(pdesc, basestring):
pdesc = self._load_project_desc(pdesc, self.base_path)
self._add_project_desc(pdesc, self.base_path)
for name, project in sorted(self._projects.items(), key=lambda x: x[0]):
project.initialize()
def _load_project_desc(self, path, base_path=None):
if not os.path.isabs(path):
if base_path is not None:
path = os.path.join(base_path, path)
else:
path = os.path.abspath(path)
if not os.path.exists(path):
raise Exception("Project path not found: {}".format(path))
if os.path.isdir(path):
path = os.path.join(path, "project.conf")
if not os.path.isfile(path):
raise Exception("Project configuration not found: {}".format(path))
project = Data.element()
project.merge(ConfigLoader(path).load())
base_path = os.path.dirname(path)
if "path" not in project:
project["path"] = base_path
if not os.path.isabs(project["path"]):
project["path"] = os.path.normpath(
os.path.join(base_path, project["path"]))
if "conf" in project and isinstance(project["conf"], basestring):
conf_path = os.path.join(base_path, project["conf"])
project["conf"] = ConfigLoader(conf_path).load()
if "conf_rules" in project and isinstance(project["conf_rules"],
basestring):
base_path = os.path.dirname(path)
conf_path = os.path.join(base_path, project["conf_rules"])
project["conf_rules"] = ConfigLoader(conf_path).load()
if "conf_rules" in project and Data.is_list(project["conf_rules"]):
for rule in project["conf_rules"]:
if Data.is_element(rule) and "merge" in rule and isinstance(
rule["merge"], basestring):
rule["merge"] = ConfigLoader(
os.path.join(base_path, rule["merge"])).load()
return project
def initialize(self):
self._log.info("Initializing projects ...")
if Data.is_element(self.conf):
iter_conf = self._iter_dict(self.conf)
elif Data.is_list(self.conf):
iter_conf = self._iter_list(self.conf)
else:
iter_conf = iter([])
for name, pdesc in iter_conf:
if isinstance(pdesc, basestring):
pdesc = self._load_project_desc(pdesc, self.base_path)
self._add_project_desc(pdesc, self.base_path)
for name, project in sorted(self._projects.items(),
key=lambda x: x[0]):
project.initialize()
def _load_project_desc(self, path, base_path=None):
if not os.path.isabs(path):
if base_path is not None:
path = os.path.join(base_path, path)
else:
path = os.path.abspath(path)
if not os.path.exists(path):
raise Exception("Project path not found: {}".format(path))
if os.path.isdir(path):
path = os.path.join(path, "project.conf")
if not os.path.isfile(path):
raise Exception("Project configuration not found: {}".format(path))
project = Data.element()
project.merge(ConfigLoader(path).load())
base_path = os.path.dirname(path)
if "path" not in project:
project["path"] = base_path
if not os.path.isabs(project["path"]):
project["path"] = os.path.normpath(os.path.join(base_path, project["path"]))
if "conf" in project and isinstance(project["conf"], basestring):
conf_path = os.path.join(base_path, project["conf"])
project["conf"] = ConfigLoader(conf_path).load()
if "conf_rules" in project and isinstance(project["conf_rules"], basestring):
base_path = os.path.dirname(path)
conf_path = os.path.join(base_path, project["conf_rules"])
project["conf_rules"] = ConfigLoader(conf_path).load()
if "conf_rules" in project and Data.is_list(project["conf_rules"]):
for rule in project["conf_rules"]:
if Data.is_element(rule) and "merge" in rule and isinstance(rule["merge"], basestring):
rule["merge"] = ConfigLoader(os.path.join(base_path, rule["merge"])).load()
return project
def initialize(conf=None, format=None, datefmt=None, level=None):
"""
Initialize the logging system.
If conf is a dictionary then the parameters considered for configuration are:
- format: Logger format
- datefmt: Date format
- loggers: list of tuples (name, conf) to configure loggers
If conf is a list then only the loggers are configured.
If conf is an string then the default logger is configured for the logging level.
"""
global _initialized
if conf is None:
conf = Data.element()
elif not isinstance(conf, basestring):
conf = Data.create(conf)
if Data.is_list(conf):
loggers_conf = conf
conf = Data.element()
elif Data.is_element(conf):
loggers_conf = conf.get("loggers", default=Data.list)
elif isinstance(conf, basestring):
loggers_conf = Data.list([["", conf]])
conf = Data.element()
format = format or conf.get("format", _DEFAULT_FORMAT)
datefmt = datefmt or conf.get("datefmt", _DEFAULT_DATEFMT)
logging.basicConfig(format=format, datefmt=datefmt)
for (log_name, log_conf) in loggers_conf:
init_logger(log_name, conf=log_conf)
if level is not None:
init_logger("", conf=level)
_initialized = True
def initialize(conf=None, format=None, datefmt=None, level=None):
"""
Initialize the logging system.
If conf is a dictionary then the parameters considered for configuration are:
- format: Logger format
- datefmt: Date format
- loggers: list of tuples (name, conf) to configure loggers
If conf is a list then only the loggers are configured.
If conf is an string then the default logger is configured for the logging level.
"""
global _initialized
if conf is None:
conf = Data.element()
elif not isinstance(conf, basestring):
conf = Data.create(conf)
if Data.is_list(conf):
loggers_conf = conf
conf = Data.element()
elif Data.is_element(conf):
loggers_conf = conf.get("loggers", default=Data.list)
elif isinstance(conf, basestring):
loggers_conf = Data.list([["", conf]])
conf = Data.element()
format = format or conf.get("format", _DEFAULT_FORMAT)
datefmt = datefmt or conf.get("datefmt", _DEFAULT_DATEFMT)
logging.basicConfig(format=format, datefmt=datefmt)
for (log_name, log_conf) in loggers_conf:
init_logger(log_name, conf=log_conf)
if level is not None:
init_logger("", conf=level)
_initialized = True
def get_handler(logger, conf):
if not Data.is_element(conf):
logger.error("Malformed log handler:\n{1}".format(repr(conf)))
return
type = conf.get("type")
if type is None or not isinstance(type, basestring) or type.lower() not in _HANDLERS.keys():
logger.error("Unknown or unsupported handler type: {0}\n{1}".format(type, repr(conf)))
return
handler = _HANDLERS[type](conf)
level = conf.get("level")
if level is not None:
handler.setLevel(get_level(level))
format = conf.get("format", _DEFAULT_FORMAT)
if format is not None:
if Data.is_list(format):
format = "".join(format.to_native())
handler.setFormatter(logging.Formatter(format))
return handler
class NativeCommmandBuilder(CommmandBuilder):
def _plain_conf(self, value, path=None):
if path is None:
path = []
if not Data.is_element(value):
yield (".".join(path), value)
else:
for key in value.keys():
for k, v in self._plain_conf(value[key], path + [key]):
yield (k, v)
def prepare(self, case, task, index):
execution = task.execution
exec_conf = execution.conf
if exec_conf is None:
exec_conf = Data.element()
if "script_path" not in exec_conf:
raise MissingValueError("script_path")
script_path = exec_conf["script_path"]
lang = exec_conf.get("language", "python")
case_conf = case.conf.clone().expand_vars()
# Environment variables
env = Data.element()
#for k, v in os.environ.items():
# env[k] = v
env.merge(task.conf.get(rtconf.TASK_ENV))
env.merge(exec_conf.get("env"))
# Default module script path
platform_project_path = task.conf.get(rtconf.PROJECT_PATH,
case.project.path)
flow_path = os.path.abspath(os.path.dirname(task.flow_path))
flow_rel_path = os.path.relpath(flow_path, case.project.path)
platform_script_path = os.path.join(platform_project_path,
flow_rel_path, script_path)
env[ENV_PROJECT_PATH] = platform_project_path
env[ENV_FLOW_PATH] = flow_rel_path
env[ENV_SCRIPT_PATH] = script_path
env[ENV_PLATFORM_SCRIPT_PATH] = platform_script_path
script = []
sources = task.conf.get(rtconf.TASK_SOURCES, default=Data.list)
if isinstance(sources, basestring):
sources = Data.list([sources])
for source in sources:
script += ['source "{}"'.format(source)]
if lang == "python":
virtualenv = task.conf.get(rtconf.TASK_PYTHON_VIRTUALENV)
if virtualenv is not None:
#script += ["set -x"]
#script += ["echo Activating virtualenv {} ...".format(virtualenv)]
script += [
'source "{}"'.format(
os.path.join(virtualenv, "bin", "activate"))
]
#script += ["set +x"]
#script += ["echo Running workitem ..."]
cmd = [task.conf.get(rtconf.TASK_PYTHON_BIN, "python")]
cmd += ["${}".format(ENV_PLATFORM_SCRIPT_PATH)]
lib_path = task.conf.get(rtconf.TASK_PYTHON_LIBS)
if lib_path is not None:
if Data.is_list(lib_path):
lib_path = ":".join(lib_path)
if "PYTHONPATH" in env:
env["PYTHONPATH"] = lib_path + ":" + env["PYTHONPATH"]
else:
env["PYTHONPATH"] = lib_path
else:
raise LanguageError(lang)
cmd += [
"-D", "case={}".format(case.name), "-D",
"task={}".format(task.cname), "-D", "index={}".format(index)
]
#for key, value in self._storage_conf(workitem.case.engine.storage.basic_conf):
# cmd += ["-D", "storage.{}={}".format(key, value)]
for key, value in self._plain_conf(
Data.create(task.platform.data.context_conf(CTX_EXEC))):
cmd += ["-D", "data.{}={}".format(key, value)]
for key, value in self._plain_conf(
task.platform.storage.context_conf(CTX_EXEC)):
cmd += ["-D", "storage.{}={}".format(key, value)]
script += [" ".join(cmd)]
return "\n".join(script), env.to_native()
def prepare(self, case, task, index):
execution = task.execution
exec_conf = execution.conf
if exec_conf is None:
exec_conf = Data.element()
if "script_path" not in exec_conf:
raise MissingValueError("script_path")
script_path = exec_conf["script_path"]
lang = exec_conf.get("language", "python")
case_conf = case.conf.clone().expand_vars()
# Environment variables
env = Data.element()
#for k, v in os.environ.items():
# env[k] = v
env.merge(task.conf.get(rtconf.TASK_ENV))
env.merge(exec_conf.get("env"))
# Default module script path
platform_project_path = task.conf.get(rtconf.PROJECT_PATH, case.project.path)
flow_path = os.path.abspath(os.path.dirname(task.flow_path))
flow_rel_path = os.path.relpath(flow_path, case.project.path)
platform_script_path = os.path.join(platform_project_path, flow_rel_path, script_path)
env[ENV_PROJECT_PATH] = platform_project_path
env[ENV_FLOW_PATH] = flow_rel_path
env[ENV_SCRIPT_PATH] = script_path
env[ENV_PLATFORM_SCRIPT_PATH] = platform_script_path
script = []
sources = task.conf.get(rtconf.TASK_SOURCES, default=Data.list)
if isinstance(sources, basestring):
sources = Data.list([sources])
for source in sources:
script += ['source "{}"'.format(source)]
if lang == "python":
virtualenv = task.conf.get(rtconf.TASK_PYTHON_VIRTUALENV)
if virtualenv is not None:
#script += ["set -x"]
#script += ["echo Activating virtualenv {} ...".format(virtualenv)]
script += ['source "{}"'.format(os.path.join(virtualenv, "bin", "activate"))]
#script += ["set +x"]
#script += ["echo Running workitem ..."]
cmd = [task.conf.get(rtconf.TASK_PYTHON_BIN, "python")]
cmd += ["${}".format(ENV_PLATFORM_SCRIPT_PATH)]
lib_path = task.conf.get(rtconf.TASK_PYTHON_LIBS)
if lib_path is not None:
if Data.is_list(lib_path):
lib_path = ":".join(lib_path)
if "PYTHONPATH" in env:
env["PYTHONPATH"] = lib_path + ":" + env["PYTHONPATH"]
else:
env["PYTHONPATH"] = lib_path
else:
raise LanguageError(lang)
cmd += ["-D", "case={}".format(case.name),
"-D", "task={}".format(task.cname),
"-D", "index={}".format(index)]
#for key, value in self._storage_conf(workitem.case.engine.storage.basic_conf):
# cmd += ["-D", "storage.{}={}".format(key, value)]
for key, value in self._plain_conf(Data.create(task.platform.data.context_conf(CTX_EXEC))):
cmd += ["-D", "data.{}={}".format(key, value)]
for key, value in self._plain_conf(task.platform.storage.context_conf(CTX_EXEC)):
cmd += ["-D", "storage.{}={}".format(key, value)]
script += [" ".join(cmd)]
return "\n".join(script), env.to_native()
def multiple_projects(projects_port):
log = task.logger
conf = task.conf
log.info("Scanning multiple projects ...")
scan_paths = conf.get("scan_paths")
if scan_paths is None or not Data.is_list(scan_paths):
raise InternalError("Unexpected scan_paths type: {0}".format(type(scan_paths)))
# Prepare includes and excludes
includes = conf.get("includes")
if includes is not None and not Data.is_list(includes):
raise InternalError("Unexpected includes type: {0}".format(type(includes)))
if includes is None or len(includes) == 0:
includes = ["^.*$"]
excludes = conf.get("excludes")
if excludes is not None and not Data.is_list(excludes):
raise InternalError("Unexpected excludes type: {0}".format(type(excludes)))
if excludes is None:
excludes = []
log.debug("scan_paths:\n{0}".format("\n ".join(conf["scan_paths"])))
log.debug("Includes:\n{0}".format("\n ".join(includes)))
log.debug("Excludes:\n{0}".format("\n ".join(excludes)))
# compile regular expressions
includes = [re.compile(inc) for inc in includes]
excludes = [re.compile(exc) for exc in excludes]
# scan paths
project_ids = set()
for scan_path in scan_paths:
for path, project in list_projects(log, scan_path):
if "id" not in project:
log.warn("Discarding project that doesn't have 'id': {0}".format(path))
continue
if "files" not in project:
log.warn("Discarding project that doesn't have 'files': {0}".format(path))
continue
project_id = project["id"]
if "name" in project:
project_name = ": " + project["name"]
else:
project_name = ""
if match_id(project_id, includes) and not match_id(project_id, excludes):
if project_id in project_ids:
raise Exception("Duplicated project id at {0}".format(path))
log.info("Included {0}{1}".format(project_id, project_name))
make_project(log, conf, project, base_path=os.path.dirname(path))
projects_port.send(project)
else:
log.info("Excluded {0}{1}".format(project_id, project_name))
