def __init__(self, wf_ctrl, python,
transfer_timeout=-24, jobs_timeout=1):
# Define example directories
import soma_workflow
self.examples_dir = os.path.join(soma_workflow.__path__[0],
"test", "data", "jobExamples")
self.output_dir = os.path.join(soma_workflow.__path__[0],
"test", "out")
if not os.path.isdir(self.output_dir):
os.mkdir(self.output_dir)
if (not os.path.isdir(self.examples_dir) or
not os.path.isdir(self.output_dir)):
raise ConfigurationError("%s or %s does not exist." % (
self.examples_dir,
self.output_dir))
self.wf_ctrl = wf_ctrl
self.tr_timeout = transfer_timeout
self.jobs_timeout = jobs_timeout
self.python = python
self.complete_path = os.path.join(self.examples_dir, "complete")
self.models_path = os.path.join(self.complete_path,
"output_models")
self.output_models = {}
for i in [11, 12, 2, 3, 4]:
self.output_models[i] = os.path.join(self.models_path,
"file" + str(i))
self.job1_output_file_models = [
self.output_models[11], self.output_models[12]]
self.job2_output_file_models = [
self.output_models[2]]
self.job3_output_file_models = [
self.output_models[3]]
self.job4_output_file_models = [
self.output_models[4]]
self.stdout_models = {}
for i in range(1, 5):
self.stdout_models[i] = os.path.join(self.models_path,
"stdout_job" + str(i))
self.job1_stdouterr_models = [
self.stdout_models[1],
os.path.join(self.models_path, "stderr_job1")]
self.job2_stdouterr_models = [
self.stdout_models[2],
os.path.join(self.models_path, "stderr_job2")]
self.job3_stdouterr_models = [
self.stdout_models[3],
os.path.join(self.models_path, "stderr_job3")]
self.job4_stdouterr_models = [
self.stdout_models[4],
os.path.join(self.models_path, "stderr_job4")]
self.exceptionjobstdouterr = [
os.path.join(self.models_path, "stdout_exception_job"),
os.path.join(self.models_path, "stderr_exception_job")]
def load_from_file(cls, config_file_path=None):
'''
Load LocalSchedulerCfg and return it
Parameters
----------
config_file_path: string or file object
filename or file object to read config from
Returns
-------
LocalSchedulerCfg object
'''
hostname = socket.gethostname()
if config_file_path:
config_path = config_file_path
else:
config_path = Configuration.search_config_path()
config_parser = configparser.ConfigParser()
if hasattr(config_path, 'readline'):
config_parser.readfp(config_path)
else:
config_parser.read(config_path)
if not config_parser.has_section(hostname):
raise ConfigurationError("Wrong config file format. Can not find "
"section " + hostname +
" in configuration "
"file: " + config_path)
proc_nb = 0
max_proc_nb = 0
interval = None
if config_parser.has_option(hostname, OCFG_SCDL_CPU_NB):
proc_nb_str = config_parser.get(socket.gethostname(),
OCFG_SCDL_CPU_NB)
proc_nb = int(proc_nb_str)
if config_parser.has_option(hostname, OCFG_SCDL_INTERVAL):
interval_str = config_parser.get(hostname, OCFG_SCDL_INTERVAL)
interval = int(interval_str)
if config_parser.has_option(hostname, OCFG_SCDL_MAX_CPU_NB):
max_proc_nb_str = config_parser.get(socket.gethostname(),
OCFG_SCDL_MAX_CPU_NB)
max_proc_nb = int(max_proc_nb_str)
config = cls(proc_nb=proc_nb,
interval=interval,
max_proc_nb=max_proc_nb)
config._config_path = config_path
return config
def get_server_name(self):
if self._config_parser == None or self._server_name != None:
return self._server_name
if not self._config_parser.has_option(self._resource_id,
CFG_SERVER_NAME):
raise ConfigurationError(
"Can not find the configuration item %s "
"for the resource %s, in the configuration "
"file %s." %
(CFG_SERVER_NAME, self._resource_id, self._config_path))
self._server_name = self._config_parser.get(self._resource_id,
CFG_SERVER_NAME)
return self._server_name
def get_cluster_address(self):
if self._config_parser == None or self._cluster_address:
return self._cluster_address
if not self._config_parser.has_option(self._resource_id,
CFG_CLUSTER_ADDRESS):
raise ConfigurationError(
"Can not find the configuration item %s for the "
"resource %s, in the configuration file %s." %
(CFG_CLUSTER_ADDRESS, self._resource_id, self._config_path))
self._cluster_address = self._config_parser.get(
self._resource_id, CFG_CLUSTER_ADDRESS)
return self._cluster_address
def __init__(self):
# Define example directories
import soma_workflow
module_file = soma_workflow.__file__
if module_file.endswith('.pyc') or module_file.endswith('.pyo'):
module_file = module_file[:-1]
module_file = os.path.realpath(module_file)
module_path = os.path.dirname(module_file)
self.examples_dir = os.path.join(module_path, "test", "data",
"jobExamples")
tmp = tempfile.mkdtemp('', prefix='swf_test_')
self.output_dir = tmp
if (not os.path.isdir(self.examples_dir)
or not os.path.isdir(self.output_dir)):
raise ConfigurationError("%s or %s does not exist." %
(self.examples_dir, self.output_dir))
def get_submitting_machines(self):
if self._config_parser == None or self._submitting_machines:
return self._submitting_machines
if not self._config_parser.has_option(self._resource_id,
CFG_SUBMITTING_MACHINES):
raise ConfigurationError(
"Can not find the configuration item %s for the "
"resource %s, in the configuration file %s." %
(CFG_SUBMITTING_MACHINES, self._resource_id,
self._config_path))
submitting_machines = self._config_parser.get(
self._resource_id, CFG_SUBMITTING_MACHINES).split()
self._submitting_machines = []
for sub_machine in submitting_machines:
self._submitting_machines.append(os.path.expandvars(sub_machine))
return self._submitting_machines
def get_path_translation(self):
if self._config_parser == None or self.path_translation != None:
return self.path_translation
self.path_translation = {}
if self._config_parser.has_option(self._resource_id,
OCFG_PATH_TRANSLATION_FILES):
translation_files_str = self._config_parser.get(
self._resource_id, OCFG_PATH_TRANSLATION_FILES)
# logger.info("Path translation files configured:")
for ns_file_str in translation_files_str.split():
ns_file = ns_file_str.split("{")
namespace = ns_file[0]
filename = ns_file[1].rstrip("}")
filename = os.path.expandvars(filename)
# logger.info(" -namespace: " + namespace + ", translation
# file: " + filename)
try:
f = open(filename, "r")
except IOError as e:
raise ConfigurationError(
"Can not read the translation file %s" % (filename))
if not namespace in self.path_translation.keys():
self.path_translation[namespace] = {}
line = f.readline()
while line:
splitted_line = line.split(None, 1)
if len(splitted_line) > 1:
uuid = splitted_line[0]
content = splitted_line[1].rstrip()
# logger.info(" uuid: " + uuid + " translation:" +
# content)
self.path_translation[namespace][
uuid] = os.path.expandvars(content)
line = f.readline()
f.close()
return self.path_translation
def load_from_file(cls, resource_id=None, config_file_path=None):
'''
Load config from a file, and return it
Parameters
----------
resource_id: string
computing resource identifier to get specific config for
config_file_path: string or file object
filename or file object for config to be read.
Returns
-------
Configuration object
'''
if config_file_path:
config_path = config_file_path
else:
config_path = Configuration.search_config_path()
config = None
home_dir = Configuration.get_home_dir()
if resource_id == None \
or resource_id in (socket.gethostname(),
socket.gethostname().split('.')[0]):
# scheduler local on the local machine
resource_id = socket.gethostname()
mode = LIGHT_MODE
scheduler_type = LOCAL_SCHEDULER
swf_dir = os.path.join(home_dir, ".soma-workflow")
database_file = None
if config_path is not None:
config_parser = configparser.ConfigParser()
if hasattr(config_path, 'readline'):
config_parser.readfp(config_path)
else:
config_parser.read(config_path)
if config_parser.has_section(resource_id):
if config_parser.has_option(resource_id, OCFG_SWF_DIR):
swf_dir = config_parser.get(resource_id, OCFG_SWF_DIR)
if config_parser.has_option(resource_id,
CFG_DATABASE_FILE):
database_file = config_parser.get(
resource_id, CFG_DATABASE_FILE)
if database_file is None:
database_file = os.path.join(
swf_dir, "soma_workflow-%s.db" % DB_VERSION)
config = cls(resource_id=resource_id,
mode=mode,
scheduler_type=scheduler_type,
database_file=database_file,
transfered_file_dir=None)
if config_path is not None \
and config_parser.has_section(resource_id):
config._config_parser = config_parser
config._config_path = config_path
try:
transfered_file_dir = config.get_transfered_file_dir()
except:
transfered_file_dir = os.path.join(swf_dir, "transfered_files")
config._transfered_file_dir = transfered_file_dir
if not os.path.isdir(os.path.join(home_dir, ".soma-workflow")):
try:
os.mkdir(os.path.join(home_dir, ".soma-workflow"))
except OSError:
pass # ignore failed mkdir
if not os.path.isdir(transfered_file_dir):
try:
os.mkdir(transfered_file_dir)
except OSError:
pass # ignore failed mkdir
return config
else:
if config_path == None:
raise ConfigurationError(
"A configuration file is required to connect "
"to " + repr(resource_id) + ": the soma-workflow "
"configuration file could not be found.")
config_parser = configparser.ConfigParser()
if hasattr(config_path, 'readline'):
config_parser.readfp(config_path)
else:
config_parser.read(config_path)
if not config_parser.has_section(resource_id):
raise ConfigurationError("Can not find section " +
repr(resource_id) + " "
"in configuration file: " +
config_path)
scheduler_type = None
if config_parser.has_option(resource_id, OCFG_SCHEDULER_TYPE):
scheduler_type = config_parser.get(resource_id,
OCFG_SCHEDULER_TYPE)
if scheduler_type not in SCHEDULER_TYPES:
raise ConfigurationError(
"Unknown scheduler type:"
" " + repr(scheduler_type) +
"Scheduler type must be one of the "
"following:" + repr(SCHEDULER_TYPES))
else:
scheduler_type = DRMAA_SCHEDULER
config = cls(resource_id=resource_id,
mode=None,
scheduler_type=scheduler_type,
database_file=None,
transfered_file_dir=None)
config._config_parser = config_parser
config._config_path = config_path
return config