def write_source_vc_info(run_source_dir, output=None, popen=None):
"""Write version control information of sources used in run time.
run_source_dir -- The source directory we are interested in.
output -- An open file handle or a string containing a writable path.
If not specified, use sys.stdout.
popen -- A metomi.rose.popen.RosePopener instance for running vc commands.
If not specified, use a new local instance.
"""
if popen is None:
popen = RosePopener()
if output is None:
handle = sys.stdout
elif hasattr(output, "write"):
handle = output
else:
handle = open(output, "wb")
msg = "%s\n" % run_source_dir
write_safely(msg, handle)
environ = dict(os.environ)
environ["LANG"] = "C"
for vcs, args_list in [
(
"svn",
[
["info", "--non-interactive"],
["status", "--non-interactive"],
["diff", "--internal-diff", "--non-interactive"],
],
),
("git", [["describe"], ["status"], ["diff"]]),
]:
if not popen.which(vcs):
continue
cwd = os.getcwd()
os.chdir(run_source_dir)
try:
for args in args_list:
cmd = [vcs, *args]
ret_code, out, _ = popen.run(*cmd, env=environ)
if out:
write_safely(("#" * 80 + "\n"), handle)
write_safely(("# %s\n" % popen.shlex_join(cmd)), handle)
write_safely(("#" * 80 + "\n"), handle)
write_safely(out, handle)
if ret_code: # If cmd fails once, it will likely fail again
break
finally:
os.chdir(cwd)
def __init__(self,
prefixes=None,
prompt_func=None,
popen=None,
event_handler=None):
if not event_handler:
event_handler = Reporter()
if not popen:
popen = RosePopener(event_handler=event_handler)
self.event_handler = event_handler
self.popen = popen
self.prompt_func = prompt_func
self.prefixes = []
self.unreachable_prefixes = []
self.auth_managers = {}
conf = ResourceLocator.default().get_conf()
conf_rosie_id = conf.get(["rosie-id"], no_ignore=True)
if conf_rosie_id is None:
raise RosieWSClientConfError()
for key, node in conf_rosie_id.value.items():
if node.is_ignored() or not key.startswith("prefix-ws."):
continue
prefix = key.replace("prefix-ws.", "")
self.auth_managers[prefix] = RosieWSClientAuthManager(
prefix, popen=self.popen, prompt_func=self.prompt_func)
if not prefixes:
prefixes_str = conf_rosie_id.get_value(["prefixes-ws-default"])
if prefixes_str:
prefixes = shlex.split(prefixes_str)
else:
prefixes = sorted(self.auth_managers.keys())
self.set_prefixes(prefixes)
def main():
"""Implement "rose suite-hook" command."""
opt_parser = RoseOptionParser()
opt_parser.add_my_options("mail_cc", "mail", "retrieve_job_logs",
"shutdown")
opts, args = opt_parser.parse_args()
for key in ["mail_cc"]:
values = []
if getattr(opts, key):
for value in getattr(opts, key):
values.extend(value.split(","))
setattr(opts, key, values)
report = Reporter(opts.verbosity - opts.quietness - 1) # Reduced default
popen = RosePopener(event_handler=report)
suite_engine_proc = SuiteEngineProcessor.get_processor(
event_handler=report, popen=popen)
args = suite_engine_proc.process_suite_hook_args(*args, **vars(opts))
hook = RoseSuiteHook(event_handler=report,
popen=popen,
suite_engine_proc=suite_engine_proc)
hook(*args,
should_mail=opts.mail,
mail_cc_list=opts.mail_cc,
should_shutdown=opts.shutdown,
should_retrieve_job_logs=opts.retrieve_job_logs)
def __init__(
self,
event_handler=None,
popen=None,
config_pm=None,
fs_util=None,
suite_engine_proc=None,
):
if not self.CONF_NAME:
self.CONF_NAME = self.NAME
self.event_handler = event_handler
if popen is None:
popen = RosePopener(event_handler)
self.popen = popen
if fs_util is None:
fs_util = FileSystemUtil(event_handler)
self.fs_util = fs_util
if config_pm is None:
config_pm = ConfigProcessorsManager(event_handler, popen, fs_util)
self.config_pm = config_pm
if suite_engine_proc is None:
suite_engine_proc = SuiteEngineProcessor.get_processor(
event_handler=event_handler, popen=popen, fs_util=fs_util
)
self.suite_engine_proc = suite_engine_proc
self.conf_tree_loader = ConfigTreeLoader()
def __init__(self, opts, reporter=None, popen=None, fs_util=None):
self.opts = opts
if reporter is None:
self.reporter = Reporter(opts.verbosity - opts.quietness)
else:
self.reporter = reporter
if popen is None:
self.popen = RosePopener(event_handler=self.reporter)
else:
self.popen = popen
if fs_util is None:
self.fs_util = FileSystemUtil(event_handler=self.reporter)
else:
self.fs_util = fs_util
self.host_selector = HostSelector(event_handler=self.reporter,
popen=self.popen)
def __init__(self, event_handler=None, popen=None):
self.event_handler = event_handler
if popen is None:
popen = RosePopener(event_handler=event_handler)
self.popen = popen
self.scorers = {}
self.local_host_strs = None
def __init__(self,
event_handler=None,
popen=None,
fs_util=None,
force_mode=False):
if event_handler is None:
event_handler = self._dummy
self.event_handler = event_handler
if popen is None:
popen = RosePopener(event_handler)
self.popen = popen
if fs_util is None:
fs_util = FileSystemUtil(event_handler)
self.fs_util = fs_util
self.force_mode = force_mode
self._work_dir = None
atexit.register(self._delete_work_dir)
self.subversion_servers_conf = None
subversion_servers_conf = os.getenv("ROSIE_SUBVERSION_SERVERS_CONF")
if subversion_servers_conf:
self.subversion_servers_conf = subversion_servers_conf
else:
subversion_servers_conf = os.path.expanduser(
self.SUBVERSION_SERVERS_CONF)
if os.path.exists(subversion_servers_conf):
self.subversion_servers_conf = subversion_servers_conf
def __init__(self, event_handler=None, popen=None, suite_engine_proc=None):
self.event_handler = event_handler
if popen is None:
popen = RosePopener(event_handler)
self.popen = popen
if suite_engine_proc is None:
suite_engine_proc = SuiteEngineProcessor.get_processor(
event_handler=event_handler, popen=popen)
self.suite_engine_proc = suite_engine_proc
def __init__(self, event_handler=None, popen=None):
if event_handler is None:
event_handler = Reporter()
self.event_handler = event_handler
if popen is None:
popen = RosePopener(self.event_handler)
self.popen = popen
self.path = os.path.dirname(
os.path.dirname(sys.modules["metomi.rosie"].__file__))
def rose_fileinstall(srcdir=None, opts=None, rundir=None):
"""Call Rose Fileinstall.
Args:
srcdir(pathlib.Path):
Search for a ``rose-suite.conf`` file in this location.
rundir (pathlib.Path)
"""
if not rose_config_exists(rundir, opts):
return False
# Load the config tree
config_tree = rose_config_tree_loader(rundir, opts)
if any(i.startswith('file') for i in config_tree.node.value):
try:
startpoint = os.getcwd()
os.chdir(rundir)
except FileNotFoundError as exc:
raise exc
else:
# Carry out imports.
import asyncio
from metomi.rose.config_processor import ConfigProcessorsManager
from metomi.rose.popen import RosePopener
from metomi.rose.reporter import Reporter
from metomi.rose.fs_util import FileSystemUtil
# Update config tree with install location
# NOTE-TO-SELF: value=os.environ["CYLC_WORKFLOW_RUN_DIR"]
config_tree.node = config_tree.node.set(
keys=["file-install-root"], value=str(rundir)
)
# Artificially set rose to verbose.
event_handler = Reporter(3)
fs_util = FileSystemUtil(event_handler)
popen = RosePopener(event_handler)
# Get an Asyncio loop if one doesn't exist:
# Rose may need an event loop to invoke async interfaces,
# doing this here incase we want to go async in cylc-rose.
# See https://github.com/cylc/cylc-rose/pull/130/files
try:
asyncio.get_event_loop()
except RuntimeError:
asyncio.set_event_loop(asyncio.new_event_loop())
# Process fileinstall.
config_pm = ConfigProcessorsManager(event_handler, popen, fs_util)
config_pm(config_tree, "file")
finally:
os.chdir(startpoint)
return config_tree.node
def test_oserror_has_filename(self):
"""Does what it says."""
rose_popen = RosePopener()
name = "bad-command"
try:
rose_popen.run(name)
except RosePopenError as exc:
ose = FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT),
name)
try:
self.assertEqual(str(ose), exc.stderr)
except AssertionError:
# This is horrible, but refers to a bug in some versions of
# Python 2.6 - https://bugs.python.org/issue32490
err_msg = ("[Errno 2] No such file or directory:"
" 'bad-command': 'bad-command'")
self.assertEqual(err_msg, exc.stderr)
else:
self.fail("should return FileNotFoundError")
def _bash_login_cmd(cmd: List[str]) -> List[str]:
"""Return the given command as a bash login shell command.
This allows users to set env vars.
Example:
>>> HostSelector._bash_login_cmd(["echo", "-n", "Multiple words"])
['bash', '-l', '-c', "echo -n 'Multiple words'"]
"""
return ['bash', '-l', '-c', RosePopener.shlex_join(cmd)]
def __init__(self, config, opts, args, method_paths, reporter=None,
popen=None):
if reporter is None:
self.reporter = Reporter(opts.verbosity - opts.quietness)
else:
self.reporter = reporter
if popen is None:
self.popen = RosePopener(event_handler=self.reporter)
else:
self.popen = popen
self.opts = opts
self.args = args
self.config = config
self.tasks = self.load_tasks()
modules = []
for path in method_paths:
for filename in glob.glob(path + "/*.py"):
modules.append(filename)
self.load_user_comparison_modules(modules)
def __init__(self, event_handler=None, popen=None):
if event_handler is None:
event_handler = Reporter()
self.event_handler = event_handler
if popen is None:
popen = RosePopener(self.event_handler)
self.popen = popen
path = os.path.dirname(
os.path.dirname(sys.modules["metomi.rosie"].__file__))
self.usertools_manager = SchemeHandlersManager([path],
"rosie.usertools",
["get_emails"])
def __init__(self, event_handler=None, popen=None, fs_util=None):
self.event_handler = event_handler
if popen is None:
popen = RosePopener(event_handler)
self.popen = popen
if fs_util is None:
fs_util = FileSystemUtil(event_handler)
self.fs_util = fs_util
path = os.path.dirname(
os.path.dirname(sys.modules["metomi.rose"].__file__))
SchemeHandlersManager.__init__(self, [path], "rose.config_processors",
["process"])
def __init__(self, event_handler=None, popen=None, fs_util=None):
self.event_handler = event_handler
if popen is None:
popen = RosePopener(event_handler)
self.popen = popen
if fs_util is None:
fs_util = FileSystemUtil(event_handler)
self.fs_util = fs_util
path = os.path.dirname(os.path.dirname(sys.modules["rose"].__file__))
SchemeHandlersManager.__init__(
self, [path], ns="rose.loc_handlers", attrs=["parse", "pull"],
can_handle="can_pull")
def __init__(self, event_handler=None, popen=None, fs_util=None):
if event_handler is None:
event_handler = self._dummy
self.event_handler = event_handler
if popen is None:
popen = RosePopener(event_handler)
self.popen = popen
if fs_util is None:
fs_util = FileSystemUtil(event_handler)
self.fs_util = fs_util
self.post_commit_hook = RosieSvnPostCommitHook(
event_handler=event_handler, popen=popen)
def _launch(name, event_handler=None, run_fg=False, *args, **kwargs):
popen = RosePopener(event_handler)
command = popen.get_cmd(name, *args)
kwargs['stdin'] = sys.stdin
if run_fg:
return popen.run(*command, **kwargs)
popen.run_bg(*command, **kwargs)
def __init__(self, event_handler=None, popen=None, fs_util=None,
host_selector=None, **_):
self.event_handler = event_handler
if popen is None:
popen = RosePopener(event_handler)
self.popen = popen
if fs_util is None:
fs_util = FileSystemUtil(event_handler)
self.fs_util = fs_util
if host_selector is None:
host_selector = HostSelector(event_handler, popen)
self.host_selector = host_selector
self.date_time_oper = RoseDateTimeOperator()
def rose_fileinstall(srcdir=None, opts=None, rundir=None):
"""Call Rose Fileinstall.
Args:
srcdir(pathlib.Path):
Search for a ``rose-suite.conf`` file in this location.
rundir (pathlib.Path)
"""
if not rose_config_exists(rundir, opts):
return False
# Load the config tree
config_tree = rose_config_tree_loader(rundir, opts)
if any(i.startswith('file') for i in config_tree.node.value):
try:
startpoint = os.getcwd()
os.chdir(rundir)
except FileNotFoundError as exc:
raise exc
else:
# Carry out imports.
from metomi.rose.config_processor import ConfigProcessorsManager
from metomi.rose.popen import RosePopener
from metomi.rose.reporter import Reporter
from metomi.rose.fs_util import FileSystemUtil
# Update config tree with install location
# NOTE-TO-SELF: value=os.environ["CYLC_WORKFLOW_RUN_DIR"]
config_tree.node = config_tree.node.set(
keys=["file-install-root"], value=str(rundir)
)
# Artificially set rose to verbose.
event_handler = Reporter(3)
fs_util = FileSystemUtil(event_handler)
popen = RosePopener(event_handler)
# Process files
config_pm = ConfigProcessorsManager(event_handler, popen, fs_util)
config_pm(config_tree, "file")
finally:
os.chdir(startpoint)
return config_tree.node
def __init__(
self, prefix, popen=None, prompt_func=None, event_handler=None
):
self.prefix = prefix
root = self._get_conf_value("ws")
if root is None:
raise UndefinedRosiePrefixWS(self.prefix)
if not root.endswith("/"):
root += "/"
self.root = root
urlparse_res = urlparse(self.root)
self.scheme = urlparse_res[0]
self.host = urlparse_res[1]
self.password_orig = None
self.username_orig = None
self.password = None
self.username = None
if popen is None:
popen = RosePopener()
self.popen = popen
self.prompt_func = prompt_func
if event_handler is None:
self.event_handler = Reporter()
else:
self.event_handler = event_handler
res_loc = ResourceLocator.default()
password_stores_str = (
res_loc.default()
.get_conf()
.get_value(
keys=["rosie-id", "prefix-password-store." + self.prefix],
default=self.PASSWORD_STORES_STR,
)
)
for password_store_name in shlex.split(password_stores_str):
password_store_cls = self.PASSWORD_STORE_CLASSES.get(
password_store_name
)
if password_store_cls is not None and password_store_cls.usable():
self.password_store = password_store_cls()
break
else:
self.password_store = None
self.requests_kwargs = {}
self._init_https_params()
def main():
"""Implement the "rose host-select" command."""
opt_parser = RoseOptionParser()
opt_parser.add_my_options("choice", "rank_method", "thresholds", "timeout")
opts, args = opt_parser.parse_args()
report = Reporter(opts.verbosity - opts.quietness)
popen = RosePopener(event_handler=report)
select = HostSelector(event_handler=report, popen=popen)
try:
host_score_list = select(names=args,
rank_method=opts.rank_method,
thresholds=opts.thresholds,
ssh_cmd_timeout=opts.timeout)
except (NoHostError, NoHostSelectError) as exc:
report(exc)
if opts.debug_mode:
traceback.print_exc()
sys.exit(1)
opts.choice = int(opts.choice)
report(choice(host_score_list[0:opts.choice])[0] + "\n", level=0)
def rose_fileinstall(dir_=None, opts=None, dest_root=None):
"""Call Rose Fileinstall.
Args:
dir_(string or pathlib.Path):
Search for a ``rose-suite.conf`` file in this location.
dest_root (string or pathlib.Path)
"""
if not rose_config_exists(dir_):
return False
# Load the config tree
config_tree = rose_config_tree_loader(dir_, opts)
if any(['file' in i for i in config_tree.node.value]):
# Carry out imports.
from metomi.rose.config_processor import ConfigProcessorsManager
from metomi.rose.popen import RosePopener
from metomi.rose.reporter import Reporter
from metomi.rose.fs_util import FileSystemUtil
# Update config tree with install location
# NOTE-TO-SELF: value=os.environ["CYLC_SUITE_RUN_DIR"]
config_tree.node = config_tree.node.set(
keys=["file-install-root"], value=dest_root
)
# Artificially set rose to verbose.
# TODO - either use Cylc Log as event handler, or get Cylc Verbosity
# settings to pass to Rose Reporter.
event_handler = Reporter(3)
fs_util = FileSystemUtil(event_handler)
popen = RosePopener(event_handler)
# Process files
config_pm = ConfigProcessorsManager(event_handler, popen, fs_util)
config_pm(config_tree, "file")
return True
def process(self):
"""Process STEM options into 'rose suite-run' options."""
# Generate options for source trees
repos = {}
repos_with_hosts = {}
if not self.opts.source:
self.opts.source = ['.']
self.opts.project = list()
for i, url in enumerate(self.opts.source):
project, url, base, rev, mirror = self._ascertain_project(url)
self.opts.source[i] = url
self.opts.project.append(project)
# Versions of variables with hostname prepended for working copies
url_host = self._prepend_localhost(url)
base_host = self._prepend_localhost(base)
if project in repos:
repos[project].append(url)
repos_with_hosts[project].append(url_host)
else:
repos[project] = [url]
repos_with_hosts[project] = [url_host]
self._add_define_option('SOURCE_' + project.upper() + '_REV',
'"' + rev + '"')
self._add_define_option('SOURCE_' + project.upper() + '_BASE',
'"' + base + '"')
self._add_define_option(
'HOST_SOURCE_' + project.upper() + '_BASE',
'"' + base_host + '"')
self._add_define_option(
'SOURCE_' + project.upper() + '_MIRROR',
'"' + mirror + '"')
self.reporter(SourceTreeAddedAsBranchEvent(url))
for project, branches in repos.items():
var = 'SOURCE_' + project.upper()
branchstring = RosePopener.list_to_shell_str(branches)
self._add_define_option(var, '"' + branchstring + '"')
for project, branches in repos_with_hosts.items():
var_host = 'HOST_SOURCE_' + project.upper()
branchstring = RosePopener.list_to_shell_str(branches)
self._add_define_option(var_host, '"' + branchstring + '"')
# Generate the variable containing tasks to run
if self.opts.group:
if not self.opts.defines:
self.opts.defines = []
expanded_groups = []
for i in self.opts.group:
expanded_groups.extend(i.split(','))
self.opts.defines.append(SUITE_RC_PREFIX + 'RUN_NAMES=' +
str(expanded_groups))
# Load the config file and return any automatic-options
auto_opts = self._read_auto_opts()
if auto_opts:
automatic_options = auto_opts.split()
for option in automatic_options:
elements = option.split("=")
if len(elements) == 2:
self._add_define_option(elements[0],
'"' + elements[1] + '"')
# Change into the suite directory
if self.opts.conf_dir:
self.reporter(SuiteSelectionEvent(self.opts.conf_dir))
self._check_suite_version(
os.path.join(self.opts.conf_dir, 'rose-suite.conf'))
else:
thissuite = self._this_suite()
self.fs_util.chdir(thissuite)
self.reporter(SuiteSelectionEvent(thissuite))
# Create a default name for the suite; allow override by user
if not self.opts.name:
self.opts.name = self._generate_name()
return self.opts
def select(
self,
names=None,
rank_method=None,
thresholds=None,
ssh_cmd_timeout=None,
):
"""Return a list. Element 0 is most desirable.
Each element of the list is a tuple (host, score).
names: a list of known host groups or host names.
rank_method: the ranking method. Can be one of:
load:1, load:5, load:15 (=load =default), fs:FS and
random. The "load" methods determines the load using the
average load as returned by the "uptime" command divided
by the number of CPUs. The "fs" method determines the load
using the usage in the file system specified by FS. The
"mem" method ranks by highest free memory. The "random"
method ranks everything by random.
thresholds: a list of thresholds which each host must not exceed.
Should be in the format rank_method:value, where
rank_method is one of load:1, load:5, load:15 or fs:FS; and
value is number that must be be exceeded.
ssh_cmd_timeout: timeout of SSH commands to hosts. A float in seconds.
"""
host_names, rank_method, thresholds = self.expand(
names, rank_method, thresholds)
# Load scorers, ranking and thresholds
rank_method_arg = None
if rank_method:
if ":" in rank_method:
rank_method, rank_method_arg = rank_method.split(":", 1)
else:
rank_method = self.RANK_METHOD_DEFAULT
rank_conf = ScorerConf(self.get_scorer(rank_method), rank_method_arg)
self.handle_event(RankMethodEvent(rank_method, rank_method_arg))
threshold_confs = []
if thresholds:
for threshold in thresholds:
method = self.RANK_METHOD_DEFAULT
method_arg = None
value = threshold
if ":" in threshold:
head, value = threshold.rsplit(":", 1)
method = head
if ":" in head:
method, method_arg = head.split(":", 1)
try:
float(value)
except ValueError:
raise ValueError(threshold)
scorer = self.get_scorer(method)
if method_arg is None:
method_arg = scorer.ARG
threshold_conf = ScorerConf(self.get_scorer(method),
method_arg, value)
threshold_confs.append(threshold_conf)
if ssh_cmd_timeout is None:
conf = ResourceLocator.default().get_conf()
ssh_cmd_timeout = float(
conf.get_value(["rose-host-select", "timeout"],
self.SSH_CMD_TIMEOUT))
host_name_list = list(host_names)
host_names = []
for host_name in host_name_list:
if self.is_local_host(host_name):
if self.get_local_host() not in host_names:
host_names.append(self.get_local_host())
else:
host_names.append(host_name)
# Random selection with no thresholds. Return the 1st available host.
if rank_conf.method == self.RANK_METHOD_RANDOM and not threshold_confs:
shuffle(host_names)
for host_name in host_names:
if self.is_local_host(host_name):
return [("localhost", 1)]
command = self.popen.get_cmd("ssh", host_name, "true")
proc = self.popen.run_bg(*command, preexec_fn=os.setpgrp)
time0 = time()
while (proc.poll() is None
and time() - time0 <= ssh_cmd_timeout):
sleep(self.SSH_CMD_POLL_DELAY)
if proc.poll() is None:
os.killpg(proc.pid, signal.SIGTERM)
proc.wait()
self.handle_event(TimedOutHostEvent(host_name))
elif proc.wait():
self.handle_event(
HostSelectCommandFailedEvent(host_name,
proc.returncode))
else:
return [(host_name, 1)]
else:
raise NoHostSelectError()
# ssh to each host to return its score(s).
host_proc_dict = {}
for host_name in sorted(host_names):
# build host-select-client command
command: List[str] = []
# pass through CYLC_VERSION to support use of cylc wrapper script
try:
import cylc.flow
except ModuleNotFoundError:
pass
else:
command.extend([
'env',
f'CYLC_VERSION={cylc.flow.__version__}',
])
cylc_env_name = os.getenv('CYLC_ENV_NAME')
if cylc_env_name:
command.append(f'CYLC_ENV_NAME={cylc_env_name}')
command.extend(self._bash_login_cmd(['rose',
'host-select-client']))
# build list of metrics to obtain for each host
metrics = rank_conf.get_command()
for threshold_conf in threshold_confs:
for metric in threshold_conf.get_command():
if metric not in metrics:
metrics.append(metric)
# convert metrics list to JSON stdin
stdin = '\n***start**\n' + json.dumps(metrics) + '\n**end**\n'
if not self.is_local_host(host_name):
command = [
*self.popen.get_cmd('ssh', host_name),
RosePopener.shlex_join(command)
]
# fire off host-select-client processes
proc = self.popen.run_bg(*command,
stdin=stdin,
preexec_fn=os.setpgrp)
proc.stdin.write(stdin)
proc.stdin.flush()
host_proc_dict[host_name] = (proc, metrics)
# Retrieve score for each host name
host_score_list = []
time0 = time()
while host_proc_dict:
sleep(self.SSH_CMD_POLL_DELAY)
for host_name, (proc, metrics) in list(host_proc_dict.items()):
if proc.poll() is None: # still running
continue
stdout, stderr = proc.communicate()
if proc.returncode:
self.handle_event(
HostSelectCommandFailedEvent(host_name,
proc.returncode, stderr))
host_proc_dict.pop(host_name)
else:
out = _deserialise(metrics, json.loads(stdout.strip()))
host_proc_dict.pop(host_name)
for threshold_conf in threshold_confs:
try:
score = threshold_conf.command_out_parser(
out, metrics)
is_bad = threshold_conf.check_threshold(score)
except ValueError:
is_bad = True
score = None
if is_bad:
self.handle_event(
HostThresholdNotMetEvent(
host_name, threshold_conf, score))
break
else:
try:
score = rank_conf.command_out_parser(out, metrics)
host_score_list.append((host_name, score))
except ValueError:
score = None
self.handle_event(
HostSelectScoreEvent(host_name, score))
if time() - time0 > ssh_cmd_timeout:
break
# Report timed out hosts
for host_name, (proc, _) in sorted(host_proc_dict.items()):
self.handle_event(TimedOutHostEvent(host_name))
os.killpg(proc.pid, signal.SIGTERM)
proc.wait()
if not host_score_list:
raise NoHostSelectError()
host_score_list.sort(key=lambda a: a[1],
reverse=rank_conf.scorer.SIGN < 0)
return host_score_list
class StemRunner(object):
"""Set up options for running a STEM job through Rose."""
def __init__(self, opts, reporter=None, popen=None, fs_util=None):
self.opts = opts
if reporter is None:
self.reporter = Reporter(opts.verbosity - opts.quietness)
else:
self.reporter = reporter
if popen is None:
self.popen = RosePopener(event_handler=self.reporter)
else:
self.popen = popen
if fs_util is None:
self.fs_util = FileSystemUtil(event_handler=self.reporter)
else:
self.fs_util = fs_util
self.host_selector = HostSelector(event_handler=self.reporter,
popen=self.popen)
def _add_define_option(self, var, val):
"""Add a define option passed to the SuiteRunner."""
if self.opts.defines:
self.opts.defines.append(SUITE_RC_PREFIX + var + '=' + val)
else:
self.opts.defines = [SUITE_RC_PREFIX + var + '=' + val]
self.reporter(ConfigVariableSetEvent(var, val))
return
def _get_base_dir(self, item):
"""Given a source tree return the following from 'fcm loc-layout':
* url
* sub_tree
* peg_rev
* root
* project
"""
ret_code, output, stderr = self.popen.run('fcm', 'loc-layout', item)
output = output.decode()
if ret_code != 0:
raise ProjectNotFoundException(item, stderr)
ret = {}
for line in output.splitlines():
if ":" not in line:
continue
key, value = line.split(":", 1)
if key:
if value:
ret[key] = value.strip()
return ret
def _get_project_from_url(self, source_dict):
"""Run 'fcm keyword-print' to work out the project name."""
repo = source_dict['root']
if source_dict['project']:
repo += '/' + source_dict['project']
kpoutput = self.popen.run('fcm', 'kp', source_dict['url'])[1]
project = None
for line in kpoutput.splitlines():
if line.rstrip().endswith(repo.encode('UTF-8')):
kpresult = re.search(r'^location{primary}\[(.*)\]',
line.decode())
if kpresult:
project = kpresult.group(1)
break
return project
def _deduce_mirror(self, source_dict, project):
"""Deduce the mirror location of this source tree."""
# Root location for project
proj_root = source_dict['root'] + '/' + source_dict['project']
# Swap project to mirror
project = re.sub(r'\.x$', r'.xm', project)
mirror_repo = "fcm:" + project
# Generate mirror location
mirror = re.sub(proj_root, mirror_repo, source_dict['url'])
# Remove any sub-tree
mirror = re.sub(source_dict['sub_tree'], r'', mirror)
mirror = re.sub(r'/@', r'@', mirror)
# Add forwards slash after .xm if missing
if '.xm/' not in mirror:
mirror = re.sub(r'\.xm', r'.xm/', mirror)
return mirror
def _ascertain_project(self, item):
"""Set the project name and top-level from 'fcm loc-layout'.
Returns:
* project name
* top-level location of the source tree with revision number
* top-level location of the source tree without revision number
* revision number
"""
project = None
try:
project, item = item.split("=", 1)
except ValueError:
pass
if re.search(r'^\.', item):
item = os.path.abspath(os.path.join(os.getcwd(), item))
if project is not None:
print("[WARN] Forcing project for '{0}' to be '{1}'".format(
item, project))
return project, item, item, '', ''
source_dict = self._get_base_dir(item)
project = self._get_project_from_url(source_dict)
if not project:
raise ProjectNotFoundException(item)
mirror = self._deduce_mirror(source_dict, project)
if 'peg_rev' in source_dict and '@' in item:
revision = '@' + source_dict['peg_rev']
base = re.sub(r'@.*', r'', item)
else:
revision = ''
base = item
# Remove subtree from base and item
if 'sub_tree' in source_dict:
item = re.sub(r'(.*)%s/?$' % (source_dict['sub_tree']),
r'\1',
item,
count=1)
base = re.sub(r'(.*)%s/?$' % (source_dict['sub_tree']),
r'\1',
base,
count=1)
# Remove trailing forwards-slash
item = re.sub(r'/$', r'', item)
base = re.sub(r'/$', r'', base)
# Remove anything after a point
project = re.sub(r'\..*', r'', project)
return project, item, base, revision, mirror
def _generate_name(self):
"""Generate a suite name from the name of the first source tree."""
try:
basedir = self._ascertain_project(os.getcwd())[1]
except ProjectNotFoundException:
if self.opts.conf_dir:
basedir = os.path.abspath(self.opts.conf_dir)
else:
basedir = os.getcwd()
name = os.path.basename(basedir)
self.reporter(NameSetEvent(name))
return name
def _this_suite(self):
"""Find the location of the suite in the first source tree."""
# Get base of first source
basedir = ''
if self.opts.source:
basedir = self.opts.source[0]
else:
basedir = self._ascertain_project(os.getcwd())[1]
suitedir = os.path.join(basedir, DEFAULT_TEST_DIR)
suitefile = os.path.join(suitedir, "rose-suite.conf")
if not os.path.isfile(suitefile):
raise RoseSuiteConfNotFoundException(suitedir)
self._check_suite_version(suitefile)
return suitedir
def _read_auto_opts(self):
"""Read the site metomi.rose.conf file."""
return ResourceLocator.default().get_conf().get_value(
["rose-stem", "automatic-options"])
def _check_suite_version(self, fname):
"""Check the suite is compatible with this version of metomi.rose-stem.
"""
if not os.path.isfile(fname):
raise RoseSuiteConfNotFoundException(os.path.dirname(fname))
config = metomi.rose.config.load(fname)
suite_rose_stem_version = config.get(['ROSE_STEM_VERSION'])
if suite_rose_stem_version:
suite_rose_stem_version = int(suite_rose_stem_version.value)
else:
suite_rose_stem_version = None
if not suite_rose_stem_version == ROSE_STEM_VERSION:
raise RoseStemVersionException(suite_rose_stem_version)
def _prepend_localhost(self, url):
"""Prepend the local hostname to urls which do not point to repository
locations."""
if ':' not in url or url.split(
':', 1)[0] not in ['svn', 'fcm', 'http', 'https', 'svn+ssh']:
url = self.host_selector.get_local_host() + ':' + url
return url
def process(self):
"""Process STEM options into 'rose suite-run' options."""
# Generate options for source trees
repos = {}
repos_with_hosts = {}
if not self.opts.source:
self.opts.source = ['.']
self.opts.project = list()
for i, url in enumerate(self.opts.source):
project, url, base, rev, mirror = self._ascertain_project(url)
self.opts.source[i] = url
self.opts.project.append(project)
# Versions of variables with hostname prepended for working copies
url_host = self._prepend_localhost(url)
base_host = self._prepend_localhost(base)
if project in repos:
repos[project].append(url)
repos_with_hosts[project].append(url_host)
else:
repos[project] = [url]
repos_with_hosts[project] = [url_host]
self._add_define_option('SOURCE_' + project.upper() + '_REV',
'"' + rev + '"')
self._add_define_option('SOURCE_' + project.upper() + '_BASE',
'"' + base + '"')
self._add_define_option(
'HOST_SOURCE_' + project.upper() + '_BASE',
'"' + base_host + '"')
self._add_define_option(
'SOURCE_' + project.upper() + '_MIRROR',
'"' + mirror + '"')
self.reporter(SourceTreeAddedAsBranchEvent(url))
for project, branches in repos.items():
var = 'SOURCE_' + project.upper()
branchstring = RosePopener.list_to_shell_str(branches)
self._add_define_option(var, '"' + branchstring + '"')
for project, branches in repos_with_hosts.items():
var_host = 'HOST_SOURCE_' + project.upper()
branchstring = RosePopener.list_to_shell_str(branches)
self._add_define_option(var_host, '"' + branchstring + '"')
# Generate the variable containing tasks to run
if self.opts.group:
if not self.opts.defines:
self.opts.defines = []
expanded_groups = []
for i in self.opts.group:
expanded_groups.extend(i.split(','))
self.opts.defines.append(SUITE_RC_PREFIX + 'RUN_NAMES=' +
str(expanded_groups))
# Load the config file and return any automatic-options
auto_opts = self._read_auto_opts()
if auto_opts:
automatic_options = auto_opts.split()
for option in automatic_options:
elements = option.split("=")
if len(elements) == 2:
self._add_define_option(elements[0],
'"' + elements[1] + '"')
# Change into the suite directory
if self.opts.conf_dir:
self.reporter(SuiteSelectionEvent(self.opts.conf_dir))
self._check_suite_version(
os.path.join(self.opts.conf_dir, 'rose-suite.conf'))
else:
thissuite = self._this_suite()
self.fs_util.chdir(thissuite)
self.reporter(SuiteSelectionEvent(thissuite))
# Create a default name for the suite; allow override by user
if not self.opts.name:
self.opts.name = self._generate_name()
return self.opts
def main():
"""Implement the "rose host-select" command."""
opt_parser = RoseOptionParser(
usage='rose host-select [OPTIONS] [GROUP/HOST ...]',
description='''
Select a host from a set of groups or names by load, by free memory
or by random.
Print the selected host name.
''',
epilog='''
RANKING METHODS IN DETAIL (--rank-method):
`load`
Rank by average load as reported by `uptime` divided by
number of virtual processors.
If `METHOD-ARG` is specified, it must be `1`, `5` or `15`.
The default is to use the 15 minute load.
`fs`
Rank by % usage of a file system as reported by `df`.
`METHOD-ARG` must be a valid file system in all the given
hosts and host groups. The default is to use the `~`
directory.
`mem`
Rank by largest amount of free memory. Uses `free -m` to
return memory in Mb
`random`
No ranking is used.
CONFIGURATION
The command reads its settings from the `[rose-host-select]` section in
the Rose configuration. All settings are optional. Type
`rose config rose-host-select` to print settings.
Valid settings are:
default = GROUP/HOST ...
The default arguments to use for this command.
group{NAME} = GROUP/HOST ...
Declare a named group of hosts.
method{NAME} = METHOD[:METHOD-ARG]
Declare the default ranking method for a group of hosts.
thresholds{NAME} = [METHOD[:METHOD-ARG]:]VALUE ...
Declare the default threshold(s) for a group of hosts.
timeout = FLOAT
Set the timeout in seconds of SSH commands to hosts.
(default=10.0)
'''
)
opt_parser.add_my_options("choice", "rank_method", "thresholds", "timeout")
opt_parser.modify_option(
'timeout',
help='Set the timeout in seconds of SSH commands to hosts.',
)
opts, args = opt_parser.parse_args()
report = Reporter(opts.verbosity - opts.quietness)
popen = RosePopener(event_handler=report)
select = HostSelector(event_handler=report, popen=popen)
try:
host_score_list = select(
names=args,
rank_method=opts.rank_method,
thresholds=opts.thresholds,
ssh_cmd_timeout=opts.timeout,
)
except (NoHostError, NoHostSelectError) as exc:
report(exc)
if opts.debug_mode:
traceback.print_exc()
sys.exit(1)
opts.choice = int(opts.choice)
report(choice(host_score_list[0 : opts.choice])[0] + "\n", level=0)
class Analyse:
"""A comparison engine for Rose."""
def __init__(self,
config,
opts,
args,
method_paths,
reporter=None,
popen=None):
if reporter is None:
self.reporter = Reporter(opts.verbosity - opts.quietness)
else:
self.reporter = reporter
if popen is None:
self.popen = RosePopener(event_handler=self.reporter)
else:
self.popen = popen
self.opts = opts
self.args = args
self.config = config
self.tasks = self.load_tasks()
modules = []
for path in method_paths:
for filename in glob.glob(path + "/*.py"):
modules.append(filename)
self.load_user_comparison_modules(modules)
def analyse(self):
"""Perform comparisons given a list of tasks."""
ret_code = 0
for task in self.tasks:
if self.check_extract(task):
# Internal AnalysisEngine extract+comparison test
# Extract data from results and from kgoX
task = self.do_extract(task, "result")
for i in range(task.numkgofiles):
var = "kgo" + str(i + 1)
task = self.do_extract(task, var)
task = self.do_comparison(task)
else:
# External program(s) doing test
# Command to run
command = task.extract
result = re.search(r"\$file", command)
# If the command contains $file, it is run separately for both
# the KGO and the result files
if result:
# Replace $file token with resultfile or kgofile
resultcommand = self._expand_tokens(
command, task, "result")
kgocommand = self._expand_tokens(command, task, "kgo1")
# Run the command on the resultfile
task.resultdata = self._run_command(resultcommand)
# Run the command on the KGO file
task.kgo1data = self._run_command(kgocommand)
else:
# The command works on both files at the same time
# Replace tokens $kgofile and $resultfile for actual values
command = self._expand_tokens(command, task)
# Run the command
task.resultdata = self._run_command(command)
# Run the comparison
task = self.do_comparison(task)
self.reporter(TaskCompletionEvent(task),
prefix="[%s]" % (task.userstatus))
if task.numericstatus != PASS:
ret_code += 1
return ret_code, self.tasks
def check_extract(self, task):
"""Check if an extract name is present in a user method."""
for _, class_name, _, _ in self.user_methods:
if task.extract == class_name:
return True
return False
def do_comparison(self, task):
"""Run the comparison."""
for module_name, class_name, _, _ in self.user_methods:
if task.comparison == class_name:
for module in self.modules:
if module.__name__ == module_name:
comparison_inst = getattr(module, class_name)()
getattr(comparison_inst, "run")(task)
return task
def do_extract(self, task, var):
"""Extract the specified data."""
for module_name, class_name, _, _ in self.user_methods:
if task.extract == class_name:
for module in self.modules:
if module.__name__ == module_name:
extract_inst = getattr(module, class_name)()
getattr(extract_inst, "run")(task, var)
return task
def _run_command(self, command):
"""Run an external command using metomi.rose.popen."""
output = self.popen.run_ok(command, shell=True)[0]
output = "".join(output).splitlines()
return output
def _expand_tokens(self, inputstring, task, var=None):
"""Expands tokens $resultfile, $file and $kgoXfile."""
filename = ''
if var:
filename = getattr(task, var + "file")
expansions = {'resultfile': task.resultfile, 'file': filename}
for i in range(1, task.numkgofiles + 1):
key = "kgo" + str(i) + "file"
value = getattr(task, key)
expansions[key] = value
inputstring = inputstring.format(**expansions)
return inputstring
def _find_file(self, var, task):
"""Finds a file given a variable name containing the filename.
Given a variable name and task object, this returns the filename it
points to, including expanding any * characters with glob.
"""
filevar = var + "file"
if hasattr(task, filevar):
configvar = var + "fileconfig"
setattr(task, configvar, getattr(task, filevar))
filenames = glob.glob(env_var_process(getattr(task, filevar)))
if len(filenames) > 0:
setattr(task, filevar, os.path.abspath(filenames[0]))
return task
def load_tasks(self):
"""Loads AnalysisTasks from files.
Given a list of files, return AnalysisTasks generated from those files.
This also expands environment variables in filenames, but saves the
original contents for use when writing out config files
"""
tasks = []
for task in self.config.value.keys():
if task == "env":
continue
if task.startswith("file:"):
continue
newtask = AnalysisTask()
newtask.name = task
value = self.config.get_value([task, "resultfile"])
# If the task is ignored, this will be None, so continue
# on to the next task
if value is None:
continue
if "{}" in value:
newtask.resultfile = value.replace("{}", self.args[0])
else:
newtask.resultfile = value
newtask = self._find_file("result", newtask)
newtask.extract = self.config.get_value([task, "extract"])
result = re.search(r":", newtask.extract)
if result:
newtask.subextract = ":".join(newtask.extract.split(":")[1:])
newtask.extract = newtask.extract.split(":")[0]
newtask.comparison = self.config.get_value([task, "comparison"])
newtask.tolerance = env_var_process(
self.config.get_value([task, "tolerance"]))
newtask.warnonfail = self.config.get_value([task, "warnonfail"
]) in ["yes", "true"]
# Allow for multiple KGO, e.g. kgo1file, kgo2file, for
# statistical comparisons of results
newtask.numkgofiles = 0
for i in range(1, MAX_KGO_FILES):
kgovar = "kgo" + str(i)
kgofilevar = kgovar + "file"
if self.config.get([task, kgofilevar]):
value = self.config.get([task, kgofilevar])[:]
if "{}" in value:
setattr(
newtask,
kgofilevar,
value.replace("{}", self.args[0]),
)
else:
setattr(newtask, kgofilevar, value)
newtask.numkgofiles += 1
newtask = self._find_file(kgovar, newtask)
else:
break
tasks.append(newtask)
return tasks
def load_user_comparison_modules(self, files):
"""Import comparison modules and store them."""
modules = []
for filename in files:
directory = os.path.dirname(filename)
if not directory.endswith(
USRCOMPARISON_DIRNAME) or not filename.endswith(
USRCOMPARISON_EXT):
continue
comparison_name = os.path.basename(filename).rpartition(
USRCOMPARISON_EXT)[0]
sys.path.insert(0, os.path.abspath(directory))
try:
modules.append(__import__(comparison_name))
except ImportError as exc:
self.reporter(exc)
sys.path.pop(0)
modules.sort(key=str)
self.modules = modules
user_methods = []
for module in modules:
comparison_name = module.__name__
contents = inspect.getmembers(module, inspect.isclass)
for obj_name, obj in contents:
att_name = "run"
if hasattr(obj, att_name) and callable(getattr(obj, att_name)):
doc_string = obj.__doc__
user_methods.append(
(comparison_name, obj_name, att_name, doc_string))
self.user_methods = user_methods
return user_methods
def write_config(self, filename, tasks):
"""Write an analysis config file based on a list of tasks provided"""
config = metomi.rose.config.ConfigNode()
for task in tasks:
sectionname = task.name
if task.resultfileconfig:
config.set([sectionname, "resultfile"], task.resultfileconfig)
for i in range(1, task.numkgofiles + 1):
origvar = "kgo" + str(i) + "fileconfig"
valvar = "kgo" + str(i) + "file"
if hasattr(task, origvar):
config.set([sectionname, valvar], getattr(task, origvar))
if task.extract:
config.set([sectionname, "extract"], task.extract)
if task.subextract:
config.set(
[sectionname, "extract"],
task.extract + ":" + task.subextract,
)
if task.comparison:
config.set([sectionname, "comparison"], task.comparison)
if task.tolerance:
config.set([sectionname, "tolerance"], task.tolerance)
if task.warnonfail:
config.set([sectionname, "warnonfail"], "true")
metomi.rose.config.dump(config, filename)
'cylc.sphinx_ext.diff_selection',
'cylc.sphinx_ext.grid_table',
'cylc.sphinx_ext.hieroglyph_patch',
'cylc.sphinx_ext.hieroglyph_theme_addons',
'cylc.sphinx_ext.minicylc',
'cylc.sphinx_ext.practical',
'cylc.sphinx_ext.rtd_theme_addons',
'cylc.sphinx_ext.sub_lang',
]
# Select best available SVG image converter.
for svg_converter, extension in [('rsvg', 'sphinxcontrib.rsvgconverter'),
('inkscape',
'sphinxcontrib.inkscapeconverter')]:
try:
assert RosePopener.which(svg_converter)
__import__(extension)
except (AssertionError, ImportError):
# converter or extension not available
pass
else:
extensions.append(extension)
break
else:
# no extensions or converters available, fall-back to default
# vector graphics will be converted to bitmaps in all documents
extensions.append('sphinx.ext.imgconverter')
# Slide (hieroglyph) settings.
slide_theme = 'single-level'
slide_link_to_html = True
def __init__(self, event_handler=None):
self.event_handler = event_handler
self.popen = RosePopener(self.event_handler)
self.suite_engine_proc = SuiteEngineProcessor.get_processor(
event_handler=self.event_handler, popen=self.popen)
