def __init__(self,
event_handler=None,
popen=None,
fs_util=None,
force_mode=False):
if event_handler is None:
event_handler = self._dummy
self.event_handler = event_handler
if popen is None:
popen = RosePopener(event_handler)
self.popen = popen
if fs_util is None:
fs_util = FileSystemUtil(event_handler)
self.fs_util = fs_util
self.force_mode = force_mode
self._work_dir = None
atexit.register(self._delete_work_dir)
self.subversion_servers_conf = None
subversion_servers_conf = os.getenv("ROSIE_SUBVERSION_SERVERS_CONF")
if subversion_servers_conf:
self.subversion_servers_conf = subversion_servers_conf
else:
subversion_servers_conf = os.path.expanduser(
self.SUBVERSION_SERVERS_CONF)
if os.path.exists(subversion_servers_conf):
self.subversion_servers_conf = subversion_servers_conf
def __init__(self,
prefixes=None,
prompt_func=None,
popen=None,
event_handler=None):
if not event_handler:
event_handler = Reporter()
if not popen:
popen = RosePopener(event_handler=event_handler)
self.event_handler = event_handler
self.popen = popen
self.prompt_func = prompt_func
self.prefixes = []
self.unreachable_prefixes = []
self.auth_managers = {}
conf = ResourceLocator.default().get_conf()
conf_rosie_id = conf.get(["rosie-id"], no_ignore=True)
if conf_rosie_id is None:
raise RosieWSClientConfError()
for key, node in conf_rosie_id.value.items():
if node.is_ignored() or not key.startswith("prefix-ws."):
continue
prefix = key.replace("prefix-ws.", "")
self.auth_managers[prefix] = RosieWSClientAuthManager(
prefix, popen=self.popen, prompt_func=self.prompt_func)
if not prefixes:
prefixes_str = conf_rosie_id.get_value(["prefixes-ws-default"])
if prefixes_str:
prefixes = shlex.split(prefixes_str)
else:
prefixes = sorted(self.auth_managers.keys())
self.set_prefixes(prefixes)
def __init__(self, event_handler=None, popen=None):
self.event_handler = event_handler
if popen is None:
popen = RosePopener(event_handler=event_handler)
self.popen = popen
self.scorers = {}
self.local_host_strs = None
def main():
"""Implement "rose suite-hook" command."""
opt_parser = RoseOptionParser()
opt_parser.add_my_options("mail_cc", "mail", "retrieve_job_logs",
"shutdown")
opts, args = opt_parser.parse_args()
for key in ["mail_cc"]:
values = []
if getattr(opts, key):
for value in getattr(opts, key):
values.extend(value.split(","))
setattr(opts, key, values)
report = Reporter(opts.verbosity - opts.quietness - 1) # Reduced default
popen = RosePopener(event_handler=report)
suite_engine_proc = SuiteEngineProcessor.get_processor(
event_handler=report, popen=popen)
args = suite_engine_proc.process_suite_hook_args(*args, **vars(opts))
hook = RoseSuiteHook(event_handler=report,
popen=popen,
suite_engine_proc=suite_engine_proc)
hook(*args,
should_mail=opts.mail,
mail_cc_list=opts.mail_cc,
should_shutdown=opts.shutdown,
should_retrieve_job_logs=opts.retrieve_job_logs)
def __init__(
self,
event_handler=None,
popen=None,
config_pm=None,
fs_util=None,
suite_engine_proc=None,
):
if not self.CONF_NAME:
self.CONF_NAME = self.NAME
self.event_handler = event_handler
if popen is None:
popen = RosePopener(event_handler)
self.popen = popen
if fs_util is None:
fs_util = FileSystemUtil(event_handler)
self.fs_util = fs_util
if config_pm is None:
config_pm = ConfigProcessorsManager(event_handler, popen, fs_util)
self.config_pm = config_pm
if suite_engine_proc is None:
suite_engine_proc = SuiteEngineProcessor.get_processor(
event_handler=event_handler, popen=popen, fs_util=fs_util
)
self.suite_engine_proc = suite_engine_proc
self.conf_tree_loader = ConfigTreeLoader()
def _launch(name, event_handler=None, run_fg=False, *args, **kwargs):
popen = RosePopener(event_handler)
command = popen.get_cmd(name, *args)
kwargs['stdin'] = sys.stdin
if run_fg:
return popen.run(*command, **kwargs)
popen.run_bg(*command, **kwargs)
def __init__(self, event_handler=None, popen=None, suite_engine_proc=None):
self.event_handler = event_handler
if popen is None:
popen = RosePopener(event_handler)
self.popen = popen
if suite_engine_proc is None:
suite_engine_proc = SuiteEngineProcessor.get_processor(
event_handler=event_handler, popen=popen)
self.suite_engine_proc = suite_engine_proc
def __init__(self, event_handler=None, popen=None):
if event_handler is None:
event_handler = Reporter()
self.event_handler = event_handler
if popen is None:
popen = RosePopener(self.event_handler)
self.popen = popen
self.path = os.path.dirname(
os.path.dirname(sys.modules["metomi.rosie"].__file__))
def rose_fileinstall(srcdir=None, opts=None, rundir=None):
"""Call Rose Fileinstall.
Args:
srcdir(pathlib.Path):
Search for a ``rose-suite.conf`` file in this location.
rundir (pathlib.Path)
"""
if not rose_config_exists(rundir, opts):
return False
# Load the config tree
config_tree = rose_config_tree_loader(rundir, opts)
if any(i.startswith('file') for i in config_tree.node.value):
try:
startpoint = os.getcwd()
os.chdir(rundir)
except FileNotFoundError as exc:
raise exc
else:
# Carry out imports.
import asyncio
from metomi.rose.config_processor import ConfigProcessorsManager
from metomi.rose.popen import RosePopener
from metomi.rose.reporter import Reporter
from metomi.rose.fs_util import FileSystemUtil
# Update config tree with install location
# NOTE-TO-SELF: value=os.environ["CYLC_WORKFLOW_RUN_DIR"]
config_tree.node = config_tree.node.set(
keys=["file-install-root"], value=str(rundir)
)
# Artificially set rose to verbose.
event_handler = Reporter(3)
fs_util = FileSystemUtil(event_handler)
popen = RosePopener(event_handler)
# Get an Asyncio loop if one doesn't exist:
# Rose may need an event loop to invoke async interfaces,
# doing this here incase we want to go async in cylc-rose.
# See https://github.com/cylc/cylc-rose/pull/130/files
try:
asyncio.get_event_loop()
except RuntimeError:
asyncio.set_event_loop(asyncio.new_event_loop())
# Process fileinstall.
config_pm = ConfigProcessorsManager(event_handler, popen, fs_util)
config_pm(config_tree, "file")
finally:
os.chdir(startpoint)
return config_tree.node
def __init__(self, event_handler=None, popen=None):
if event_handler is None:
event_handler = Reporter()
self.event_handler = event_handler
if popen is None:
popen = RosePopener(self.event_handler)
self.popen = popen
path = os.path.dirname(
os.path.dirname(sys.modules["metomi.rosie"].__file__))
self.usertools_manager = SchemeHandlersManager([path],
"rosie.usertools",
["get_emails"])
def __init__(self, event_handler=None, popen=None, fs_util=None):
if event_handler is None:
event_handler = self._dummy
self.event_handler = event_handler
if popen is None:
popen = RosePopener(event_handler)
self.popen = popen
if fs_util is None:
fs_util = FileSystemUtil(event_handler)
self.fs_util = fs_util
self.post_commit_hook = RosieSvnPostCommitHook(
event_handler=event_handler, popen=popen)
def __init__(self, event_handler=None, popen=None, fs_util=None):
self.event_handler = event_handler
if popen is None:
popen = RosePopener(event_handler)
self.popen = popen
if fs_util is None:
fs_util = FileSystemUtil(event_handler)
self.fs_util = fs_util
path = os.path.dirname(
os.path.dirname(sys.modules["metomi.rose"].__file__))
SchemeHandlersManager.__init__(self, [path], "rose.config_processors",
["process"])
def __init__(self, event_handler=None, popen=None, fs_util=None):
self.event_handler = event_handler
if popen is None:
popen = RosePopener(event_handler)
self.popen = popen
if fs_util is None:
fs_util = FileSystemUtil(event_handler)
self.fs_util = fs_util
path = os.path.dirname(os.path.dirname(sys.modules["rose"].__file__))
SchemeHandlersManager.__init__(
self, [path], ns="rose.loc_handlers", attrs=["parse", "pull"],
can_handle="can_pull")
def __init__(self, event_handler=None, popen=None, fs_util=None,
host_selector=None, **_):
self.event_handler = event_handler
if popen is None:
popen = RosePopener(event_handler)
self.popen = popen
if fs_util is None:
fs_util = FileSystemUtil(event_handler)
self.fs_util = fs_util
if host_selector is None:
host_selector = HostSelector(event_handler, popen)
self.host_selector = host_selector
self.date_time_oper = RoseDateTimeOperator()
def write_source_vc_info(run_source_dir, output=None, popen=None):
"""Write version control information of sources used in run time.
run_source_dir -- The source directory we are interested in.
output -- An open file handle or a string containing a writable path.
If not specified, use sys.stdout.
popen -- A metomi.rose.popen.RosePopener instance for running vc commands.
If not specified, use a new local instance.
"""
if popen is None:
popen = RosePopener()
if output is None:
handle = sys.stdout
elif hasattr(output, "write"):
handle = output
else:
handle = open(output, "wb")
msg = "%s\n" % run_source_dir
write_safely(msg, handle)
environ = dict(os.environ)
environ["LANG"] = "C"
for vcs, args_list in [
(
"svn",
[
["info", "--non-interactive"],
["status", "--non-interactive"],
["diff", "--internal-diff", "--non-interactive"],
],
),
("git", [["describe"], ["status"], ["diff"]]),
]:
if not popen.which(vcs):
continue
cwd = os.getcwd()
os.chdir(run_source_dir)
try:
for args in args_list:
cmd = [vcs] + args
ret_code, out, _ = popen.run(*cmd, env=environ)
if out:
write_safely(("#" * 80 + "\n"), handle)
write_safely(("# %s\n" % popen.list_to_shell_str(cmd)),
handle)
write_safely(("#" * 80 + "\n"), handle)
write_safely(out, handle)
if ret_code: # If cmd fails once, it will likely fail again
break
finally:
os.chdir(cwd)
def __init__(self, opts, reporter=None, popen=None, fs_util=None):
self.opts = opts
if reporter is None:
self.reporter = Reporter(opts.verbosity - opts.quietness)
else:
self.reporter = reporter
if popen is None:
self.popen = RosePopener(event_handler=self.reporter)
else:
self.popen = popen
if fs_util is None:
self.fs_util = FileSystemUtil(event_handler=self.reporter)
else:
self.fs_util = fs_util
self.host_selector = HostSelector(event_handler=self.reporter,
popen=self.popen)
def rose_fileinstall(srcdir=None, opts=None, rundir=None):
"""Call Rose Fileinstall.
Args:
srcdir(pathlib.Path):
Search for a ``rose-suite.conf`` file in this location.
rundir (pathlib.Path)
"""
if not rose_config_exists(rundir, opts):
return False
# Load the config tree
config_tree = rose_config_tree_loader(rundir, opts)
if any(i.startswith('file') for i in config_tree.node.value):
try:
startpoint = os.getcwd()
os.chdir(rundir)
except FileNotFoundError as exc:
raise exc
else:
# Carry out imports.
from metomi.rose.config_processor import ConfigProcessorsManager
from metomi.rose.popen import RosePopener
from metomi.rose.reporter import Reporter
from metomi.rose.fs_util import FileSystemUtil
# Update config tree with install location
# NOTE-TO-SELF: value=os.environ["CYLC_WORKFLOW_RUN_DIR"]
config_tree.node = config_tree.node.set(
keys=["file-install-root"], value=str(rundir)
)
# Artificially set rose to verbose.
event_handler = Reporter(3)
fs_util = FileSystemUtil(event_handler)
popen = RosePopener(event_handler)
# Process files
config_pm = ConfigProcessorsManager(event_handler, popen, fs_util)
config_pm(config_tree, "file")
finally:
os.chdir(startpoint)
return config_tree.node
def __init__(
self, prefix, popen=None, prompt_func=None, event_handler=None
):
self.prefix = prefix
root = self._get_conf_value("ws")
if root is None:
raise UndefinedRosiePrefixWS(self.prefix)
if not root.endswith("/"):
root += "/"
self.root = root
urlparse_res = urlparse(self.root)
self.scheme = urlparse_res[0]
self.host = urlparse_res[1]
self.password_orig = None
self.username_orig = None
self.password = None
self.username = None
if popen is None:
popen = RosePopener()
self.popen = popen
self.prompt_func = prompt_func
if event_handler is None:
self.event_handler = Reporter()
else:
self.event_handler = event_handler
res_loc = ResourceLocator.default()
password_stores_str = (
res_loc.default()
.get_conf()
.get_value(
keys=["rosie-id", "prefix-password-store." + self.prefix],
default=self.PASSWORD_STORES_STR,
)
)
for password_store_name in shlex.split(password_stores_str):
password_store_cls = self.PASSWORD_STORE_CLASSES.get(
password_store_name
)
if password_store_cls is not None and password_store_cls.usable():
self.password_store = password_store_cls()
break
else:
self.password_store = None
self.requests_kwargs = {}
self._init_https_params()
def test_oserror_has_filename(self):
"""Does what it says."""
rose_popen = RosePopener()
name = "bad-command"
try:
rose_popen.run(name)
except RosePopenError as exc:
ose = FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT),
name)
try:
self.assertEqual(str(ose), exc.stderr)
except AssertionError:
# This is horrible, but refers to a bug in some versions of
# Python 2.6 - https://bugs.python.org/issue32490
err_msg = ("[Errno 2] No such file or directory:"
" 'bad-command': 'bad-command'")
self.assertEqual(err_msg, exc.stderr)
else:
self.fail("should return FileNotFoundError")
def __init__(self, config, opts, args, method_paths, reporter=None,
popen=None):
if reporter is None:
self.reporter = Reporter(opts.verbosity - opts.quietness)
else:
self.reporter = reporter
if popen is None:
self.popen = RosePopener(event_handler=self.reporter)
else:
self.popen = popen
self.opts = opts
self.args = args
self.config = config
self.tasks = self.load_tasks()
modules = []
for path in method_paths:
for filename in glob.glob(path + "/*.py"):
modules.append(filename)
self.load_user_comparison_modules(modules)
def main():
"""Implement the "rose host-select" command."""
opt_parser = RoseOptionParser()
opt_parser.add_my_options("choice", "rank_method", "thresholds", "timeout")
opts, args = opt_parser.parse_args()
report = Reporter(opts.verbosity - opts.quietness)
popen = RosePopener(event_handler=report)
select = HostSelector(event_handler=report, popen=popen)
try:
host_score_list = select(names=args,
rank_method=opts.rank_method,
thresholds=opts.thresholds,
ssh_cmd_timeout=opts.timeout)
except (NoHostError, NoHostSelectError) as exc:
report(exc)
if opts.debug_mode:
traceback.print_exc()
sys.exit(1)
opts.choice = int(opts.choice)
report(choice(host_score_list[0:opts.choice])[0] + "\n", level=0)
def rose_fileinstall(dir_=None, opts=None, dest_root=None):
"""Call Rose Fileinstall.
Args:
dir_(string or pathlib.Path):
Search for a ``rose-suite.conf`` file in this location.
dest_root (string or pathlib.Path)
"""
if not rose_config_exists(dir_):
return False
# Load the config tree
config_tree = rose_config_tree_loader(dir_, opts)
if any(['file' in i for i in config_tree.node.value]):
# Carry out imports.
from metomi.rose.config_processor import ConfigProcessorsManager
from metomi.rose.popen import RosePopener
from metomi.rose.reporter import Reporter
from metomi.rose.fs_util import FileSystemUtil
# Update config tree with install location
# NOTE-TO-SELF: value=os.environ["CYLC_SUITE_RUN_DIR"]
config_tree.node = config_tree.node.set(
keys=["file-install-root"], value=dest_root
)
# Artificially set rose to verbose.
# TODO - either use Cylc Log as event handler, or get Cylc Verbosity
# settings to pass to Rose Reporter.
event_handler = Reporter(3)
fs_util = FileSystemUtil(event_handler)
popen = RosePopener(event_handler)
# Process files
config_pm = ConfigProcessorsManager(event_handler, popen, fs_util)
config_pm(config_tree, "file")
return True
def __init__(self, event_handler=None):
self.event_handler = event_handler
self.popen = RosePopener(self.event_handler)
self.suite_engine_proc = SuiteEngineProcessor.get_processor(
event_handler=self.event_handler, popen=self.popen)
def main():
"""Implement the "rose host-select" command."""
opt_parser = RoseOptionParser(
usage='rose host-select [OPTIONS] [GROUP/HOST ...]',
description='''
Select a host from a set of groups or names by load, by free memory
or by random.
Print the selected host name.
''',
epilog='''
RANKING METHODS IN DETAIL (--rank-method):
`load`
Rank by average load as reported by `uptime` divided by
number of virtual processors.
If `METHOD-ARG` is specified, it must be `1`, `5` or `15`.
The default is to use the 15 minute load.
`fs`
Rank by % usage of a file system as reported by `df`.
`METHOD-ARG` must be a valid file system in all the given
hosts and host groups. The default is to use the `~`
directory.
`mem`
Rank by largest amount of free memory. Uses `free -m` to
return memory in Mb
`random`
No ranking is used.
CONFIGURATION
The command reads its settings from the `[rose-host-select]` section in
the Rose configuration. All settings are optional. Type
`rose config rose-host-select` to print settings.
Valid settings are:
default = GROUP/HOST ...
The default arguments to use for this command.
group{NAME} = GROUP/HOST ...
Declare a named group of hosts.
method{NAME} = METHOD[:METHOD-ARG]
Declare the default ranking method for a group of hosts.
thresholds{NAME} = [METHOD[:METHOD-ARG]:]VALUE ...
Declare the default threshold(s) for a group of hosts.
timeout = FLOAT
Set the timeout in seconds of SSH commands to hosts.
(default=10.0)
'''
)
opt_parser.add_my_options("choice", "rank_method", "thresholds", "timeout")
opt_parser.modify_option(
'timeout',
help='Set the timeout in seconds of SSH commands to hosts.',
)
opts, args = opt_parser.parse_args()
report = Reporter(opts.verbosity - opts.quietness)
popen = RosePopener(event_handler=report)
select = HostSelector(event_handler=report, popen=popen)
try:
host_score_list = select(
names=args,
rank_method=opts.rank_method,
thresholds=opts.thresholds,
ssh_cmd_timeout=opts.timeout,
)
except (NoHostError, NoHostSelectError) as exc:
report(exc)
if opts.debug_mode:
traceback.print_exc()
sys.exit(1)
opts.choice = int(opts.choice)
report(choice(host_score_list[0 : opts.choice])[0] + "\n", level=0)
