def _from_location(self, location):
"""Return the ID of a location (origin URL or local copy path)."""
# Is location a "~/cylc-run/$SUITE/" directory?
# Use a hacky way to read the "log/rose-suite-run.version" file
suite_engine_proc = SuiteEngineProcessor.get_processor()
suite_dir_rel_root = getattr(
suite_engine_proc, "SUITE_DIR_REL_ROOT", None)
if suite_dir_rel_root and "/" + suite_dir_rel_root + "/" in location:
loc = location
while "/" + suite_dir_rel_root + "/" in loc:
suite_version_file_name = os.path.join(
loc, "log/rose-suite-run.version")
loc = os.path.dirname(loc)
if not os.access(suite_version_file_name, os.F_OK | os.R_OK):
continue
state = None
url = None
rev = None
for i, line in enumerate(open(suite_version_file_name)):
line = line.strip()
if state is None:
if line == "# SVN INFO":
state = line
elif state == "# SVN INFO":
if line.startswith("URL:"):
url = line.split(":", 1)[1].strip()
elif line.startswith("Revision:"):
rev = line.split(":", 1)[1].strip()
elif not line:
break
location = url + "@" + rev
break
# Assume location is a Subversion working copy of a Rosie suite
info_parser = SvnInfoXMLParser()
try:
info_entry = info_parser.parse(self.svn("info", "--xml", location))
except RosePopenError:
raise SuiteIdLocationError(location)
if "url" not in info_entry:
raise SuiteIdLocationError(location)
root = info_entry["repository:root"]
url = info_entry["url"]
path = url[len(root):]
if not path:
raise SuiteIdLocationError(location)
self.prefix = self.get_prefix_from_location_root(root)
names = path.lstrip("/").split("/", self.SID_LEN + 1)
if len(names) < self.SID_LEN:
raise SuiteIdLocationError(location)
sid = "".join(names[0:self.SID_LEN])
if not self.REC_IDX.match(sid):
raise SuiteIdLocationError(location)
self.idx = self.FORMAT_IDX % (self.prefix, sid)
self.sid = sid
if len(names) > self.SID_LEN:
self.branch = names[self.SID_LEN]
if "commit:revision" in info_entry:
self.revision = info_entry["commit:revision"]
def main():
"""Implement "rose suite-hook" command."""
opt_parser = RoseOptionParser()
opt_parser.add_my_options(
"mail_cc", "mail", "retrieve_job_logs", "shutdown")
opts, args = opt_parser.parse_args()
for key in ["mail_cc"]:
values = []
if getattr(opts, key):
for value in getattr(opts, key):
values.extend(value.split(","))
setattr(opts, key, values)
report = Reporter(opts.verbosity - opts.quietness - 1) # Reduced default
popen = RosePopener(event_handler=report)
suite_engine_proc = SuiteEngineProcessor.get_processor(
event_handler=report, popen=popen)
args = suite_engine_proc.process_suite_hook_args(*args, **vars(opts))
hook = RoseSuiteHook(event_handler=report,
popen=popen,
suite_engine_proc=suite_engine_proc)
hook(*args,
should_mail=opts.mail,
mail_cc_list=opts.mail_cc,
should_shutdown=opts.shutdown,
should_retrieve_job_logs=opts.retrieve_job_logs)
def main():
"""Implement "rose suite-hook" command."""
opt_parser = RoseOptionParser()
opt_parser.add_my_options(
"mail_cc", "mail", "retrieve_job_logs", "shutdown")
opts, args = opt_parser.parse_args()
for key in ["mail_cc"]:
values = []
if getattr(opts, key):
for value in getattr(opts, key):
values.extend(value.split(","))
setattr(opts, key, values)
report = Reporter(opts.verbosity - opts.quietness - 1) # Reduced default
popen = RosePopener(event_handler=report)
suite_engine_proc = SuiteEngineProcessor.get_processor(
event_handler=report, popen=popen)
args = suite_engine_proc.process_suite_hook_args(*args, **vars(opts))
hook = RoseSuiteHook(event_handler=report,
popen=popen,
suite_engine_proc=suite_engine_proc)
hook(*args,
should_mail=opts.mail,
mail_cc_list=opts.mail_cc,
should_shutdown=opts.shutdown,
should_retrieve_job_logs=opts.retrieve_job_logs)
def main():
"""rose task-env."""
opt_parser = RoseOptionParser()
opt_parser.add_my_options("cycle", "cycle_offsets", "path_globs", "prefix_delim", "suffix_delim")
opts, args = opt_parser.parse_args()
report = Reporter(opts.verbosity - opts.quietness - 1)
suite_engine_proc = SuiteEngineProcessor.get_processor(event_handler=report)
kwargs = {}
for k, v in vars(opts).items():
kwargs[k] = v
try:
task_props = suite_engine_proc.get_task_props(*args, **kwargs)
for k, v in task_props:
report(str(EnvExportEvent(k, v)) + "\n", level=0)
path_globs = opts.path_globs
if path_globs is None:
path_globs = []
prepend_paths_map = get_prepend_paths(report, task_props.suite_dir, path_globs, full_mode=True)
for k, prepend_paths in prepend_paths_map.items():
orig_paths = []
orig_v = os.getenv(k, "")
if orig_v:
orig_paths = orig_v.split(os.pathsep)
v = os.pathsep.join(prepend_paths + orig_paths)
report(str(EnvExportEvent(k, v)) + "\n", level=0)
except Exception as e:
report(e)
if opts.debug_mode:
traceback.print_exc(e)
sys.exit(1)
def main():
"""rose task-env."""
opt_parser = RoseOptionParser()
opt_parser.add_my_options("cycle", "cycle_offsets", "path_globs",
"prefix_delim", "suffix_delim")
opts, args = opt_parser.parse_args()
report = Reporter(opts.verbosity - opts.quietness - 1)
suite_engine_proc = SuiteEngineProcessor.get_processor(
event_handler=report)
kwargs = {}
for k, v in vars(opts).items():
kwargs[k] = v
try:
task_props = suite_engine_proc.get_task_props(*args, **kwargs)
for k, v in task_props:
report(str(EnvExportEvent(k, v)) + "\n", level=0)
path_globs = opts.path_globs
if path_globs is None:
path_globs = []
prepend_paths_map = get_prepend_paths(report,
task_props.suite_dir,
path_globs,
full_mode=True)
for k, prepend_paths in prepend_paths_map.items():
orig_paths = []
orig_v = os.getenv(k, "")
if orig_v:
orig_paths = orig_v.split(os.pathsep)
v = os.pathsep.join(prepend_paths + orig_paths)
report(str(EnvExportEvent(k, v)) + "\n", level=0)
except Exception as e:
report(e)
if opts.debug_mode:
traceback.print_exc(e)
sys.exit(1)
def __init__(self, template_env):
self.exposed = True
self.suite_engine_proc = SuiteEngineProcessor.get_processor()
self.template_env = template_env
self.host_name = socket.gethostname()
if self.host_name and "." in self.host_name:
self.host_name = self.host_name.split(".", 1)[0]
self.rose_version = ResourceLocator.default().get_version()
def __init__(self, event_handler=None, popen=None, suite_engine_proc=None):
self.event_handler = event_handler
if popen is None:
popen = RosePopener(event_handler)
self.popen = popen
if suite_engine_proc is None:
suite_engine_proc = SuiteEngineProcessor.get_processor(event_handler=event_handler, popen=popen)
self.suite_engine_proc = suite_engine_proc
def __init__(self, template_env):
self.exposed = True
self.suite_engine_proc = SuiteEngineProcessor.get_processor()
self.template_env = template_env
self.host_name = socket.gethostname()
if self.host_name and "." in self.host_name:
self.host_name = self.host_name.split(".", 1)[0]
self.rose_version = ResourceLocator.default().get_version()
def __init__(self, event_handler=None, popen=None, suite_engine_proc=None):
self.event_handler = event_handler
if popen is None:
popen = RosePopener(event_handler)
self.popen = popen
if suite_engine_proc is None:
suite_engine_proc = SuiteEngineProcessor.get_processor(
event_handler=event_handler, popen=popen)
self.suite_engine_proc = suite_engine_proc
def __init__(
self, event_handler=None, fs_util=None, suite_engine_proc=None):
self.event_handler = event_handler
if fs_util is None:
fs_util = FileSystemUtil(event_handler=event_handler)
self.fs_util = fs_util
if suite_engine_proc is None:
suite_engine_proc = SuiteEngineProcessor.get_processor(
event_handler=event_handler)
self.suite_engine_proc = suite_engine_proc
def get_output_root(cls):
"""Return the root output directory for suites."""
config = rose.config.default_node()
node = config.get(["rosie-id", "output-root"], no_ignore=True)
if node:
path = node.value
else:
suite_engine_proc = SuiteEngineProcessor.get_processor()
path = os.path.join("~", suite_engine_proc.RUN_DIR_REL_ROOT)
return rose.env.env_var_process(os.path.expanduser(path))
def __init__(self, event_handler=None):
self.event_handler = event_handler
self.popen = RosePopener(self.event_handler)
self.fs_util = FileSystemUtil(self.event_handler)
self.config_pm = ConfigProcessorsManager(
self.event_handler, self.popen, self.fs_util)
self.host_selector = HostSelector(self.event_handler, self.popen)
self.suite_engine_proc = SuiteEngineProcessor.get_processor(
event_handler=self.event_handler,
popen=self.popen,
fs_util=self.fs_util)
def __init__(self, event_handler=None):
self.event_handler = event_handler
self.popen = RosePopener(self.event_handler)
self.fs_util = FileSystemUtil(self.event_handler)
self.config_pm = ConfigProcessorsManager(self.event_handler,
self.popen, self.fs_util)
self.host_selector = HostSelector(self.event_handler, self.popen)
self.suite_engine_proc = SuiteEngineProcessor.get_processor(
event_handler=self.event_handler,
popen=self.popen,
fs_util=self.fs_util)
def __init__(self, event_handler=None, host_selector=None,
suite_engine_proc=None):
if event_handler is None:
event_handler = Reporter()
self.event_handler = event_handler
if host_selector is None:
host_selector = HostSelector(event_handler=event_handler)
self.host_selector = host_selector
if suite_engine_proc is None:
suite_engine_proc = SuiteEngineProcessor.get_processor(
event_handler=event_handler)
self.suite_engine_proc = suite_engine_proc
def __init__(self, event_handler=None, host_selector=None,
suite_engine_proc=None):
if event_handler is None:
event_handler = Reporter()
self.event_handler = event_handler
if host_selector is None:
host_selector = HostSelector(event_handler=event_handler)
self.host_selector = host_selector
if suite_engine_proc is None:
suite_engine_proc = SuiteEngineProcessor.get_processor(
event_handler=event_handler)
self.suite_engine_proc = suite_engine_proc
def __init__(self, *args, **kwargs):
self.exposed = True
self.suite_engine_proc = SuiteEngineProcessor.get_processor()
rose_conf = ResourceLocator.default().get_conf()
self.logo = rose_conf.get_value(["rose-bush", "logo"])
self.title = rose_conf.get_value(["rose-bush", "title"], self.TITLE)
self.host_name = rose_conf.get_value(["rose-bush", "host"])
if self.host_name is None:
self.host_name = HostSelector().get_local_host()
if self.host_name and "." in self.host_name:
self.host_name = self.host_name.split(".", 1)[0]
self.rose_version = ResourceLocator.default().get_version()
template_env = jinja2.Environment(loader=jinja2.FileSystemLoader(
ResourceLocator.default().get_util_home(
"lib", "html", "template", "rose-bush")))
self.template_env = template_env
def __init__(self, *args, **kwargs):
self.exposed = True
self.suite_engine_proc = SuiteEngineProcessor.get_processor()
rose_conf = ResourceLocator.default().get_conf()
self.logo = rose_conf.get_value(["rose-bush", "logo"])
self.title = rose_conf.get_value(["rose-bush", "title"], self.TITLE)
self.host_name = rose_conf.get_value(["rose-bush", "host"])
if self.host_name is None:
self.host_name = HostSelector().get_local_host()
if self.host_name and "." in self.host_name:
self.host_name = self.host_name.split(".", 1)[0]
self.rose_version = ResourceLocator.default().get_version()
template_env = jinja2.Environment(loader=jinja2.FileSystemLoader(
ResourceLocator.default().get_util_home(
"lib", "html", "template", "rose-bush")))
self.template_env = template_env
def suite_log_view(opts, args, event_handler=None):
"""Implement "rose suite-log" CLI functionality."""
suite_engine_proc = SuiteEngineProcessor.get_processor(
event_handler=event_handler)
opts.update_mode = (opts.update_mode or opts.archive_mode
or opts.force_mode)
if opts.force_mode:
args = ["*"]
if not opts.name:
opts.name = get_suite_name(event_handler)
if not opts.update_mode and not opts.user:
opts.user = pwd.getpwuid(os.stat(".").st_uid).pw_name
if opts.archive_mode:
suite_engine_proc.job_logs_archive(opts.name, args)
elif opts.update_mode:
suite_engine_proc.job_logs_pull_remote(opts.name, args,
opts.prune_remote_mode,
opts.force_mode)
if opts.view_mode or not opts.update_mode:
n_tries_left = 1
is_rose_bush_started = False
url = suite_engine_proc.get_suite_log_url(opts.user, opts.name)
if url.startswith("file://"):
if (opts.non_interactive or
raw_input("Start rose bush? [y/n] (default=n) ") == "y"):
suite_engine_proc.popen.run_bg("rose",
"bush",
"start",
preexec_fn=os.setpgrp)
is_rose_bush_started = True
n_tries_left = 5 # Give the server a chance to start
while n_tries_left:
n_tries_left -= 1
if n_tries_left:
url = suite_engine_proc.get_suite_log_url(opts.user, opts.name)
if url.startswith("file://"):
sleep(1)
continue
suite_engine_proc.launch_suite_log_browser(opts.user, opts.name)
break
if is_rose_bush_started:
status = suite_engine_proc.popen("rose", "bush")[0]
event_handler(RoseBushStartEvent(status))
return
def __init__(self, event_handler=None, popen=None, config_pm=None,
fs_util=None, suite_engine_proc=None):
if not self.CONF_NAME:
self.CONF_NAME = self.NAME
self.event_handler = event_handler
if popen is None:
popen = RosePopener(event_handler)
self.popen = popen
if fs_util is None:
fs_util = FileSystemUtil(event_handler)
self.fs_util = fs_util
if config_pm is None:
config_pm = ConfigProcessorsManager(event_handler, popen, fs_util)
self.config_pm = config_pm
if suite_engine_proc is None:
suite_engine_proc = SuiteEngineProcessor.get_processor(
event_handler=event_handler, popen=popen, fs_util=fs_util)
self.suite_engine_proc = suite_engine_proc
self.conf_tree_loader = ConfigTreeLoader()
def __init__(self, event_handler=None, popen=None, config_pm=None,
fs_util=None, host_selector=None, suite_engine_proc=None):
self.event_handler = event_handler
if popen is None:
popen = RosePopener(event_handler)
self.popen = popen
if fs_util is None:
fs_util = FileSystemUtil(event_handler)
self.fs_util = fs_util
if config_pm is None:
config_pm = ConfigProcessorsManager(event_handler, popen, fs_util)
self.config_pm = config_pm
if host_selector is None:
host_selector = HostSelector(event_handler, popen)
self.host_selector = host_selector
if suite_engine_proc is None:
suite_engine_proc = SuiteEngineProcessor.get_processor(
event_handler=event_handler, popen=popen, fs_util=fs_util)
self.suite_engine_proc = suite_engine_proc
def __init__(self, event_handler=None, popen=None, config_pm=None,
fs_util=None, suite_engine_proc=None):
if not self.CONF_NAME:
self.CONF_NAME = self.NAME
self.event_handler = event_handler
if popen is None:
popen = RosePopener(event_handler)
self.popen = popen
if fs_util is None:
fs_util = FileSystemUtil(event_handler)
self.fs_util = fs_util
if config_pm is None:
config_pm = ConfigProcessorsManager(event_handler, popen, fs_util)
self.config_pm = config_pm
if suite_engine_proc is None:
suite_engine_proc = SuiteEngineProcessor.get_processor(
event_handler=event_handler, popen=popen, fs_util=fs_util)
self.suite_engine_proc = suite_engine_proc
self.conf_tree_loader = ConfigTreeLoader()
def suite_log_view(opts, args, event_handler=None):
"""Implement "rose suite-log" CLI functionality."""
suite_engine_proc = SuiteEngineProcessor.get_processor(
event_handler=event_handler)
opts.update_mode = (
opts.update_mode or opts.archive_mode or opts.force_mode)
if opts.force_mode:
args = ["*"]
if not opts.name:
opts.name = get_suite_name(event_handler)
if not opts.update_mode and not opts.user:
opts.user = pwd.getpwuid(os.stat(".").st_uid).pw_name
if opts.archive_mode:
suite_engine_proc.job_logs_archive(opts.name, args)
elif opts.update_mode:
suite_engine_proc.job_logs_pull_remote(
opts.name, args, opts.prune_remote_mode, opts.force_mode)
if opts.view_mode or not opts.update_mode:
n_tries_left = 1
is_rose_bush_started = False
url = suite_engine_proc.get_suite_log_url(opts.user, opts.name)
if url.startswith("file://"):
if (opts.non_interactive or
raw_input("Start rose bush? [y/n] (default=n) ") == "y"):
suite_engine_proc.popen.run_bg(
"rose", "bush", "start", preexec_fn=os.setpgrp)
is_rose_bush_started = True
n_tries_left = 5 # Give the server a chance to start
while n_tries_left:
n_tries_left -= 1
if n_tries_left:
url = suite_engine_proc.get_suite_log_url(opts.user, opts.name)
if url.startswith("file://"):
sleep(1)
continue
suite_engine_proc.launch_suite_log_browser(opts.user, opts.name)
break
if is_rose_bush_started:
status = suite_engine_proc.popen("rose", "bush")[0]
event_handler(RoseBushStartEvent(status))
return
def main():
opt_parser = RoseOptionParser()
opt_parser.add_my_options("cycle", "cycle_offsets",
"prefix_delim", "suffix_delim")
opts, args = opt_parser.parse_args()
report = Reporter(opts.verbosity - opts.quietness)
suite_engine_proc = SuiteEngineProcessor.get_processor(
event_handler=report)
kwargs = {}
for k, v in vars(opts).items():
kwargs[k] = v
if opts.debug_mode:
task_props = suite_engine_proc.get_task_props(*args, **kwargs)
else:
try:
task_props = suite_engine_proc.get_task_props(*args, **kwargs)
except Exception as e:
report(e)
sys.exit(1)
for k, v in task_props:
report(str(EnvExportEvent(k, v)) + "\n", level=0)
def __init__(self, *args, **kwargs):
self.exposed = True
self.suite_engine_proc = SuiteEngineProcessor.get_processor()
rose_conf = ResourceLocator.default().get_conf()
self.logo = rose_conf.get_value(["rose-bush", "logo"])
self.title = rose_conf.get_value(["rose-bush", "title"], self.TITLE)
self.host_name = rose_conf.get_value(["rose-bush", "host"])
if self.host_name is None:
self.host_name = HostSelector().get_local_host()
if self.host_name and "." in self.host_name:
self.host_name = self.host_name.split(".", 1)[0]
self.rose_version = ResourceLocator.default().get_version()
template_env = jinja2.Environment(
loader=jinja2.FileSystemLoader(ResourceLocator.default(
).get_util_home("lib", "html", "template", "rose-bush")))
def urlise(text):
pattern = '((https?):\/\/[^\s\(\)&\[\]\{\}]+)'
replacement = '<a href="\g<1>">\g<1></a>'
text = re.sub(pattern, replacement, text)
return text
template_env.filters['urlise'] = urlise
self.template_env = template_env
def __init__(self, *args, **kwargs):
self.exposed = True
self.suite_engine_proc = SuiteEngineProcessor.get_processor()
rose_conf = ResourceLocator.default().get_conf()
self.logo = rose_conf.get_value(["rose-bush", "logo"])
self.title = rose_conf.get_value(["rose-bush", "title"], self.TITLE)
self.host_name = rose_conf.get_value(["rose-bush", "host"])
if self.host_name is None:
self.host_name = HostSelector().get_local_host()
if self.host_name and "." in self.host_name:
self.host_name = self.host_name.split(".", 1)[0]
self.rose_version = ResourceLocator.default().get_version()
template_env = jinja2.Environment(loader=jinja2.FileSystemLoader(
ResourceLocator.default().get_util_home(
"lib", "html", "template", "rose-bush")))
def urlise(text):
pattern = '((https?):\/\/[^\s\(\)&\[\]\{\}]+)'
replacement = '<a href="\g<1>">\g<1></a>'
text = re.sub(pattern, replacement, text)
return text
template_env.filters['urlise'] = urlise
self.template_env = template_env
def __init__(self, event_handler=None):
self.event_handler = event_handler
self.suite_engine_proc = SuiteEngineProcessor.get_processor(
event_handler=self.event_handler)
def to_output(self):
"""Return the output directory for this suite."""
suite_engine_proc = SuiteEngineProcessor.get_processor()
return suite_engine_proc.get_suite_log_url(None, str(self))
def _from_location(self, location):
"""Return the ID of a location (origin URL or local copy path)."""
# Is location a "~/cylc-run/$SUITE/" directory?
# Use a hacky way to read the "log/rose-suite-run.version" file
suite_engine_proc = SuiteEngineProcessor.get_processor()
suite_dir_rel_root = getattr(suite_engine_proc, "SUITE_DIR_REL_ROOT",
None)
if suite_dir_rel_root and "/" + suite_dir_rel_root + "/" in location:
loc = location
while "/" + suite_dir_rel_root + "/" in loc:
suite_version_file_name = os.path.join(
loc, "log/rose-suite-run.version")
loc = os.path.dirname(loc)
if not os.access(suite_version_file_name, os.F_OK | os.R_OK):
continue
state = None
url = None
rev = None
for i, line in enumerate(open(suite_version_file_name)):
line = line.strip()
if state is None:
if line.startswith("# svn info"):
state = line
elif state.startswith("# svn info"):
if line.startswith("URL:"):
url = line.split(":", 1)[1].strip()
elif line.startswith("Revision:"):
rev = line.split(":", 1)[1].strip()
elif not line:
break
if url and rev:
location = url + "@" + rev
break
# Assume location is a Subversion working copy of a Rosie suite
info_parser = SvnInfoXMLParser()
try:
info_entry = info_parser.parse(self.svn("info", "--xml", location))
except RosePopenError:
raise SuiteIdLocationError(location)
if "url" not in info_entry:
raise SuiteIdLocationError(location)
root = info_entry["repository:root"]
url = info_entry["url"]
path = url[len(root):]
if not path:
raise SuiteIdLocationError(location)
self.prefix = self.get_prefix_from_location_root(root)
names = path.lstrip("/").split("/", self.SID_LEN + 1)
if len(names) < self.SID_LEN:
raise SuiteIdLocationError(location)
sid = "".join(names[0:self.SID_LEN])
if not self.REC_IDX.match(sid):
raise SuiteIdLocationError(location)
self.idx = self.FORMAT_IDX % (self.prefix, sid)
self.sid = sid
if len(names) > self.SID_LEN:
self.branch = names[self.SID_LEN]
if "commit:revision" in info_entry:
self.revision = info_entry["commit:revision"]
def to_output(self):
"""Return the output directory for this suite."""
proc = SuiteEngineProcessor.get_processor()
return proc.get_suite_log_url(None, str(self))
def __init__(self, event_handler=None):
self.event_handler = event_handler
self.popen = RosePopener(self.event_handler)
self.suite_engine_proc = SuiteEngineProcessor.get_processor(
event_handler=self.event_handler, popen=self.popen)
def __init__(self, event_handler=None):
self.event_handler = event_handler
self.suite_engine_proc = SuiteEngineProcessor.get_processor(
event_handler=self.event_handler)
def __init__(self, event_handler=None):
self.event_handler = event_handler
self.popen = RosePopener(self.event_handler)
self.suite_engine_proc = SuiteEngineProcessor.get_processor(
event_handler=self.event_handler, popen=self.popen)