def write_source_vc_info(run_source_dir, output=None, popen=None): """Write version control information of sources used in run time. run_source_dir -- The source directory we are interested in. output -- An open file handle or a string containing a writable path. If not specified, use sys.stdout. popen -- A metomi.rose.popen.RosePopener instance for running vc commands. If not specified, use a new local instance. """ if popen is None: popen = RosePopener() if output is None: handle = sys.stdout elif hasattr(output, "write"): handle = output else: handle = open(output, "wb") msg = "%s\n" % run_source_dir write_safely(msg, handle) environ = dict(os.environ) environ["LANG"] = "C" for vcs, args_list in [ ( "svn", [ ["info", "--non-interactive"], ["status", "--non-interactive"], ["diff", "--internal-diff", "--non-interactive"], ], ), ("git", [["describe"], ["status"], ["diff"]]), ]: if not popen.which(vcs): continue cwd = os.getcwd() os.chdir(run_source_dir) try: for args in args_list: cmd = [vcs, *args] ret_code, out, _ = popen.run(*cmd, env=environ) if out: write_safely(("#" * 80 + "\n"), handle) write_safely(("# %s\n" % popen.shlex_join(cmd)), handle) write_safely(("#" * 80 + "\n"), handle) write_safely(out, handle) if ret_code: # If cmd fails once, it will likely fail again break finally: os.chdir(cwd)
def __init__(self, prefixes=None, prompt_func=None, popen=None, event_handler=None): if not event_handler: event_handler = Reporter() if not popen: popen = RosePopener(event_handler=event_handler) self.event_handler = event_handler self.popen = popen self.prompt_func = prompt_func self.prefixes = [] self.unreachable_prefixes = [] self.auth_managers = {} conf = ResourceLocator.default().get_conf() conf_rosie_id = conf.get(["rosie-id"], no_ignore=True) if conf_rosie_id is None: raise RosieWSClientConfError() for key, node in conf_rosie_id.value.items(): if node.is_ignored() or not key.startswith("prefix-ws."): continue prefix = key.replace("prefix-ws.", "") self.auth_managers[prefix] = RosieWSClientAuthManager( prefix, popen=self.popen, prompt_func=self.prompt_func) if not prefixes: prefixes_str = conf_rosie_id.get_value(["prefixes-ws-default"]) if prefixes_str: prefixes = shlex.split(prefixes_str) else: prefixes = sorted(self.auth_managers.keys()) self.set_prefixes(prefixes)
def main(): """Implement "rose suite-hook" command.""" opt_parser = RoseOptionParser() opt_parser.add_my_options("mail_cc", "mail", "retrieve_job_logs", "shutdown") opts, args = opt_parser.parse_args() for key in ["mail_cc"]: values = [] if getattr(opts, key): for value in getattr(opts, key): values.extend(value.split(",")) setattr(opts, key, values) report = Reporter(opts.verbosity - opts.quietness - 1) # Reduced default popen = RosePopener(event_handler=report) suite_engine_proc = SuiteEngineProcessor.get_processor( event_handler=report, popen=popen) args = suite_engine_proc.process_suite_hook_args(*args, **vars(opts)) hook = RoseSuiteHook(event_handler=report, popen=popen, suite_engine_proc=suite_engine_proc) hook(*args, should_mail=opts.mail, mail_cc_list=opts.mail_cc, should_shutdown=opts.shutdown, should_retrieve_job_logs=opts.retrieve_job_logs)
def __init__( self, event_handler=None, popen=None, config_pm=None, fs_util=None, suite_engine_proc=None, ): if not self.CONF_NAME: self.CONF_NAME = self.NAME self.event_handler = event_handler if popen is None: popen = RosePopener(event_handler) self.popen = popen if fs_util is None: fs_util = FileSystemUtil(event_handler) self.fs_util = fs_util if config_pm is None: config_pm = ConfigProcessorsManager(event_handler, popen, fs_util) self.config_pm = config_pm if suite_engine_proc is None: suite_engine_proc = SuiteEngineProcessor.get_processor( event_handler=event_handler, popen=popen, fs_util=fs_util ) self.suite_engine_proc = suite_engine_proc self.conf_tree_loader = ConfigTreeLoader()
def __init__(self, opts, reporter=None, popen=None, fs_util=None): self.opts = opts if reporter is None: self.reporter = Reporter(opts.verbosity - opts.quietness) else: self.reporter = reporter if popen is None: self.popen = RosePopener(event_handler=self.reporter) else: self.popen = popen if fs_util is None: self.fs_util = FileSystemUtil(event_handler=self.reporter) else: self.fs_util = fs_util self.host_selector = HostSelector(event_handler=self.reporter, popen=self.popen)
def __init__(self, event_handler=None, popen=None): self.event_handler = event_handler if popen is None: popen = RosePopener(event_handler=event_handler) self.popen = popen self.scorers = {} self.local_host_strs = None
def __init__(self, event_handler=None, popen=None, fs_util=None, force_mode=False): if event_handler is None: event_handler = self._dummy self.event_handler = event_handler if popen is None: popen = RosePopener(event_handler) self.popen = popen if fs_util is None: fs_util = FileSystemUtil(event_handler) self.fs_util = fs_util self.force_mode = force_mode self._work_dir = None atexit.register(self._delete_work_dir) self.subversion_servers_conf = None subversion_servers_conf = os.getenv("ROSIE_SUBVERSION_SERVERS_CONF") if subversion_servers_conf: self.subversion_servers_conf = subversion_servers_conf else: subversion_servers_conf = os.path.expanduser( self.SUBVERSION_SERVERS_CONF) if os.path.exists(subversion_servers_conf): self.subversion_servers_conf = subversion_servers_conf
def __init__(self, event_handler=None, popen=None, suite_engine_proc=None): self.event_handler = event_handler if popen is None: popen = RosePopener(event_handler) self.popen = popen if suite_engine_proc is None: suite_engine_proc = SuiteEngineProcessor.get_processor( event_handler=event_handler, popen=popen) self.suite_engine_proc = suite_engine_proc
def __init__(self, event_handler=None, popen=None): if event_handler is None: event_handler = Reporter() self.event_handler = event_handler if popen is None: popen = RosePopener(self.event_handler) self.popen = popen self.path = os.path.dirname( os.path.dirname(sys.modules["metomi.rosie"].__file__))
def rose_fileinstall(srcdir=None, opts=None, rundir=None): """Call Rose Fileinstall. Args: srcdir(pathlib.Path): Search for a ``rose-suite.conf`` file in this location. rundir (pathlib.Path) """ if not rose_config_exists(rundir, opts): return False # Load the config tree config_tree = rose_config_tree_loader(rundir, opts) if any(i.startswith('file') for i in config_tree.node.value): try: startpoint = os.getcwd() os.chdir(rundir) except FileNotFoundError as exc: raise exc else: # Carry out imports. import asyncio from metomi.rose.config_processor import ConfigProcessorsManager from metomi.rose.popen import RosePopener from metomi.rose.reporter import Reporter from metomi.rose.fs_util import FileSystemUtil # Update config tree with install location # NOTE-TO-SELF: value=os.environ["CYLC_WORKFLOW_RUN_DIR"] config_tree.node = config_tree.node.set( keys=["file-install-root"], value=str(rundir) ) # Artificially set rose to verbose. event_handler = Reporter(3) fs_util = FileSystemUtil(event_handler) popen = RosePopener(event_handler) # Get an Asyncio loop if one doesn't exist: # Rose may need an event loop to invoke async interfaces, # doing this here incase we want to go async in cylc-rose. # See https://github.com/cylc/cylc-rose/pull/130/files try: asyncio.get_event_loop() except RuntimeError: asyncio.set_event_loop(asyncio.new_event_loop()) # Process fileinstall. config_pm = ConfigProcessorsManager(event_handler, popen, fs_util) config_pm(config_tree, "file") finally: os.chdir(startpoint) return config_tree.node
def test_oserror_has_filename(self): """Does what it says.""" rose_popen = RosePopener() name = "bad-command" try: rose_popen.run(name) except RosePopenError as exc: ose = FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), name) try: self.assertEqual(str(ose), exc.stderr) except AssertionError: # This is horrible, but refers to a bug in some versions of # Python 2.6 - https://bugs.python.org/issue32490 err_msg = ("[Errno 2] No such file or directory:" " 'bad-command': 'bad-command'") self.assertEqual(err_msg, exc.stderr) else: self.fail("should return FileNotFoundError")
def _bash_login_cmd(cmd: List[str]) -> List[str]: """Return the given command as a bash login shell command. This allows users to set env vars. Example: >>> HostSelector._bash_login_cmd(["echo", "-n", "Multiple words"]) ['bash', '-l', '-c', "echo -n 'Multiple words'"] """ return ['bash', '-l', '-c', RosePopener.shlex_join(cmd)]
def __init__(self, config, opts, args, method_paths, reporter=None, popen=None): if reporter is None: self.reporter = Reporter(opts.verbosity - opts.quietness) else: self.reporter = reporter if popen is None: self.popen = RosePopener(event_handler=self.reporter) else: self.popen = popen self.opts = opts self.args = args self.config = config self.tasks = self.load_tasks() modules = [] for path in method_paths: for filename in glob.glob(path + "/*.py"): modules.append(filename) self.load_user_comparison_modules(modules)
def __init__(self, event_handler=None, popen=None): if event_handler is None: event_handler = Reporter() self.event_handler = event_handler if popen is None: popen = RosePopener(self.event_handler) self.popen = popen path = os.path.dirname( os.path.dirname(sys.modules["metomi.rosie"].__file__)) self.usertools_manager = SchemeHandlersManager([path], "rosie.usertools", ["get_emails"])
def __init__(self, event_handler=None, popen=None, fs_util=None): self.event_handler = event_handler if popen is None: popen = RosePopener(event_handler) self.popen = popen if fs_util is None: fs_util = FileSystemUtil(event_handler) self.fs_util = fs_util path = os.path.dirname( os.path.dirname(sys.modules["metomi.rose"].__file__)) SchemeHandlersManager.__init__(self, [path], "rose.config_processors", ["process"])
def __init__(self, event_handler=None, popen=None, fs_util=None): self.event_handler = event_handler if popen is None: popen = RosePopener(event_handler) self.popen = popen if fs_util is None: fs_util = FileSystemUtil(event_handler) self.fs_util = fs_util path = os.path.dirname(os.path.dirname(sys.modules["rose"].__file__)) SchemeHandlersManager.__init__( self, [path], ns="rose.loc_handlers", attrs=["parse", "pull"], can_handle="can_pull")
def __init__(self, event_handler=None, popen=None, fs_util=None): if event_handler is None: event_handler = self._dummy self.event_handler = event_handler if popen is None: popen = RosePopener(event_handler) self.popen = popen if fs_util is None: fs_util = FileSystemUtil(event_handler) self.fs_util = fs_util self.post_commit_hook = RosieSvnPostCommitHook( event_handler=event_handler, popen=popen)
def _launch(name, event_handler=None, run_fg=False, *args, **kwargs): popen = RosePopener(event_handler) command = popen.get_cmd(name, *args) kwargs['stdin'] = sys.stdin if run_fg: return popen.run(*command, **kwargs) popen.run_bg(*command, **kwargs)
def __init__(self, event_handler=None, popen=None, fs_util=None, host_selector=None, **_): self.event_handler = event_handler if popen is None: popen = RosePopener(event_handler) self.popen = popen if fs_util is None: fs_util = FileSystemUtil(event_handler) self.fs_util = fs_util if host_selector is None: host_selector = HostSelector(event_handler, popen) self.host_selector = host_selector self.date_time_oper = RoseDateTimeOperator()
def rose_fileinstall(srcdir=None, opts=None, rundir=None): """Call Rose Fileinstall. Args: srcdir(pathlib.Path): Search for a ``rose-suite.conf`` file in this location. rundir (pathlib.Path) """ if not rose_config_exists(rundir, opts): return False # Load the config tree config_tree = rose_config_tree_loader(rundir, opts) if any(i.startswith('file') for i in config_tree.node.value): try: startpoint = os.getcwd() os.chdir(rundir) except FileNotFoundError as exc: raise exc else: # Carry out imports. from metomi.rose.config_processor import ConfigProcessorsManager from metomi.rose.popen import RosePopener from metomi.rose.reporter import Reporter from metomi.rose.fs_util import FileSystemUtil # Update config tree with install location # NOTE-TO-SELF: value=os.environ["CYLC_WORKFLOW_RUN_DIR"] config_tree.node = config_tree.node.set( keys=["file-install-root"], value=str(rundir) ) # Artificially set rose to verbose. event_handler = Reporter(3) fs_util = FileSystemUtil(event_handler) popen = RosePopener(event_handler) # Process files config_pm = ConfigProcessorsManager(event_handler, popen, fs_util) config_pm(config_tree, "file") finally: os.chdir(startpoint) return config_tree.node
def __init__( self, prefix, popen=None, prompt_func=None, event_handler=None ): self.prefix = prefix root = self._get_conf_value("ws") if root is None: raise UndefinedRosiePrefixWS(self.prefix) if not root.endswith("/"): root += "/" self.root = root urlparse_res = urlparse(self.root) self.scheme = urlparse_res[0] self.host = urlparse_res[1] self.password_orig = None self.username_orig = None self.password = None self.username = None if popen is None: popen = RosePopener() self.popen = popen self.prompt_func = prompt_func if event_handler is None: self.event_handler = Reporter() else: self.event_handler = event_handler res_loc = ResourceLocator.default() password_stores_str = ( res_loc.default() .get_conf() .get_value( keys=["rosie-id", "prefix-password-store." + self.prefix], default=self.PASSWORD_STORES_STR, ) ) for password_store_name in shlex.split(password_stores_str): password_store_cls = self.PASSWORD_STORE_CLASSES.get( password_store_name ) if password_store_cls is not None and password_store_cls.usable(): self.password_store = password_store_cls() break else: self.password_store = None self.requests_kwargs = {} self._init_https_params()
def main(): """Implement the "rose host-select" command.""" opt_parser = RoseOptionParser() opt_parser.add_my_options("choice", "rank_method", "thresholds", "timeout") opts, args = opt_parser.parse_args() report = Reporter(opts.verbosity - opts.quietness) popen = RosePopener(event_handler=report) select = HostSelector(event_handler=report, popen=popen) try: host_score_list = select(names=args, rank_method=opts.rank_method, thresholds=opts.thresholds, ssh_cmd_timeout=opts.timeout) except (NoHostError, NoHostSelectError) as exc: report(exc) if opts.debug_mode: traceback.print_exc() sys.exit(1) opts.choice = int(opts.choice) report(choice(host_score_list[0:opts.choice])[0] + "\n", level=0)
def rose_fileinstall(dir_=None, opts=None, dest_root=None): """Call Rose Fileinstall. Args: dir_(string or pathlib.Path): Search for a ``rose-suite.conf`` file in this location. dest_root (string or pathlib.Path) """ if not rose_config_exists(dir_): return False # Load the config tree config_tree = rose_config_tree_loader(dir_, opts) if any(['file' in i for i in config_tree.node.value]): # Carry out imports. from metomi.rose.config_processor import ConfigProcessorsManager from metomi.rose.popen import RosePopener from metomi.rose.reporter import Reporter from metomi.rose.fs_util import FileSystemUtil # Update config tree with install location # NOTE-TO-SELF: value=os.environ["CYLC_SUITE_RUN_DIR"] config_tree.node = config_tree.node.set( keys=["file-install-root"], value=dest_root ) # Artificially set rose to verbose. # TODO - either use Cylc Log as event handler, or get Cylc Verbosity # settings to pass to Rose Reporter. event_handler = Reporter(3) fs_util = FileSystemUtil(event_handler) popen = RosePopener(event_handler) # Process files config_pm = ConfigProcessorsManager(event_handler, popen, fs_util) config_pm(config_tree, "file") return True
def process(self): """Process STEM options into 'rose suite-run' options.""" # Generate options for source trees repos = {} repos_with_hosts = {} if not self.opts.source: self.opts.source = ['.'] self.opts.project = list() for i, url in enumerate(self.opts.source): project, url, base, rev, mirror = self._ascertain_project(url) self.opts.source[i] = url self.opts.project.append(project) # Versions of variables with hostname prepended for working copies url_host = self._prepend_localhost(url) base_host = self._prepend_localhost(base) if project in repos: repos[project].append(url) repos_with_hosts[project].append(url_host) else: repos[project] = [url] repos_with_hosts[project] = [url_host] self._add_define_option('SOURCE_' + project.upper() + '_REV', '"' + rev + '"') self._add_define_option('SOURCE_' + project.upper() + '_BASE', '"' + base + '"') self._add_define_option( 'HOST_SOURCE_' + project.upper() + '_BASE', '"' + base_host + '"') self._add_define_option( 'SOURCE_' + project.upper() + '_MIRROR', '"' + mirror + '"') self.reporter(SourceTreeAddedAsBranchEvent(url)) for project, branches in repos.items(): var = 'SOURCE_' + project.upper() branchstring = RosePopener.list_to_shell_str(branches) self._add_define_option(var, '"' + branchstring + '"') for project, branches in repos_with_hosts.items(): var_host = 'HOST_SOURCE_' + project.upper() branchstring = RosePopener.list_to_shell_str(branches) self._add_define_option(var_host, '"' + branchstring + '"') # Generate the variable containing tasks to run if self.opts.group: if not self.opts.defines: self.opts.defines = [] expanded_groups = [] for i in self.opts.group: expanded_groups.extend(i.split(',')) self.opts.defines.append(SUITE_RC_PREFIX + 'RUN_NAMES=' + str(expanded_groups)) # Load the config file and return any automatic-options auto_opts = self._read_auto_opts() if auto_opts: automatic_options = auto_opts.split() for option in automatic_options: elements = option.split("=") if len(elements) == 2: self._add_define_option(elements[0], '"' + elements[1] + '"') # Change into the suite directory if self.opts.conf_dir: self.reporter(SuiteSelectionEvent(self.opts.conf_dir)) self._check_suite_version( os.path.join(self.opts.conf_dir, 'rose-suite.conf')) else: thissuite = self._this_suite() self.fs_util.chdir(thissuite) self.reporter(SuiteSelectionEvent(thissuite)) # Create a default name for the suite; allow override by user if not self.opts.name: self.opts.name = self._generate_name() return self.opts
def select( self, names=None, rank_method=None, thresholds=None, ssh_cmd_timeout=None, ): """Return a list. Element 0 is most desirable. Each element of the list is a tuple (host, score). names: a list of known host groups or host names. rank_method: the ranking method. Can be one of: load:1, load:5, load:15 (=load =default), fs:FS and random. The "load" methods determines the load using the average load as returned by the "uptime" command divided by the number of CPUs. The "fs" method determines the load using the usage in the file system specified by FS. The "mem" method ranks by highest free memory. The "random" method ranks everything by random. thresholds: a list of thresholds which each host must not exceed. Should be in the format rank_method:value, where rank_method is one of load:1, load:5, load:15 or fs:FS; and value is number that must be be exceeded. ssh_cmd_timeout: timeout of SSH commands to hosts. A float in seconds. """ host_names, rank_method, thresholds = self.expand( names, rank_method, thresholds) # Load scorers, ranking and thresholds rank_method_arg = None if rank_method: if ":" in rank_method: rank_method, rank_method_arg = rank_method.split(":", 1) else: rank_method = self.RANK_METHOD_DEFAULT rank_conf = ScorerConf(self.get_scorer(rank_method), rank_method_arg) self.handle_event(RankMethodEvent(rank_method, rank_method_arg)) threshold_confs = [] if thresholds: for threshold in thresholds: method = self.RANK_METHOD_DEFAULT method_arg = None value = threshold if ":" in threshold: head, value = threshold.rsplit(":", 1) method = head if ":" in head: method, method_arg = head.split(":", 1) try: float(value) except ValueError: raise ValueError(threshold) scorer = self.get_scorer(method) if method_arg is None: method_arg = scorer.ARG threshold_conf = ScorerConf(self.get_scorer(method), method_arg, value) threshold_confs.append(threshold_conf) if ssh_cmd_timeout is None: conf = ResourceLocator.default().get_conf() ssh_cmd_timeout = float( conf.get_value(["rose-host-select", "timeout"], self.SSH_CMD_TIMEOUT)) host_name_list = list(host_names) host_names = [] for host_name in host_name_list: if self.is_local_host(host_name): if self.get_local_host() not in host_names: host_names.append(self.get_local_host()) else: host_names.append(host_name) # Random selection with no thresholds. Return the 1st available host. if rank_conf.method == self.RANK_METHOD_RANDOM and not threshold_confs: shuffle(host_names) for host_name in host_names: if self.is_local_host(host_name): return [("localhost", 1)] command = self.popen.get_cmd("ssh", host_name, "true") proc = self.popen.run_bg(*command, preexec_fn=os.setpgrp) time0 = time() while (proc.poll() is None and time() - time0 <= ssh_cmd_timeout): sleep(self.SSH_CMD_POLL_DELAY) if proc.poll() is None: os.killpg(proc.pid, signal.SIGTERM) proc.wait() self.handle_event(TimedOutHostEvent(host_name)) elif proc.wait(): self.handle_event( HostSelectCommandFailedEvent(host_name, proc.returncode)) else: return [(host_name, 1)] else: raise NoHostSelectError() # ssh to each host to return its score(s). host_proc_dict = {} for host_name in sorted(host_names): # build host-select-client command command: List[str] = [] # pass through CYLC_VERSION to support use of cylc wrapper script try: import cylc.flow except ModuleNotFoundError: pass else: command.extend([ 'env', f'CYLC_VERSION={cylc.flow.__version__}', ]) cylc_env_name = os.getenv('CYLC_ENV_NAME') if cylc_env_name: command.append(f'CYLC_ENV_NAME={cylc_env_name}') command.extend(self._bash_login_cmd(['rose', 'host-select-client'])) # build list of metrics to obtain for each host metrics = rank_conf.get_command() for threshold_conf in threshold_confs: for metric in threshold_conf.get_command(): if metric not in metrics: metrics.append(metric) # convert metrics list to JSON stdin stdin = '\n***start**\n' + json.dumps(metrics) + '\n**end**\n' if not self.is_local_host(host_name): command = [ *self.popen.get_cmd('ssh', host_name), RosePopener.shlex_join(command) ] # fire off host-select-client processes proc = self.popen.run_bg(*command, stdin=stdin, preexec_fn=os.setpgrp) proc.stdin.write(stdin) proc.stdin.flush() host_proc_dict[host_name] = (proc, metrics) # Retrieve score for each host name host_score_list = [] time0 = time() while host_proc_dict: sleep(self.SSH_CMD_POLL_DELAY) for host_name, (proc, metrics) in list(host_proc_dict.items()): if proc.poll() is None: # still running continue stdout, stderr = proc.communicate() if proc.returncode: self.handle_event( HostSelectCommandFailedEvent(host_name, proc.returncode, stderr)) host_proc_dict.pop(host_name) else: out = _deserialise(metrics, json.loads(stdout.strip())) host_proc_dict.pop(host_name) for threshold_conf in threshold_confs: try: score = threshold_conf.command_out_parser( out, metrics) is_bad = threshold_conf.check_threshold(score) except ValueError: is_bad = True score = None if is_bad: self.handle_event( HostThresholdNotMetEvent( host_name, threshold_conf, score)) break else: try: score = rank_conf.command_out_parser(out, metrics) host_score_list.append((host_name, score)) except ValueError: score = None self.handle_event( HostSelectScoreEvent(host_name, score)) if time() - time0 > ssh_cmd_timeout: break # Report timed out hosts for host_name, (proc, _) in sorted(host_proc_dict.items()): self.handle_event(TimedOutHostEvent(host_name)) os.killpg(proc.pid, signal.SIGTERM) proc.wait() if not host_score_list: raise NoHostSelectError() host_score_list.sort(key=lambda a: a[1], reverse=rank_conf.scorer.SIGN < 0) return host_score_list
class StemRunner(object): """Set up options for running a STEM job through Rose.""" def __init__(self, opts, reporter=None, popen=None, fs_util=None): self.opts = opts if reporter is None: self.reporter = Reporter(opts.verbosity - opts.quietness) else: self.reporter = reporter if popen is None: self.popen = RosePopener(event_handler=self.reporter) else: self.popen = popen if fs_util is None: self.fs_util = FileSystemUtil(event_handler=self.reporter) else: self.fs_util = fs_util self.host_selector = HostSelector(event_handler=self.reporter, popen=self.popen) def _add_define_option(self, var, val): """Add a define option passed to the SuiteRunner.""" if self.opts.defines: self.opts.defines.append(SUITE_RC_PREFIX + var + '=' + val) else: self.opts.defines = [SUITE_RC_PREFIX + var + '=' + val] self.reporter(ConfigVariableSetEvent(var, val)) return def _get_base_dir(self, item): """Given a source tree return the following from 'fcm loc-layout': * url * sub_tree * peg_rev * root * project """ ret_code, output, stderr = self.popen.run('fcm', 'loc-layout', item) output = output.decode() if ret_code != 0: raise ProjectNotFoundException(item, stderr) ret = {} for line in output.splitlines(): if ":" not in line: continue key, value = line.split(":", 1) if key: if value: ret[key] = value.strip() return ret def _get_project_from_url(self, source_dict): """Run 'fcm keyword-print' to work out the project name.""" repo = source_dict['root'] if source_dict['project']: repo += '/' + source_dict['project'] kpoutput = self.popen.run('fcm', 'kp', source_dict['url'])[1] project = None for line in kpoutput.splitlines(): if line.rstrip().endswith(repo.encode('UTF-8')): kpresult = re.search(r'^location{primary}\[(.*)\]', line.decode()) if kpresult: project = kpresult.group(1) break return project def _deduce_mirror(self, source_dict, project): """Deduce the mirror location of this source tree.""" # Root location for project proj_root = source_dict['root'] + '/' + source_dict['project'] # Swap project to mirror project = re.sub(r'\.x$', r'.xm', project) mirror_repo = "fcm:" + project # Generate mirror location mirror = re.sub(proj_root, mirror_repo, source_dict['url']) # Remove any sub-tree mirror = re.sub(source_dict['sub_tree'], r'', mirror) mirror = re.sub(r'/@', r'@', mirror) # Add forwards slash after .xm if missing if '.xm/' not in mirror: mirror = re.sub(r'\.xm', r'.xm/', mirror) return mirror def _ascertain_project(self, item): """Set the project name and top-level from 'fcm loc-layout'. Returns: * project name * top-level location of the source tree with revision number * top-level location of the source tree without revision number * revision number """ project = None try: project, item = item.split("=", 1) except ValueError: pass if re.search(r'^\.', item): item = os.path.abspath(os.path.join(os.getcwd(), item)) if project is not None: print("[WARN] Forcing project for '{0}' to be '{1}'".format( item, project)) return project, item, item, '', '' source_dict = self._get_base_dir(item) project = self._get_project_from_url(source_dict) if not project: raise ProjectNotFoundException(item) mirror = self._deduce_mirror(source_dict, project) if 'peg_rev' in source_dict and '@' in item: revision = '@' + source_dict['peg_rev'] base = re.sub(r'@.*', r'', item) else: revision = '' base = item # Remove subtree from base and item if 'sub_tree' in source_dict: item = re.sub(r'(.*)%s/?$' % (source_dict['sub_tree']), r'\1', item, count=1) base = re.sub(r'(.*)%s/?$' % (source_dict['sub_tree']), r'\1', base, count=1) # Remove trailing forwards-slash item = re.sub(r'/$', r'', item) base = re.sub(r'/$', r'', base) # Remove anything after a point project = re.sub(r'\..*', r'', project) return project, item, base, revision, mirror def _generate_name(self): """Generate a suite name from the name of the first source tree.""" try: basedir = self._ascertain_project(os.getcwd())[1] except ProjectNotFoundException: if self.opts.conf_dir: basedir = os.path.abspath(self.opts.conf_dir) else: basedir = os.getcwd() name = os.path.basename(basedir) self.reporter(NameSetEvent(name)) return name def _this_suite(self): """Find the location of the suite in the first source tree.""" # Get base of first source basedir = '' if self.opts.source: basedir = self.opts.source[0] else: basedir = self._ascertain_project(os.getcwd())[1] suitedir = os.path.join(basedir, DEFAULT_TEST_DIR) suitefile = os.path.join(suitedir, "rose-suite.conf") if not os.path.isfile(suitefile): raise RoseSuiteConfNotFoundException(suitedir) self._check_suite_version(suitefile) return suitedir def _read_auto_opts(self): """Read the site metomi.rose.conf file.""" return ResourceLocator.default().get_conf().get_value( ["rose-stem", "automatic-options"]) def _check_suite_version(self, fname): """Check the suite is compatible with this version of metomi.rose-stem. """ if not os.path.isfile(fname): raise RoseSuiteConfNotFoundException(os.path.dirname(fname)) config = metomi.rose.config.load(fname) suite_rose_stem_version = config.get(['ROSE_STEM_VERSION']) if suite_rose_stem_version: suite_rose_stem_version = int(suite_rose_stem_version.value) else: suite_rose_stem_version = None if not suite_rose_stem_version == ROSE_STEM_VERSION: raise RoseStemVersionException(suite_rose_stem_version) def _prepend_localhost(self, url): """Prepend the local hostname to urls which do not point to repository locations.""" if ':' not in url or url.split( ':', 1)[0] not in ['svn', 'fcm', 'http', 'https', 'svn+ssh']: url = self.host_selector.get_local_host() + ':' + url return url def process(self): """Process STEM options into 'rose suite-run' options.""" # Generate options for source trees repos = {} repos_with_hosts = {} if not self.opts.source: self.opts.source = ['.'] self.opts.project = list() for i, url in enumerate(self.opts.source): project, url, base, rev, mirror = self._ascertain_project(url) self.opts.source[i] = url self.opts.project.append(project) # Versions of variables with hostname prepended for working copies url_host = self._prepend_localhost(url) base_host = self._prepend_localhost(base) if project in repos: repos[project].append(url) repos_with_hosts[project].append(url_host) else: repos[project] = [url] repos_with_hosts[project] = [url_host] self._add_define_option('SOURCE_' + project.upper() + '_REV', '"' + rev + '"') self._add_define_option('SOURCE_' + project.upper() + '_BASE', '"' + base + '"') self._add_define_option( 'HOST_SOURCE_' + project.upper() + '_BASE', '"' + base_host + '"') self._add_define_option( 'SOURCE_' + project.upper() + '_MIRROR', '"' + mirror + '"') self.reporter(SourceTreeAddedAsBranchEvent(url)) for project, branches in repos.items(): var = 'SOURCE_' + project.upper() branchstring = RosePopener.list_to_shell_str(branches) self._add_define_option(var, '"' + branchstring + '"') for project, branches in repos_with_hosts.items(): var_host = 'HOST_SOURCE_' + project.upper() branchstring = RosePopener.list_to_shell_str(branches) self._add_define_option(var_host, '"' + branchstring + '"') # Generate the variable containing tasks to run if self.opts.group: if not self.opts.defines: self.opts.defines = [] expanded_groups = [] for i in self.opts.group: expanded_groups.extend(i.split(',')) self.opts.defines.append(SUITE_RC_PREFIX + 'RUN_NAMES=' + str(expanded_groups)) # Load the config file and return any automatic-options auto_opts = self._read_auto_opts() if auto_opts: automatic_options = auto_opts.split() for option in automatic_options: elements = option.split("=") if len(elements) == 2: self._add_define_option(elements[0], '"' + elements[1] + '"') # Change into the suite directory if self.opts.conf_dir: self.reporter(SuiteSelectionEvent(self.opts.conf_dir)) self._check_suite_version( os.path.join(self.opts.conf_dir, 'rose-suite.conf')) else: thissuite = self._this_suite() self.fs_util.chdir(thissuite) self.reporter(SuiteSelectionEvent(thissuite)) # Create a default name for the suite; allow override by user if not self.opts.name: self.opts.name = self._generate_name() return self.opts
def main(): """Implement the "rose host-select" command.""" opt_parser = RoseOptionParser( usage='rose host-select [OPTIONS] [GROUP/HOST ...]', description=''' Select a host from a set of groups or names by load, by free memory or by random. Print the selected host name. ''', epilog=''' RANKING METHODS IN DETAIL (--rank-method): `load` Rank by average load as reported by `uptime` divided by number of virtual processors. If `METHOD-ARG` is specified, it must be `1`, `5` or `15`. The default is to use the 15 minute load. `fs` Rank by % usage of a file system as reported by `df`. `METHOD-ARG` must be a valid file system in all the given hosts and host groups. The default is to use the `~` directory. `mem` Rank by largest amount of free memory. Uses `free -m` to return memory in Mb `random` No ranking is used. CONFIGURATION The command reads its settings from the `[rose-host-select]` section in the Rose configuration. All settings are optional. Type `rose config rose-host-select` to print settings. Valid settings are: default = GROUP/HOST ... The default arguments to use for this command. group{NAME} = GROUP/HOST ... Declare a named group of hosts. method{NAME} = METHOD[:METHOD-ARG] Declare the default ranking method for a group of hosts. thresholds{NAME} = [METHOD[:METHOD-ARG]:]VALUE ... Declare the default threshold(s) for a group of hosts. timeout = FLOAT Set the timeout in seconds of SSH commands to hosts. (default=10.0) ''' ) opt_parser.add_my_options("choice", "rank_method", "thresholds", "timeout") opt_parser.modify_option( 'timeout', help='Set the timeout in seconds of SSH commands to hosts.', ) opts, args = opt_parser.parse_args() report = Reporter(opts.verbosity - opts.quietness) popen = RosePopener(event_handler=report) select = HostSelector(event_handler=report, popen=popen) try: host_score_list = select( names=args, rank_method=opts.rank_method, thresholds=opts.thresholds, ssh_cmd_timeout=opts.timeout, ) except (NoHostError, NoHostSelectError) as exc: report(exc) if opts.debug_mode: traceback.print_exc() sys.exit(1) opts.choice = int(opts.choice) report(choice(host_score_list[0 : opts.choice])[0] + "\n", level=0)
class Analyse: """A comparison engine for Rose.""" def __init__(self, config, opts, args, method_paths, reporter=None, popen=None): if reporter is None: self.reporter = Reporter(opts.verbosity - opts.quietness) else: self.reporter = reporter if popen is None: self.popen = RosePopener(event_handler=self.reporter) else: self.popen = popen self.opts = opts self.args = args self.config = config self.tasks = self.load_tasks() modules = [] for path in method_paths: for filename in glob.glob(path + "/*.py"): modules.append(filename) self.load_user_comparison_modules(modules) def analyse(self): """Perform comparisons given a list of tasks.""" ret_code = 0 for task in self.tasks: if self.check_extract(task): # Internal AnalysisEngine extract+comparison test # Extract data from results and from kgoX task = self.do_extract(task, "result") for i in range(task.numkgofiles): var = "kgo" + str(i + 1) task = self.do_extract(task, var) task = self.do_comparison(task) else: # External program(s) doing test # Command to run command = task.extract result = re.search(r"\$file", command) # If the command contains $file, it is run separately for both # the KGO and the result files if result: # Replace $file token with resultfile or kgofile resultcommand = self._expand_tokens( command, task, "result") kgocommand = self._expand_tokens(command, task, "kgo1") # Run the command on the resultfile task.resultdata = self._run_command(resultcommand) # Run the command on the KGO file task.kgo1data = self._run_command(kgocommand) else: # The command works on both files at the same time # Replace tokens $kgofile and $resultfile for actual values command = self._expand_tokens(command, task) # Run the command task.resultdata = self._run_command(command) # Run the comparison task = self.do_comparison(task) self.reporter(TaskCompletionEvent(task), prefix="[%s]" % (task.userstatus)) if task.numericstatus != PASS: ret_code += 1 return ret_code, self.tasks def check_extract(self, task): """Check if an extract name is present in a user method.""" for _, class_name, _, _ in self.user_methods: if task.extract == class_name: return True return False def do_comparison(self, task): """Run the comparison.""" for module_name, class_name, _, _ in self.user_methods: if task.comparison == class_name: for module in self.modules: if module.__name__ == module_name: comparison_inst = getattr(module, class_name)() getattr(comparison_inst, "run")(task) return task def do_extract(self, task, var): """Extract the specified data.""" for module_name, class_name, _, _ in self.user_methods: if task.extract == class_name: for module in self.modules: if module.__name__ == module_name: extract_inst = getattr(module, class_name)() getattr(extract_inst, "run")(task, var) return task def _run_command(self, command): """Run an external command using metomi.rose.popen.""" output = self.popen.run_ok(command, shell=True)[0] output = "".join(output).splitlines() return output def _expand_tokens(self, inputstring, task, var=None): """Expands tokens $resultfile, $file and $kgoXfile.""" filename = '' if var: filename = getattr(task, var + "file") expansions = {'resultfile': task.resultfile, 'file': filename} for i in range(1, task.numkgofiles + 1): key = "kgo" + str(i) + "file" value = getattr(task, key) expansions[key] = value inputstring = inputstring.format(**expansions) return inputstring def _find_file(self, var, task): """Finds a file given a variable name containing the filename. Given a variable name and task object, this returns the filename it points to, including expanding any * characters with glob. """ filevar = var + "file" if hasattr(task, filevar): configvar = var + "fileconfig" setattr(task, configvar, getattr(task, filevar)) filenames = glob.glob(env_var_process(getattr(task, filevar))) if len(filenames) > 0: setattr(task, filevar, os.path.abspath(filenames[0])) return task def load_tasks(self): """Loads AnalysisTasks from files. Given a list of files, return AnalysisTasks generated from those files. This also expands environment variables in filenames, but saves the original contents for use when writing out config files """ tasks = [] for task in self.config.value.keys(): if task == "env": continue if task.startswith("file:"): continue newtask = AnalysisTask() newtask.name = task value = self.config.get_value([task, "resultfile"]) # If the task is ignored, this will be None, so continue # on to the next task if value is None: continue if "{}" in value: newtask.resultfile = value.replace("{}", self.args[0]) else: newtask.resultfile = value newtask = self._find_file("result", newtask) newtask.extract = self.config.get_value([task, "extract"]) result = re.search(r":", newtask.extract) if result: newtask.subextract = ":".join(newtask.extract.split(":")[1:]) newtask.extract = newtask.extract.split(":")[0] newtask.comparison = self.config.get_value([task, "comparison"]) newtask.tolerance = env_var_process( self.config.get_value([task, "tolerance"])) newtask.warnonfail = self.config.get_value([task, "warnonfail" ]) in ["yes", "true"] # Allow for multiple KGO, e.g. kgo1file, kgo2file, for # statistical comparisons of results newtask.numkgofiles = 0 for i in range(1, MAX_KGO_FILES): kgovar = "kgo" + str(i) kgofilevar = kgovar + "file" if self.config.get([task, kgofilevar]): value = self.config.get([task, kgofilevar])[:] if "{}" in value: setattr( newtask, kgofilevar, value.replace("{}", self.args[0]), ) else: setattr(newtask, kgofilevar, value) newtask.numkgofiles += 1 newtask = self._find_file(kgovar, newtask) else: break tasks.append(newtask) return tasks def load_user_comparison_modules(self, files): """Import comparison modules and store them.""" modules = [] for filename in files: directory = os.path.dirname(filename) if not directory.endswith( USRCOMPARISON_DIRNAME) or not filename.endswith( USRCOMPARISON_EXT): continue comparison_name = os.path.basename(filename).rpartition( USRCOMPARISON_EXT)[0] sys.path.insert(0, os.path.abspath(directory)) try: modules.append(__import__(comparison_name)) except ImportError as exc: self.reporter(exc) sys.path.pop(0) modules.sort(key=str) self.modules = modules user_methods = [] for module in modules: comparison_name = module.__name__ contents = inspect.getmembers(module, inspect.isclass) for obj_name, obj in contents: att_name = "run" if hasattr(obj, att_name) and callable(getattr(obj, att_name)): doc_string = obj.__doc__ user_methods.append( (comparison_name, obj_name, att_name, doc_string)) self.user_methods = user_methods return user_methods def write_config(self, filename, tasks): """Write an analysis config file based on a list of tasks provided""" config = metomi.rose.config.ConfigNode() for task in tasks: sectionname = task.name if task.resultfileconfig: config.set([sectionname, "resultfile"], task.resultfileconfig) for i in range(1, task.numkgofiles + 1): origvar = "kgo" + str(i) + "fileconfig" valvar = "kgo" + str(i) + "file" if hasattr(task, origvar): config.set([sectionname, valvar], getattr(task, origvar)) if task.extract: config.set([sectionname, "extract"], task.extract) if task.subextract: config.set( [sectionname, "extract"], task.extract + ":" + task.subextract, ) if task.comparison: config.set([sectionname, "comparison"], task.comparison) if task.tolerance: config.set([sectionname, "tolerance"], task.tolerance) if task.warnonfail: config.set([sectionname, "warnonfail"], "true") metomi.rose.config.dump(config, filename)
'cylc.sphinx_ext.diff_selection', 'cylc.sphinx_ext.grid_table', 'cylc.sphinx_ext.hieroglyph_patch', 'cylc.sphinx_ext.hieroglyph_theme_addons', 'cylc.sphinx_ext.minicylc', 'cylc.sphinx_ext.practical', 'cylc.sphinx_ext.rtd_theme_addons', 'cylc.sphinx_ext.sub_lang', ] # Select best available SVG image converter. for svg_converter, extension in [('rsvg', 'sphinxcontrib.rsvgconverter'), ('inkscape', 'sphinxcontrib.inkscapeconverter')]: try: assert RosePopener.which(svg_converter) __import__(extension) except (AssertionError, ImportError): # converter or extension not available pass else: extensions.append(extension) break else: # no extensions or converters available, fall-back to default # vector graphics will be converted to bitmaps in all documents extensions.append('sphinx.ext.imgconverter') # Slide (hieroglyph) settings. slide_theme = 'single-level' slide_link_to_html = True
def __init__(self, event_handler=None): self.event_handler = event_handler self.popen = RosePopener(self.event_handler) self.suite_engine_proc = SuiteEngineProcessor.get_processor( event_handler=self.event_handler, popen=self.popen)