def _launch(name, event_handler=None, run_fg=False, *args, **kwargs): popen = RosePopener(event_handler) command = popen.get_cmd(name, *args) kwargs['stdin'] = sys.stdin if run_fg: return popen.run(*command, **kwargs) popen.run_bg(*command, **kwargs)
def test_oserror_has_filename(self): """Does what it says.""" rose_popen = RosePopener() name = "bad-command" try: rose_popen.run(name) except RosePopenError as exc: ose = FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), name) try: self.assertEqual(str(ose), exc.stderr) except AssertionError: # This is horrible, but refers to a bug in some versions of # Python 2.6 - https://bugs.python.org/issue32490 err_msg = ("[Errno 2] No such file or directory:" " 'bad-command': 'bad-command'") self.assertEqual(err_msg, exc.stderr) else: self.fail("should return FileNotFoundError")
def write_source_vc_info(run_source_dir, output=None, popen=None): """Write version control information of sources used in run time. run_source_dir -- The source directory we are interested in. output -- An open file handle or a string containing a writable path. If not specified, use sys.stdout. popen -- A metomi.rose.popen.RosePopener instance for running vc commands. If not specified, use a new local instance. """ if popen is None: popen = RosePopener() if output is None: handle = sys.stdout elif hasattr(output, "write"): handle = output else: handle = open(output, "wb") msg = "%s\n" % run_source_dir write_safely(msg, handle) environ = dict(os.environ) environ["LANG"] = "C" for vcs, args_list in [ ( "svn", [ ["info", "--non-interactive"], ["status", "--non-interactive"], ["diff", "--internal-diff", "--non-interactive"], ], ), ("git", [["describe"], ["status"], ["diff"]]), ]: if not popen.which(vcs): continue cwd = os.getcwd() os.chdir(run_source_dir) try: for args in args_list: cmd = [vcs] + args ret_code, out, _ = popen.run(*cmd, env=environ) if out: write_safely(("#" * 80 + "\n"), handle) write_safely(("# %s\n" % popen.list_to_shell_str(cmd)), handle) write_safely(("#" * 80 + "\n"), handle) write_safely(out, handle) if ret_code: # If cmd fails once, it will likely fail again break finally: os.chdir(cwd)
class StemRunner(object): """Set up options for running a STEM job through Rose.""" def __init__(self, opts, reporter=None, popen=None, fs_util=None): self.opts = opts if reporter is None: self.reporter = Reporter(opts.verbosity - opts.quietness) else: self.reporter = reporter if popen is None: self.popen = RosePopener(event_handler=self.reporter) else: self.popen = popen if fs_util is None: self.fs_util = FileSystemUtil(event_handler=self.reporter) else: self.fs_util = fs_util self.host_selector = HostSelector(event_handler=self.reporter, popen=self.popen) def _add_define_option(self, var, val): """Add a define option passed to the SuiteRunner.""" if self.opts.defines: self.opts.defines.append(SUITE_RC_PREFIX + var + '=' + val) else: self.opts.defines = [SUITE_RC_PREFIX + var + '=' + val] self.reporter(ConfigVariableSetEvent(var, val)) return def _get_base_dir(self, item): """Given a source tree return the following from 'fcm loc-layout': * url * sub_tree * peg_rev * root * project """ ret_code, output, stderr = self.popen.run('fcm', 'loc-layout', item) output = output.decode() if ret_code != 0: raise ProjectNotFoundException(item, stderr) ret = {} for line in output.splitlines(): if ":" not in line: continue key, value = line.split(":", 1) if key: if value: ret[key] = value.strip() return ret def _get_project_from_url(self, source_dict): """Run 'fcm keyword-print' to work out the project name.""" repo = source_dict['root'] if source_dict['project']: repo += '/' + source_dict['project'] kpoutput = self.popen.run('fcm', 'kp', source_dict['url'])[1] project = None for line in kpoutput.splitlines(): if line.rstrip().endswith(repo.encode('UTF-8')): kpresult = re.search(r'^location{primary}\[(.*)\]', line.decode()) if kpresult: project = kpresult.group(1) break return project def _deduce_mirror(self, source_dict, project): """Deduce the mirror location of this source tree.""" # Root location for project proj_root = source_dict['root'] + '/' + source_dict['project'] # Swap project to mirror project = re.sub(r'\.x$', r'.xm', project) mirror_repo = "fcm:" + project # Generate mirror location mirror = re.sub(proj_root, mirror_repo, source_dict['url']) # Remove any sub-tree mirror = re.sub(source_dict['sub_tree'], r'', mirror) mirror = re.sub(r'/@', r'@', mirror) # Add forwards slash after .xm if missing if '.xm/' not in mirror: mirror = re.sub(r'\.xm', r'.xm/', mirror) return mirror def _ascertain_project(self, item): """Set the project name and top-level from 'fcm loc-layout'. Returns: * project name * top-level location of the source tree with revision number * top-level location of the source tree without revision number * revision number """ project = None try: project, item = item.split("=", 1) except ValueError: pass if re.search(r'^\.', item): item = os.path.abspath(os.path.join(os.getcwd(), item)) if project is not None: print("[WARN] Forcing project for '{0}' to be '{1}'".format( item, project)) return project, item, item, '', '' source_dict = self._get_base_dir(item) project = self._get_project_from_url(source_dict) if not project: raise ProjectNotFoundException(item) mirror = self._deduce_mirror(source_dict, project) if 'peg_rev' in source_dict and '@' in item: revision = '@' + source_dict['peg_rev'] base = re.sub(r'@.*', r'', item) else: revision = '' base = item # Remove subtree from base and item if 'sub_tree' in source_dict: item = re.sub(r'(.*)%s/?$' % (source_dict['sub_tree']), r'\1', item, count=1) base = re.sub(r'(.*)%s/?$' % (source_dict['sub_tree']), r'\1', base, count=1) # Remove trailing forwards-slash item = re.sub(r'/$', r'', item) base = re.sub(r'/$', r'', base) # Remove anything after a point project = re.sub(r'\..*', r'', project) return project, item, base, revision, mirror def _generate_name(self): """Generate a suite name from the name of the first source tree.""" try: basedir = self._ascertain_project(os.getcwd())[1] except ProjectNotFoundException: if self.opts.conf_dir: basedir = os.path.abspath(self.opts.conf_dir) else: basedir = os.getcwd() name = os.path.basename(basedir) self.reporter(NameSetEvent(name)) return name def _this_suite(self): """Find the location of the suite in the first source tree.""" # Get base of first source basedir = '' if self.opts.source: basedir = self.opts.source[0] else: basedir = self._ascertain_project(os.getcwd())[1] suitedir = os.path.join(basedir, DEFAULT_TEST_DIR) suitefile = os.path.join(suitedir, "rose-suite.conf") if not os.path.isfile(suitefile): raise RoseSuiteConfNotFoundException(suitedir) self._check_suite_version(suitefile) return suitedir def _read_auto_opts(self): """Read the site metomi.rose.conf file.""" return ResourceLocator.default().get_conf().get_value( ["rose-stem", "automatic-options"]) def _check_suite_version(self, fname): """Check the suite is compatible with this version of metomi.rose-stem. """ if not os.path.isfile(fname): raise RoseSuiteConfNotFoundException(os.path.dirname(fname)) config = metomi.rose.config.load(fname) suite_rose_stem_version = config.get(['ROSE_STEM_VERSION']) if suite_rose_stem_version: suite_rose_stem_version = int(suite_rose_stem_version.value) else: suite_rose_stem_version = None if not suite_rose_stem_version == ROSE_STEM_VERSION: raise RoseStemVersionException(suite_rose_stem_version) def _prepend_localhost(self, url): """Prepend the local hostname to urls which do not point to repository locations.""" if ':' not in url or url.split( ':', 1)[0] not in ['svn', 'fcm', 'http', 'https', 'svn+ssh']: url = self.host_selector.get_local_host() + ':' + url return url def process(self): """Process STEM options into 'rose suite-run' options.""" # Generate options for source trees repos = {} repos_with_hosts = {} if not self.opts.source: self.opts.source = ['.'] self.opts.project = list() for i, url in enumerate(self.opts.source): project, url, base, rev, mirror = self._ascertain_project(url) self.opts.source[i] = url self.opts.project.append(project) # Versions of variables with hostname prepended for working copies url_host = self._prepend_localhost(url) base_host = self._prepend_localhost(base) if project in repos: repos[project].append(url) repos_with_hosts[project].append(url_host) else: repos[project] = [url] repos_with_hosts[project] = [url_host] self._add_define_option('SOURCE_' + project.upper() + '_REV', '"' + rev + '"') self._add_define_option('SOURCE_' + project.upper() + '_BASE', '"' + base + '"') self._add_define_option( 'HOST_SOURCE_' + project.upper() + '_BASE', '"' + base_host + '"') self._add_define_option( 'SOURCE_' + project.upper() + '_MIRROR', '"' + mirror + '"') self.reporter(SourceTreeAddedAsBranchEvent(url)) for project, branches in repos.items(): var = 'SOURCE_' + project.upper() branchstring = RosePopener.list_to_shell_str(branches) self._add_define_option(var, '"' + branchstring + '"') for project, branches in repos_with_hosts.items(): var_host = 'HOST_SOURCE_' + project.upper() branchstring = RosePopener.list_to_shell_str(branches) self._add_define_option(var_host, '"' + branchstring + '"') # Generate the variable containing tasks to run if self.opts.group: if not self.opts.defines: self.opts.defines = [] expanded_groups = [] for i in self.opts.group: expanded_groups.extend(i.split(',')) self.opts.defines.append(SUITE_RC_PREFIX + 'RUN_NAMES=' + str(expanded_groups)) # Load the config file and return any automatic-options auto_opts = self._read_auto_opts() if auto_opts: automatic_options = auto_opts.split() for option in automatic_options: elements = option.split("=") if len(elements) == 2: self._add_define_option(elements[0], '"' + elements[1] + '"') # Change into the suite directory if self.opts.conf_dir: self.reporter(SuiteSelectionEvent(self.opts.conf_dir)) self._check_suite_version( os.path.join(self.opts.conf_dir, 'rose-suite.conf')) else: thissuite = self._this_suite() self.fs_util.chdir(thissuite) self.reporter(SuiteSelectionEvent(thissuite)) # Create a default name for the suite; allow override by user if not self.opts.name: self.opts.name = self._generate_name() return self.opts