def _launch(name, event_handler=None, run_fg=False, *args, **kwargs):
popen = RosePopener(event_handler)
command = popen.get_cmd(name, *args)
kwargs['stdin'] = sys.stdin
if run_fg:
return popen.run(*command, **kwargs)
popen.run_bg(*command, **kwargs)
def test_oserror_has_filename(self):
"""Does what it says."""
rose_popen = RosePopener()
name = "bad-command"
try:
rose_popen.run(name)
except RosePopenError as exc:
ose = FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT),
name)
try:
self.assertEqual(str(ose), exc.stderr)
except AssertionError:
# This is horrible, but refers to a bug in some versions of
# Python 2.6 - https://bugs.python.org/issue32490
err_msg = ("[Errno 2] No such file or directory:"
" 'bad-command': 'bad-command'")
self.assertEqual(err_msg, exc.stderr)
else:
self.fail("should return FileNotFoundError")
def write_source_vc_info(run_source_dir, output=None, popen=None):
"""Write version control information of sources used in run time.
run_source_dir -- The source directory we are interested in.
output -- An open file handle or a string containing a writable path.
If not specified, use sys.stdout.
popen -- A metomi.rose.popen.RosePopener instance for running vc commands.
If not specified, use a new local instance.
"""
if popen is None:
popen = RosePopener()
if output is None:
handle = sys.stdout
elif hasattr(output, "write"):
handle = output
else:
handle = open(output, "wb")
msg = "%s\n" % run_source_dir
write_safely(msg, handle)
environ = dict(os.environ)
environ["LANG"] = "C"
for vcs, args_list in [
(
"svn",
[
["info", "--non-interactive"],
["status", "--non-interactive"],
["diff", "--internal-diff", "--non-interactive"],
],
),
("git", [["describe"], ["status"], ["diff"]]),
]:
if not popen.which(vcs):
continue
cwd = os.getcwd()
os.chdir(run_source_dir)
try:
for args in args_list:
cmd = [vcs] + args
ret_code, out, _ = popen.run(*cmd, env=environ)
if out:
write_safely(("#" * 80 + "\n"), handle)
write_safely(("# %s\n" % popen.list_to_shell_str(cmd)),
handle)
write_safely(("#" * 80 + "\n"), handle)
write_safely(out, handle)
if ret_code: # If cmd fails once, it will likely fail again
break
finally:
os.chdir(cwd)
class StemRunner(object):
"""Set up options for running a STEM job through Rose."""
def __init__(self, opts, reporter=None, popen=None, fs_util=None):
self.opts = opts
if reporter is None:
self.reporter = Reporter(opts.verbosity - opts.quietness)
else:
self.reporter = reporter
if popen is None:
self.popen = RosePopener(event_handler=self.reporter)
else:
self.popen = popen
if fs_util is None:
self.fs_util = FileSystemUtil(event_handler=self.reporter)
else:
self.fs_util = fs_util
self.host_selector = HostSelector(event_handler=self.reporter,
popen=self.popen)
def _add_define_option(self, var, val):
"""Add a define option passed to the SuiteRunner."""
if self.opts.defines:
self.opts.defines.append(SUITE_RC_PREFIX + var + '=' + val)
else:
self.opts.defines = [SUITE_RC_PREFIX + var + '=' + val]
self.reporter(ConfigVariableSetEvent(var, val))
return
def _get_base_dir(self, item):
"""Given a source tree return the following from 'fcm loc-layout':
* url
* sub_tree
* peg_rev
* root
* project
"""
ret_code, output, stderr = self.popen.run('fcm', 'loc-layout', item)
output = output.decode()
if ret_code != 0:
raise ProjectNotFoundException(item, stderr)
ret = {}
for line in output.splitlines():
if ":" not in line:
continue
key, value = line.split(":", 1)
if key:
if value:
ret[key] = value.strip()
return ret
def _get_project_from_url(self, source_dict):
"""Run 'fcm keyword-print' to work out the project name."""
repo = source_dict['root']
if source_dict['project']:
repo += '/' + source_dict['project']
kpoutput = self.popen.run('fcm', 'kp', source_dict['url'])[1]
project = None
for line in kpoutput.splitlines():
if line.rstrip().endswith(repo.encode('UTF-8')):
kpresult = re.search(r'^location{primary}\[(.*)\]',
line.decode())
if kpresult:
project = kpresult.group(1)
break
return project
def _deduce_mirror(self, source_dict, project):
"""Deduce the mirror location of this source tree."""
# Root location for project
proj_root = source_dict['root'] + '/' + source_dict['project']
# Swap project to mirror
project = re.sub(r'\.x$', r'.xm', project)
mirror_repo = "fcm:" + project
# Generate mirror location
mirror = re.sub(proj_root, mirror_repo, source_dict['url'])
# Remove any sub-tree
mirror = re.sub(source_dict['sub_tree'], r'', mirror)
mirror = re.sub(r'/@', r'@', mirror)
# Add forwards slash after .xm if missing
if '.xm/' not in mirror:
mirror = re.sub(r'\.xm', r'.xm/', mirror)
return mirror
def _ascertain_project(self, item):
"""Set the project name and top-level from 'fcm loc-layout'.
Returns:
* project name
* top-level location of the source tree with revision number
* top-level location of the source tree without revision number
* revision number
"""
project = None
try:
project, item = item.split("=", 1)
except ValueError:
pass
if re.search(r'^\.', item):
item = os.path.abspath(os.path.join(os.getcwd(), item))
if project is not None:
print("[WARN] Forcing project for '{0}' to be '{1}'".format(
item, project))
return project, item, item, '', ''
source_dict = self._get_base_dir(item)
project = self._get_project_from_url(source_dict)
if not project:
raise ProjectNotFoundException(item)
mirror = self._deduce_mirror(source_dict, project)
if 'peg_rev' in source_dict and '@' in item:
revision = '@' + source_dict['peg_rev']
base = re.sub(r'@.*', r'', item)
else:
revision = ''
base = item
# Remove subtree from base and item
if 'sub_tree' in source_dict:
item = re.sub(r'(.*)%s/?$' % (source_dict['sub_tree']),
r'\1',
item,
count=1)
base = re.sub(r'(.*)%s/?$' % (source_dict['sub_tree']),
r'\1',
base,
count=1)
# Remove trailing forwards-slash
item = re.sub(r'/$', r'', item)
base = re.sub(r'/$', r'', base)
# Remove anything after a point
project = re.sub(r'\..*', r'', project)
return project, item, base, revision, mirror
def _generate_name(self):
"""Generate a suite name from the name of the first source tree."""
try:
basedir = self._ascertain_project(os.getcwd())[1]
except ProjectNotFoundException:
if self.opts.conf_dir:
basedir = os.path.abspath(self.opts.conf_dir)
else:
basedir = os.getcwd()
name = os.path.basename(basedir)
self.reporter(NameSetEvent(name))
return name
def _this_suite(self):
"""Find the location of the suite in the first source tree."""
# Get base of first source
basedir = ''
if self.opts.source:
basedir = self.opts.source[0]
else:
basedir = self._ascertain_project(os.getcwd())[1]
suitedir = os.path.join(basedir, DEFAULT_TEST_DIR)
suitefile = os.path.join(suitedir, "rose-suite.conf")
if not os.path.isfile(suitefile):
raise RoseSuiteConfNotFoundException(suitedir)
self._check_suite_version(suitefile)
return suitedir
def _read_auto_opts(self):
"""Read the site metomi.rose.conf file."""
return ResourceLocator.default().get_conf().get_value(
["rose-stem", "automatic-options"])
def _check_suite_version(self, fname):
"""Check the suite is compatible with this version of metomi.rose-stem.
"""
if not os.path.isfile(fname):
raise RoseSuiteConfNotFoundException(os.path.dirname(fname))
config = metomi.rose.config.load(fname)
suite_rose_stem_version = config.get(['ROSE_STEM_VERSION'])
if suite_rose_stem_version:
suite_rose_stem_version = int(suite_rose_stem_version.value)
else:
suite_rose_stem_version = None
if not suite_rose_stem_version == ROSE_STEM_VERSION:
raise RoseStemVersionException(suite_rose_stem_version)
def _prepend_localhost(self, url):
"""Prepend the local hostname to urls which do not point to repository
locations."""
if ':' not in url or url.split(
':', 1)[0] not in ['svn', 'fcm', 'http', 'https', 'svn+ssh']:
url = self.host_selector.get_local_host() + ':' + url
return url
def process(self):
"""Process STEM options into 'rose suite-run' options."""
# Generate options for source trees
repos = {}
repos_with_hosts = {}
if not self.opts.source:
self.opts.source = ['.']
self.opts.project = list()
for i, url in enumerate(self.opts.source):
project, url, base, rev, mirror = self._ascertain_project(url)
self.opts.source[i] = url
self.opts.project.append(project)
# Versions of variables with hostname prepended for working copies
url_host = self._prepend_localhost(url)
base_host = self._prepend_localhost(base)
if project in repos:
repos[project].append(url)
repos_with_hosts[project].append(url_host)
else:
repos[project] = [url]
repos_with_hosts[project] = [url_host]
self._add_define_option('SOURCE_' + project.upper() + '_REV',
'"' + rev + '"')
self._add_define_option('SOURCE_' + project.upper() + '_BASE',
'"' + base + '"')
self._add_define_option(
'HOST_SOURCE_' + project.upper() + '_BASE',
'"' + base_host + '"')
self._add_define_option(
'SOURCE_' + project.upper() + '_MIRROR',
'"' + mirror + '"')
self.reporter(SourceTreeAddedAsBranchEvent(url))
for project, branches in repos.items():
var = 'SOURCE_' + project.upper()
branchstring = RosePopener.list_to_shell_str(branches)
self._add_define_option(var, '"' + branchstring + '"')
for project, branches in repos_with_hosts.items():
var_host = 'HOST_SOURCE_' + project.upper()
branchstring = RosePopener.list_to_shell_str(branches)
self._add_define_option(var_host, '"' + branchstring + '"')
# Generate the variable containing tasks to run
if self.opts.group:
if not self.opts.defines:
self.opts.defines = []
expanded_groups = []
for i in self.opts.group:
expanded_groups.extend(i.split(','))
self.opts.defines.append(SUITE_RC_PREFIX + 'RUN_NAMES=' +
str(expanded_groups))
# Load the config file and return any automatic-options
auto_opts = self._read_auto_opts()
if auto_opts:
automatic_options = auto_opts.split()
for option in automatic_options:
elements = option.split("=")
if len(elements) == 2:
self._add_define_option(elements[0],
'"' + elements[1] + '"')
# Change into the suite directory
if self.opts.conf_dir:
self.reporter(SuiteSelectionEvent(self.opts.conf_dir))
self._check_suite_version(
os.path.join(self.opts.conf_dir, 'rose-suite.conf'))
else:
thissuite = self._this_suite()
self.fs_util.chdir(thissuite)
self.reporter(SuiteSelectionEvent(thissuite))
# Create a default name for the suite; allow override by user
if not self.opts.name:
self.opts.name = self._generate_name()
return self.opts