def main():
"""Implement the "rose config-dump" command."""
opt_parser = RoseOptionParser()
opt_parser.add_my_options("conf_dir", "files", "no_pretty_mode")
opts = opt_parser.parse_args()[0]
verbosity = opts.verbosity - opts.quietness
report = Reporter(verbosity)
fs_util = FileSystemUtil(report)
if opts.conf_dir:
fs_util.chdir(opts.conf_dir)
file_names = []
if opts.files:
file_names = opts.files
else:
for dirpath, _, filenames in os.walk("."):
for filename in fnmatch.filter(filenames, "rose-*.conf"):
path = os.path.join(dirpath, filename)[2:] # remove leading ./
file_names.append(path)
for file_name in file_names:
handle = NamedTemporaryFile()
node = ConfigLoader()(file_name)
if (not opts.no_pretty_mode
and os.path.basename(file_name) != META_CONFIG_NAME):
pretty_format_config(node, ignore_error=True)
ConfigDumper()(node, handle)
handle.seek(0)
if not filecmp.cmp(handle.name, file_name, shallow=False):
report(ConfigDumpEvent(file_name))
ConfigDumper()(node, file_name)
def __init__(self, opts, reporter=None, popen=None, fs_util=None):
self.opts = opts
if reporter is None:
self.reporter = Reporter(opts.verbosity - opts.quietness)
else:
self.reporter = reporter
if popen is None:
self.popen = RosePopener(event_handler=self.reporter)
else:
self.popen = popen
if fs_util is None:
self.fs_util = FileSystemUtil(event_handler=self.reporter)
else:
self.fs_util = fs_util
self.host_selector = HostSelector(event_handler=self.reporter,
popen=self.popen)
def __init__(self,
event_handler=None,
popen=None,
fs_util=None,
force_mode=False):
if event_handler is None:
event_handler = self._dummy
self.event_handler = event_handler
if popen is None:
popen = RosePopener(event_handler)
self.popen = popen
if fs_util is None:
fs_util = FileSystemUtil(event_handler)
self.fs_util = fs_util
self.force_mode = force_mode
self._work_dir = None
atexit.register(self._delete_work_dir)
self.subversion_servers_conf = None
subversion_servers_conf = os.getenv("ROSIE_SUBVERSION_SERVERS_CONF")
if subversion_servers_conf:
self.subversion_servers_conf = subversion_servers_conf
else:
subversion_servers_conf = os.path.expanduser(
self.SUBVERSION_SERVERS_CONF)
if os.path.exists(subversion_servers_conf):
self.subversion_servers_conf = subversion_servers_conf
def __init__(
self,
event_handler=None,
popen=None,
config_pm=None,
fs_util=None,
suite_engine_proc=None,
):
if not self.CONF_NAME:
self.CONF_NAME = self.NAME
self.event_handler = event_handler
if popen is None:
popen = RosePopener(event_handler)
self.popen = popen
if fs_util is None:
fs_util = FileSystemUtil(event_handler)
self.fs_util = fs_util
if config_pm is None:
config_pm = ConfigProcessorsManager(event_handler, popen, fs_util)
self.config_pm = config_pm
if suite_engine_proc is None:
suite_engine_proc = SuiteEngineProcessor.get_processor(
event_handler=event_handler, popen=popen, fs_util=fs_util
)
self.suite_engine_proc = suite_engine_proc
self.conf_tree_loader = ConfigTreeLoader()
def get_suite_name(event_handler=None):
"""Find the top level of a suite directory structure"""
fs_util = FileSystemUtil(event_handler)
conf_dir = os.getcwd()
while True:
if os.path.basename(conf_dir) != "rose-stem":
for tail in [
"rose-suite.conf",
"log/rose-suite-run.conf",
"rose-stem/rose-suite.conf"]:
conf = os.path.join(conf_dir, tail)
if os.path.exists(conf):
return os.path.basename(conf_dir)
up_dir = fs_util.dirname(conf_dir)
if up_dir == conf_dir:
raise SuiteNotFoundError(os.getcwd())
conf_dir = up_dir
def rose_fileinstall(srcdir=None, opts=None, rundir=None):
"""Call Rose Fileinstall.
Args:
srcdir(pathlib.Path):
Search for a ``rose-suite.conf`` file in this location.
rundir (pathlib.Path)
"""
if not rose_config_exists(rundir, opts):
return False
# Load the config tree
config_tree = rose_config_tree_loader(rundir, opts)
if any(i.startswith('file') for i in config_tree.node.value):
try:
startpoint = os.getcwd()
os.chdir(rundir)
except FileNotFoundError as exc:
raise exc
else:
# Carry out imports.
import asyncio
from metomi.rose.config_processor import ConfigProcessorsManager
from metomi.rose.popen import RosePopener
from metomi.rose.reporter import Reporter
from metomi.rose.fs_util import FileSystemUtil
# Update config tree with install location
# NOTE-TO-SELF: value=os.environ["CYLC_WORKFLOW_RUN_DIR"]
config_tree.node = config_tree.node.set(
keys=["file-install-root"], value=str(rundir)
)
# Artificially set rose to verbose.
event_handler = Reporter(3)
fs_util = FileSystemUtil(event_handler)
popen = RosePopener(event_handler)
# Get an Asyncio loop if one doesn't exist:
# Rose may need an event loop to invoke async interfaces,
# doing this here incase we want to go async in cylc-rose.
# See https://github.com/cylc/cylc-rose/pull/130/files
try:
asyncio.get_event_loop()
except RuntimeError:
asyncio.set_event_loop(asyncio.new_event_loop())
# Process fileinstall.
config_pm = ConfigProcessorsManager(event_handler, popen, fs_util)
config_pm(config_tree, "file")
finally:
os.chdir(startpoint)
return config_tree.node
def main():
"""Implement the "rose config-dump" command."""
opt_parser = RoseOptionParser(description='''
Re-dump Rose configuration files in the common format.
Load and dump `"rose-*.conf"` files in place. Apply format-specific
pretty-printing.
By default, it recursively loads and dumps all `rose-*.conf` files in the
current working directory.
EXAMPLES
rose config-dump
rose config-dump -C /path/to/conf/dir
rose config-dump -f /path/to/file1 -f /path/to/file2
''')
opt_parser.add_my_options("conf_dir", "files", "no_pretty_mode")
opts = opt_parser.parse_args()[0]
verbosity = opts.verbosity - opts.quietness
report = Reporter(verbosity)
fs_util = FileSystemUtil(report)
if opts.conf_dir:
fs_util.chdir(opts.conf_dir)
file_names = []
if opts.files:
file_names = opts.files
else:
for dirpath, _, filenames in os.walk("."):
for filename in fnmatch.filter(filenames, "rose-*.conf"):
path = os.path.join(dirpath, filename)[2:] # remove leading ./
file_names.append(path)
for file_name in file_names:
handle = NamedTemporaryFile()
node = ConfigLoader()(file_name)
if (not opts.no_pretty_mode
and os.path.basename(file_name) != META_CONFIG_NAME):
pretty_format_config(node, ignore_error=True)
ConfigDumper()(node, handle)
handle.seek(0)
if not filecmp.cmp(handle.name, file_name, shallow=False):
report(ConfigDumpEvent(file_name))
ConfigDumper()(node, file_name)
def __init__(self, event_handler=None, popen=None, fs_util=None):
if event_handler is None:
event_handler = self._dummy
self.event_handler = event_handler
if popen is None:
popen = RosePopener(event_handler)
self.popen = popen
if fs_util is None:
fs_util = FileSystemUtil(event_handler)
self.fs_util = fs_util
self.post_commit_hook = RosieSvnPostCommitHook(
event_handler=event_handler, popen=popen)
def __init__(self, event_handler=None, popen=None, fs_util=None):
self.event_handler = event_handler
if popen is None:
popen = RosePopener(event_handler)
self.popen = popen
if fs_util is None:
fs_util = FileSystemUtil(event_handler)
self.fs_util = fs_util
path = os.path.dirname(
os.path.dirname(sys.modules["metomi.rose"].__file__))
SchemeHandlersManager.__init__(self, [path], "rose.config_processors",
["process"])
def __init__(self, event_handler=None, popen=None, fs_util=None):
self.event_handler = event_handler
if popen is None:
popen = RosePopener(event_handler)
self.popen = popen
if fs_util is None:
fs_util = FileSystemUtil(event_handler)
self.fs_util = fs_util
path = os.path.dirname(os.path.dirname(sys.modules["rose"].__file__))
SchemeHandlersManager.__init__(
self, [path], ns="rose.loc_handlers", attrs=["parse", "pull"],
can_handle="can_pull")
def __init__(self, event_handler=None, popen=None, fs_util=None,
host_selector=None, **_):
self.event_handler = event_handler
if popen is None:
popen = RosePopener(event_handler)
self.popen = popen
if fs_util is None:
fs_util = FileSystemUtil(event_handler)
self.fs_util = fs_util
if host_selector is None:
host_selector = HostSelector(event_handler, popen)
self.host_selector = host_selector
self.date_time_oper = RoseDateTimeOperator()
def rose_fileinstall(srcdir=None, opts=None, rundir=None):
"""Call Rose Fileinstall.
Args:
srcdir(pathlib.Path):
Search for a ``rose-suite.conf`` file in this location.
rundir (pathlib.Path)
"""
if not rose_config_exists(rundir, opts):
return False
# Load the config tree
config_tree = rose_config_tree_loader(rundir, opts)
if any(i.startswith('file') for i in config_tree.node.value):
try:
startpoint = os.getcwd()
os.chdir(rundir)
except FileNotFoundError as exc:
raise exc
else:
# Carry out imports.
from metomi.rose.config_processor import ConfigProcessorsManager
from metomi.rose.popen import RosePopener
from metomi.rose.reporter import Reporter
from metomi.rose.fs_util import FileSystemUtil
# Update config tree with install location
# NOTE-TO-SELF: value=os.environ["CYLC_WORKFLOW_RUN_DIR"]
config_tree.node = config_tree.node.set(
keys=["file-install-root"], value=str(rundir)
)
# Artificially set rose to verbose.
event_handler = Reporter(3)
fs_util = FileSystemUtil(event_handler)
popen = RosePopener(event_handler)
# Process files
config_pm = ConfigProcessorsManager(event_handler, popen, fs_util)
config_pm(config_tree, "file")
finally:
os.chdir(startpoint)
return config_tree.node
def rose_fileinstall(dir_=None, opts=None, dest_root=None):
"""Call Rose Fileinstall.
Args:
dir_(string or pathlib.Path):
Search for a ``rose-suite.conf`` file in this location.
dest_root (string or pathlib.Path)
"""
if not rose_config_exists(dir_):
return False
# Load the config tree
config_tree = rose_config_tree_loader(dir_, opts)
if any(['file' in i for i in config_tree.node.value]):
# Carry out imports.
from metomi.rose.config_processor import ConfigProcessorsManager
from metomi.rose.popen import RosePopener
from metomi.rose.reporter import Reporter
from metomi.rose.fs_util import FileSystemUtil
# Update config tree with install location
# NOTE-TO-SELF: value=os.environ["CYLC_SUITE_RUN_DIR"]
config_tree.node = config_tree.node.set(
keys=["file-install-root"], value=dest_root
)
# Artificially set rose to verbose.
# TODO - either use Cylc Log as event handler, or get Cylc Verbosity
# settings to pass to Rose Reporter.
event_handler = Reporter(3)
fs_util = FileSystemUtil(event_handler)
popen = RosePopener(event_handler)
# Process files
config_pm = ConfigProcessorsManager(event_handler, popen, fs_util)
config_pm(config_tree, "file")
return True
class StemRunner(object):
"""Set up options for running a STEM job through Rose."""
def __init__(self, opts, reporter=None, popen=None, fs_util=None):
self.opts = opts
if reporter is None:
self.reporter = Reporter(opts.verbosity - opts.quietness)
else:
self.reporter = reporter
if popen is None:
self.popen = RosePopener(event_handler=self.reporter)
else:
self.popen = popen
if fs_util is None:
self.fs_util = FileSystemUtil(event_handler=self.reporter)
else:
self.fs_util = fs_util
self.host_selector = HostSelector(event_handler=self.reporter,
popen=self.popen)
def _add_define_option(self, var, val):
"""Add a define option passed to the SuiteRunner."""
if self.opts.defines:
self.opts.defines.append(SUITE_RC_PREFIX + var + '=' + val)
else:
self.opts.defines = [SUITE_RC_PREFIX + var + '=' + val]
self.reporter(ConfigVariableSetEvent(var, val))
return
def _get_base_dir(self, item):
"""Given a source tree return the following from 'fcm loc-layout':
* url
* sub_tree
* peg_rev
* root
* project
"""
ret_code, output, stderr = self.popen.run('fcm', 'loc-layout', item)
output = output.decode()
if ret_code != 0:
raise ProjectNotFoundException(item, stderr)
ret = {}
for line in output.splitlines():
if ":" not in line:
continue
key, value = line.split(":", 1)
if key:
if value:
ret[key] = value.strip()
return ret
def _get_project_from_url(self, source_dict):
"""Run 'fcm keyword-print' to work out the project name."""
repo = source_dict['root']
if source_dict['project']:
repo += '/' + source_dict['project']
kpoutput = self.popen.run('fcm', 'kp', source_dict['url'])[1]
project = None
for line in kpoutput.splitlines():
if line.rstrip().endswith(repo.encode('UTF-8')):
kpresult = re.search(r'^location{primary}\[(.*)\]',
line.decode())
if kpresult:
project = kpresult.group(1)
break
return project
def _deduce_mirror(self, source_dict, project):
"""Deduce the mirror location of this source tree."""
# Root location for project
proj_root = source_dict['root'] + '/' + source_dict['project']
# Swap project to mirror
project = re.sub(r'\.x$', r'.xm', project)
mirror_repo = "fcm:" + project
# Generate mirror location
mirror = re.sub(proj_root, mirror_repo, source_dict['url'])
# Remove any sub-tree
mirror = re.sub(source_dict['sub_tree'], r'', mirror)
mirror = re.sub(r'/@', r'@', mirror)
# Add forwards slash after .xm if missing
if '.xm/' not in mirror:
mirror = re.sub(r'\.xm', r'.xm/', mirror)
return mirror
def _ascertain_project(self, item):
"""Set the project name and top-level from 'fcm loc-layout'.
Returns:
* project name
* top-level location of the source tree with revision number
* top-level location of the source tree without revision number
* revision number
"""
project = None
try:
project, item = item.split("=", 1)
except ValueError:
pass
if re.search(r'^\.', item):
item = os.path.abspath(os.path.join(os.getcwd(), item))
if project is not None:
print("[WARN] Forcing project for '{0}' to be '{1}'".format(
item, project))
return project, item, item, '', ''
source_dict = self._get_base_dir(item)
project = self._get_project_from_url(source_dict)
if not project:
raise ProjectNotFoundException(item)
mirror = self._deduce_mirror(source_dict, project)
if 'peg_rev' in source_dict and '@' in item:
revision = '@' + source_dict['peg_rev']
base = re.sub(r'@.*', r'', item)
else:
revision = ''
base = item
# Remove subtree from base and item
if 'sub_tree' in source_dict:
item = re.sub(r'(.*)%s/?$' % (source_dict['sub_tree']),
r'\1',
item,
count=1)
base = re.sub(r'(.*)%s/?$' % (source_dict['sub_tree']),
r'\1',
base,
count=1)
# Remove trailing forwards-slash
item = re.sub(r'/$', r'', item)
base = re.sub(r'/$', r'', base)
# Remove anything after a point
project = re.sub(r'\..*', r'', project)
return project, item, base, revision, mirror
def _generate_name(self):
"""Generate a suite name from the name of the first source tree."""
try:
basedir = self._ascertain_project(os.getcwd())[1]
except ProjectNotFoundException:
if self.opts.conf_dir:
basedir = os.path.abspath(self.opts.conf_dir)
else:
basedir = os.getcwd()
name = os.path.basename(basedir)
self.reporter(NameSetEvent(name))
return name
def _this_suite(self):
"""Find the location of the suite in the first source tree."""
# Get base of first source
basedir = ''
if self.opts.source:
basedir = self.opts.source[0]
else:
basedir = self._ascertain_project(os.getcwd())[1]
suitedir = os.path.join(basedir, DEFAULT_TEST_DIR)
suitefile = os.path.join(suitedir, "rose-suite.conf")
if not os.path.isfile(suitefile):
raise RoseSuiteConfNotFoundException(suitedir)
self._check_suite_version(suitefile)
return suitedir
def _read_auto_opts(self):
"""Read the site metomi.rose.conf file."""
return ResourceLocator.default().get_conf().get_value(
["rose-stem", "automatic-options"])
def _check_suite_version(self, fname):
"""Check the suite is compatible with this version of metomi.rose-stem.
"""
if not os.path.isfile(fname):
raise RoseSuiteConfNotFoundException(os.path.dirname(fname))
config = metomi.rose.config.load(fname)
suite_rose_stem_version = config.get(['ROSE_STEM_VERSION'])
if suite_rose_stem_version:
suite_rose_stem_version = int(suite_rose_stem_version.value)
else:
suite_rose_stem_version = None
if not suite_rose_stem_version == ROSE_STEM_VERSION:
raise RoseStemVersionException(suite_rose_stem_version)
def _prepend_localhost(self, url):
"""Prepend the local hostname to urls which do not point to repository
locations."""
if ':' not in url or url.split(
':', 1)[0] not in ['svn', 'fcm', 'http', 'https', 'svn+ssh']:
url = self.host_selector.get_local_host() + ':' + url
return url
def process(self):
"""Process STEM options into 'rose suite-run' options."""
# Generate options for source trees
repos = {}
repos_with_hosts = {}
if not self.opts.source:
self.opts.source = ['.']
self.opts.project = list()
for i, url in enumerate(self.opts.source):
project, url, base, rev, mirror = self._ascertain_project(url)
self.opts.source[i] = url
self.opts.project.append(project)
# Versions of variables with hostname prepended for working copies
url_host = self._prepend_localhost(url)
base_host = self._prepend_localhost(base)
if project in repos:
repos[project].append(url)
repos_with_hosts[project].append(url_host)
else:
repos[project] = [url]
repos_with_hosts[project] = [url_host]
self._add_define_option('SOURCE_' + project.upper() + '_REV',
'"' + rev + '"')
self._add_define_option('SOURCE_' + project.upper() + '_BASE',
'"' + base + '"')
self._add_define_option(
'HOST_SOURCE_' + project.upper() + '_BASE',
'"' + base_host + '"')
self._add_define_option(
'SOURCE_' + project.upper() + '_MIRROR',
'"' + mirror + '"')
self.reporter(SourceTreeAddedAsBranchEvent(url))
for project, branches in repos.items():
var = 'SOURCE_' + project.upper()
branchstring = RosePopener.list_to_shell_str(branches)
self._add_define_option(var, '"' + branchstring + '"')
for project, branches in repos_with_hosts.items():
var_host = 'HOST_SOURCE_' + project.upper()
branchstring = RosePopener.list_to_shell_str(branches)
self._add_define_option(var_host, '"' + branchstring + '"')
# Generate the variable containing tasks to run
if self.opts.group:
if not self.opts.defines:
self.opts.defines = []
expanded_groups = []
for i in self.opts.group:
expanded_groups.extend(i.split(','))
self.opts.defines.append(SUITE_RC_PREFIX + 'RUN_NAMES=' +
str(expanded_groups))
# Load the config file and return any automatic-options
auto_opts = self._read_auto_opts()
if auto_opts:
automatic_options = auto_opts.split()
for option in automatic_options:
elements = option.split("=")
if len(elements) == 2:
self._add_define_option(elements[0],
'"' + elements[1] + '"')
# Change into the suite directory
if self.opts.conf_dir:
self.reporter(SuiteSelectionEvent(self.opts.conf_dir))
self._check_suite_version(
os.path.join(self.opts.conf_dir, 'rose-suite.conf'))
else:
thissuite = self._this_suite()
self.fs_util.chdir(thissuite)
self.reporter(SuiteSelectionEvent(thissuite))
# Create a default name for the suite; allow override by user
if not self.opts.name:
self.opts.name = self._generate_name()
return self.opts
