def _launch(name, event_handler=None, run_fg=False, *args, **kwargs):
popen = RosePopener(event_handler)
command = popen.get_cmd(name, *args)
kwargs['stdin'] = sys.stdin
if run_fg:
return popen.run(*command, **kwargs)
popen.run_bg(*command, **kwargs)
def __init__(self, opts, reporter=None, popen=None, fs_util=None):
self.opts = opts
if reporter is None:
self.reporter = Reporter(opts.verbosity - opts.quietness)
else:
self.reporter = reporter
if popen is None:
self.popen = RosePopener(event_handler=self.reporter)
else:
self.popen = popen
if fs_util is None:
self.fs_util = FileSystemUtil(event_handler=self.reporter)
else:
self.fs_util = fs_util
def process(self):
"""Process STEM options into 'rose suite-run' options."""
# Generate options for source trees
repos = {}
if not self.opts.source:
self.opts.source = ['.']
self.opts.project = list()
for i, url in enumerate(self.opts.source):
project, url, base, rev = self._ascertain_project(url)
self.opts.source[i] = url
self.opts.project.append(project)
if project in repos:
repos[project].append(url)
else:
repos[project] = [ url ]
self._add_define_option('SOURCE_' + project.upper() + '_REV',
'"' + rev + '"')
self._add_define_option('SOURCE_' + project.upper() + '_BASE',
'"' + base + '"')
self.reporter(SourceTreeAddedAsBranchEvent(url))
for project, branches in repos.iteritems():
var = 'SOURCE_' + project.upper()
branchstring = RosePopener.list_to_shell_str(branches)
self._add_define_option(var, '"' + branchstring + '"')
# Generate the variable containing tasks to run
if self.opts.group:
if not self.opts.defines:
self.opts.defines = []
expanded_groups = []
for i in self.opts.group:
expanded_groups.extend(i.split(','))
self.opts.defines.append(SUITE_RC_PREFIX + 'RUN_NAMES=' +
str(expanded_groups))
# Load the config file and return any automatic-options
auto_opts = self._read_site_config_and_return_options()
if auto_opts:
automatic_options = auto_opts.split()
for option in automatic_options:
elements = option.split("=")
if len(elements) == 2:
self._add_define_option(elements[0],
'"' + elements[1] + '"')
# Change into the suite directory
if self.opts.conf_dir:
self.reporter(SuiteSelectionEvent(self.opts.conf_dir))
else:
thissuite = self._this_suite()
self.fs_util.chdir(thissuite)
self.reporter(SuiteSelectionEvent(thissuite))
# Create a default name for the suite; allow override by user
if not self.opts.name:
self.opts.name = self._generate_name()
return self.opts
def __init__(self,
event_handler=None,
popen=None,
fs_util=None,
force_mode=False):
if event_handler is None:
event_handler = self._dummy
self.event_handler = event_handler
if popen is None:
popen = RosePopener(event_handler)
self.popen = popen
if fs_util is None:
fs_util = FileSystemUtil(event_handler)
self.fs_util = fs_util
self.force_mode = force_mode
self._work_dir = None
atexit.register(self._delete_work_dir)
self.subversion_servers_conf = None
subversion_servers_conf = os.getenv("ROSIE_SUBVERSION_SERVERS_CONF")
if subversion_servers_conf:
self.subversion_servers_conf = subversion_servers_conf
else:
subversion_servers_conf = os.path.expanduser(
self.SUBVERSION_SERVERS_CONF)
if os.path.exists(subversion_servers_conf):
self.subversion_servers_conf = subversion_servers_conf
def main():
"""Implement "rose suite-hook" command."""
opt_parser = RoseOptionParser()
opt_parser.add_my_options(
"mail_cc", "mail", "retrieve_job_logs", "shutdown")
opts, args = opt_parser.parse_args()
for key in ["mail_cc"]:
values = []
if getattr(opts, key):
for value in getattr(opts, key):
values.extend(value.split(","))
setattr(opts, key, values)
report = Reporter(opts.verbosity - opts.quietness - 1) # Reduced default
popen = RosePopener(event_handler=report)
suite_engine_proc = SuiteEngineProcessor.get_processor(
event_handler=report, popen=popen)
args = suite_engine_proc.process_suite_hook_args(*args, **vars(opts))
hook = RoseSuiteHook(event_handler=report,
popen=popen,
suite_engine_proc=suite_engine_proc)
hook(*args,
should_mail=opts.mail,
mail_cc_list=opts.mail_cc,
should_shutdown=opts.shutdown,
should_retrieve_job_logs=opts.retrieve_job_logs)
def __init__(self, prefixes=None, prompt_func=None, popen=None,
event_handler=None):
if not event_handler:
event_handler = Reporter()
if not popen:
popen = RosePopener(event_handler=event_handler)
self.event_handler = event_handler
self.popen = popen
self.prompt_func = prompt_func
self.prefixes = []
self.auth_managers = {}
conf = ResourceLocator.default().get_conf()
conf_rosie_id = conf.get(["rosie-id"], no_ignore=True)
if conf_rosie_id is not None:
for key, node in conf_rosie_id.value.items():
if node.is_ignored() or not key.startswith("prefix-ws."):
continue
prefix = key.replace("prefix-ws.", "")
self.auth_managers[prefix] = RosieWSClientAuthManager(
prefix, popen=self.popen, prompt_func=self.prompt_func)
if not prefixes:
prefixes_str = conf_rosie_id.get_value(["prefixes-ws-default"])
if prefixes_str:
prefixes = shlex.split(prefixes_str)
else:
prefixes = sorted(self.auth_managers.keys())
self.set_prefixes(prefixes)
def __init__(self, prefix, popen=None, prompt_func=None):
self.prefix = prefix
root = self._get_conf_value("ws")
if root is None:
raise UndefinedRosiePrefixWS(self.prefix)
if not root.endswith("/"):
root += "/"
self.root = root
urlparse_res = urlparse(self.root)
self.scheme = urlparse_res[0]
self.host = urlparse_res[1]
self.password_orig = None
self.username_orig = None
self.password = None
self.username = None
if popen is None:
popen = RosePopener()
self.popen = popen
self.prompt_func = prompt_func
for password_store_name in self.PASSWORD_STORE_NAMES:
password_store_cls = globals()[password_store_name]
if password_store_cls.usable():
self.password_store = password_store_cls()
break
else:
self.password_store = None
self.requests_kwargs = {}
self._init_https_params()
def __init__(self, event_handler=None, popen=None):
self.event_handler = event_handler
if popen is None:
popen = RosePopener(event_handler=event_handler)
self.popen = popen
self.scorers = {}
self.local_host_strs = None
def write_source_vc_info(run_source_dir, output=None, popen=None):
"""Write version control information of sources used in run time.
run_source_dir -- The source directory we are interested in.
output -- An open file handle or a string containing a writable path.
If not specified, use sys.stdout.
popen -- A rose.popen.RosePopener instance for running vc commands.
If not specified, use a new local instance.
"""
if popen is None:
popen = RosePopener()
if output is None:
handle = sys.stdout
elif hasattr(output, "write"):
handle = output
else:
handle = open(output, "wb")
msg = "%s\n" % run_source_dir
_write_safely(msg, handle)
environ = dict(os.environ)
environ["LANG"] = "C"
for vcs, args_list in [
("svn", [
["info", "--non-interactive"],
["status", "--non-interactive"],
["diff", "--internal-diff", "--non-interactive"]]),
("git", [["describe"], ["status"], ["diff"]])]:
if not popen.which(vcs):
continue
cwd = os.getcwd()
os.chdir(run_source_dir)
try:
for args in args_list:
cmd = [vcs] + args
ret_code, out, _ = popen.run(*cmd, env=environ)
if out:
_write_safely(("#" * 80 + "\n"), handle)
_write_safely(("# %s\n" % popen.list_to_shell_str(cmd)),
handle)
_write_safely(("#" * 80 + "\n"), handle)
_write_safely(out, handle)
if ret_code: # If cmd fails once, it will likely fail again
break
finally:
os.chdir(cwd)
def __init__(self, event_handler=None, popen=None, suite_engine_proc=None):
self.event_handler = event_handler
if popen is None:
popen = RosePopener(event_handler)
self.popen = popen
if suite_engine_proc is None:
suite_engine_proc = SuiteEngineProcessor.get_processor(
event_handler=event_handler, popen=popen)
self.suite_engine_proc = suite_engine_proc
def __init__(self, event_handler=None, popen=None, fs_util=None, **kwargs):
self.event_handler = event_handler
if popen is None:
popen = RosePopener(event_handler)
self.popen = popen
if fs_util is None:
fs_util = FileSystemUtil(event_handler)
self.fs_util = fs_util
self.date_time_oper = RoseDateTimeOperator()
def write_source_vc_info(run_source_dir, output=None, popen=None):
"""Write version control information of sources used in run time.
run_source_dir -- The source directory we are interested in.
output -- An open file handle or a string containing a writable path.
If not specified, use sys.stdout.
popen -- A rose.popen.RosePopener instance for running vc commands.
If not specified, use a new local instance.
"""
if popen is None:
popen = RosePopener()
if output is None:
handle = sys.stdout
elif hasattr(output, "write"):
handle = output
else:
handle = open(output, "wb")
handle.write("%s\n" % run_source_dir)
environ = dict(os.environ)
environ["LANG"] = "C"
for vcs, args_list in [
("svn", [
["info", "--non-interactive"],
["status", "--non-interactive"],
["diff", "--internal-diff", "--non-interactive"]]),
("git", [["describe"], ["status"], ["diff"]])]:
if not popen.which(vcs):
continue
cwd = os.getcwd()
os.chdir(run_source_dir)
try:
for args in args_list:
cmd = [vcs] + args
ret_code, out, _ = popen.run(*cmd, env=environ)
if out:
handle.write("#" * 80 + "\n")
handle.write(("# %s\n" % popen.list_to_shell_str(cmd)))
handle.write("#" * 80 + "\n")
handle.write(out)
if ret_code: # If cmd fails once, it will likely fail again
break
finally:
os.chdir(cwd)
def __init__(self, event_handler=None, popen=None):
if event_handler is None:
event_handler = Reporter()
self.event_handler = event_handler
if popen is None:
popen = RosePopener(self.event_handler)
self.popen = popen
path = os.path.dirname(os.path.dirname(sys.modules["rosie"].__file__))
self.usertools_manager = SchemeHandlersManager(
[path], "rosie.usertools", ["verify_users"])
def __init__(self, event_handler=None, popen=None, fs_util=None):
self.event_handler = event_handler
if popen is None:
popen = RosePopener(event_handler)
self.popen = popen
if fs_util is None:
fs_util = FileSystemUtil(event_handler)
self.fs_util = fs_util
path = os.path.dirname(os.path.dirname(sys.modules["rose"].__file__))
SchemeHandlersManager.__init__(self, [path], "rose.config_processors",
["process"])
def __init__(self, event_handler=None):
self.event_handler = event_handler
self.popen = RosePopener(self.event_handler)
self.fs_util = FileSystemUtil(self.event_handler)
self.config_pm = ConfigProcessorsManager(self.event_handler,
self.popen, self.fs_util)
self.host_selector = HostSelector(self.event_handler, self.popen)
self.suite_engine_proc = SuiteEngineProcessor.get_processor(
event_handler=self.event_handler,
popen=self.popen,
fs_util=self.fs_util)
def __init__(self, event_handler=None, popen=None, fs_util=None):
if event_handler is None:
event_handler = self._dummy
self.event_handler = event_handler
if popen is None:
popen = RosePopener(event_handler)
self.popen = popen
if fs_util is None:
fs_util = FileSystemUtil(event_handler)
self.fs_util = fs_util
self.post_commit_hook = RosieSvnPostCommitHook(
event_handler=event_handler, popen=popen)
def __init__(self, event_handler=None, popen=None, fs_util=None):
self.event_handler = event_handler
if popen is None:
popen = RosePopener(event_handler)
self.popen = popen
if fs_util is None:
fs_util = FileSystemUtil(event_handler)
self.fs_util = fs_util
path = os.path.dirname(os.path.dirname(sys.modules["rose"].__file__))
SchemeHandlersManager.__init__(
self, [path], ns="rose.loc_handlers", attrs=["parse", "pull"],
can_handle="can_pull")
def _launch(name, event_handler=None, run_fg=False, *args, **kwargs):
popen = RosePopener(event_handler)
command = popen.get_cmd(name, *args)
kwargs['stdin'] = sys.stdin
if run_fg:
return popen.run(*command, **kwargs)
popen.run_bg(*command, **kwargs)
def __init__(self, event_handler=None, popen=None, fs_util=None,
host_selector=None, **_):
self.event_handler = event_handler
if popen is None:
popen = RosePopener(event_handler)
self.popen = popen
if fs_util is None:
fs_util = FileSystemUtil(event_handler)
self.fs_util = fs_util
if host_selector is None:
host_selector = HostSelector(event_handler, popen)
self.host_selector = host_selector
self.date_time_oper = RoseDateTimeOperator()
def write_source_vc_info(run_source_dir, output=None, popen=None):
"""Write version control information of sources used in run time.
run_source_dir -- The source directory we are interested in.
output -- An open file handle or a string containing a writable path.
If not specified, use sys.stdout.
popen -- A rose.popen.RosePopener instance for running vc commands.
If not specified, use a new local instance.
"""
if popen is None:
popen = RosePopener()
if output is None:
handle = sys.stdout
elif hasattr(output, "write"):
handle = output
else:
handle = open(output, "wb")
environ = dict(os.environ)
environ["LANG"] = "C"
for vcs, cmds in [("svn", ["info", "status", "diff"]),
("git", ["describe", "status", "diff"])]:
if not popen.which(vcs):
continue
cwd = os.getcwd()
os.chdir(run_source_dir)
try:
for cmd in cmds:
rc, out, err = popen.run(vcs, cmd, env=environ)
if out:
handle.write("#" * 80 + "\n")
handle.write(("# %s %s\n" % (vcs, cmd)).upper())
handle.write("#" * 80 + "\n")
handle.write(out)
if rc: # If cmd fails once, chances are, it will fail again
break
finally:
os.chdir(cwd)
def __init__(self, opts, reporter=None, popen=None, fs_util=None):
self.opts = opts
if reporter is None:
self.reporter = Reporter(opts.verbosity - opts.quietness)
else:
self.reporter = reporter
if popen is None:
self.popen = RosePopener(event_handler=self.reporter)
else:
self.popen = popen
if fs_util is None:
self.fs_util = FileSystemUtil(event_handler=self.reporter)
else:
self.fs_util = fs_util
def process(self):
"""Process STEM options into 'rose suite-run' options."""
# Generate options for source trees
repos = {}
if not self.opts.source:
self.opts.source = ['.']
self.opts.project = list()
for i, url in enumerate(self.opts.source):
project, url, base, rev = self._ascertain_project(url)
self.opts.source[i] = url
self.opts.project.append(project)
if project in repos:
repos[project].append(url)
else:
repos[project] = [ url ]
self._add_define_option('SOURCE_' + project.upper() + '_REV',
'"' + rev + '"')
self._add_define_option('SOURCE_' + project.upper() + '_BASE',
'"' + base + '"')
self.reporter(SourceTreeAddedAsBranchEvent(url))
for project, branches in repos.iteritems():
var = 'SOURCE_' + project.upper()
branchstring = RosePopener.list_to_shell_str(branches)
self._add_define_option(var, '"' + branchstring + '"')
# Generate the variable containing tasks to run
if self.opts.group:
if not self.opts.defines:
self.opts.defines = []
self.opts.defines.append(SUITE_RC_PREFIX + 'RUN_NAMES=' +
str(self.opts.group))
# Change into the suite directory
if self.opts.conf_dir:
self.fs_util.chdir(self.opts.conf_dir)
self.reporter(SuiteSelectionEvent(self.opts.conf_dir))
else:
thissuite = self._this_suite()
self.fs_util.chdir(thissuite)
self.opts.conf_dir = thissuite
self.reporter(SuiteSelectionEvent(thissuite))
# Create a default name for the suite; allow override by user
if not self.opts.name:
self.opts.name = self._generate_name()
return self.opts
def __init__(self,
config,
opts,
args,
method_paths,
reporter=None,
popen=None):
if reporter is None:
self.reporter = Reporter(opts.verbosity - opts.quietness)
else:
self.reporter = reporter
if popen is None:
self.popen = RosePopener(event_handler=self.reporter)
else:
self.popen = popen
self.opts = opts
self.args = args
self.config = config
self.load_tasks()
modules = []
for path in method_paths:
for filename in glob.glob(path + "/*.py"):
modules.append(filename)
self.load_user_comparison_modules(modules)
def write_source_vc_info(run_source_dir, output=None, popen=None):
"""Write version control information of sources used in run time.
run_source_dir -- The source directory we are interested in.
output -- An open file handle or a string containing a writable path.
If not specified, use sys.stdout.
popen -- A rose.popen.RosePopener instance for running vc commands.
If not specified, use a new local instance.
"""
if popen is None:
popen = RosePopener()
if output is None:
handle = sys.stdout
elif hasattr(output, "write"):
handle = output
else:
handle = open(output, "wb")
for vcs, cmds in [("svn", ["info", "status", "diff"]),
("git", ["describe", "status", "diff"])]:
if not popen.which(vcs):
continue
cwd = os.getcwd()
os.chdir(run_source_dir)
try:
for cmd in cmds:
rc, out, err = popen.run(vcs, cmd)
if out:
handle.write("#" * 80 + "\n")
handle.write(("# %s %s\n" % (vcs, cmd)).upper())
handle.write("#" * 80 + "\n")
handle.write(out)
if rc: # If cmd fails once, chances are, it will fail again
break
finally:
os.chdir(cwd)
def __init__(self, config, opts, args, method_paths, reporter=None, popen=None):
if reporter is None:
self.reporter = Reporter(opts.verbosity - opts.quietness)
else:
self.reporter = reporter
if popen is None:
self.popen = RosePopener(event_handler=self.reporter)
else:
self.popen = popen
self.opts = opts
self.args = args
self.config = config
self.load_tasks()
modules = []
for path in method_paths:
for filename in glob.glob(path + "/*.py"):
modules.append(filename)
self.load_user_comparison_modules(modules)
def __init__(self, event_handler=None, popen=None, config_pm=None,
fs_util=None, suite_engine_proc=None):
if not self.CONF_NAME:
self.CONF_NAME = self.NAME
self.event_handler = event_handler
if popen is None:
popen = RosePopener(event_handler)
self.popen = popen
if fs_util is None:
fs_util = FileSystemUtil(event_handler)
self.fs_util = fs_util
if config_pm is None:
config_pm = ConfigProcessorsManager(event_handler, popen, fs_util)
self.config_pm = config_pm
if suite_engine_proc is None:
suite_engine_proc = SuiteEngineProcessor.get_processor(
event_handler=event_handler, popen=popen, fs_util=fs_util)
self.suite_engine_proc = suite_engine_proc
self.conf_tree_loader = ConfigTreeLoader()
def __init__(self,
prefix,
popen=None,
prompt_func=None,
event_handler=None):
self.prefix = prefix
root = self._get_conf_value("ws")
if root is None:
raise UndefinedRosiePrefixWS(self.prefix)
if not root.endswith("/"):
root += "/"
self.root = root
urlparse_res = urlparse(self.root)
self.scheme = urlparse_res[0]
self.host = urlparse_res[1]
self.password_orig = None
self.username_orig = None
self.password = None
self.username = None
if popen is None:
popen = RosePopener()
self.popen = popen
self.prompt_func = prompt_func
if event_handler is None:
self.event_handler = Reporter()
else:
self.event_handler = event_handler
res_loc = ResourceLocator.default()
password_stores_str = res_loc.default().get_conf().get_value(
keys=["rosie-id", "prefix-password-store." + self.prefix],
default=self.PASSWORD_STORES_STR)
for password_store_name in shlex.split(password_stores_str):
password_store_cls = self.PASSWORD_STORE_CLASSES.get(
password_store_name)
if password_store_cls is not None and password_store_cls.usable():
self.password_store = password_store_cls()
break
else:
self.password_store = None
self.requests_kwargs = {}
self._init_https_params()
def main():
"""Implement the "rose host-select" command."""
opt_parser = RoseOptionParser()
opt_parser.add_my_options("choice", "rank_method", "thresholds", "timeout")
opts, args = opt_parser.parse_args()
report = Reporter(opts.verbosity - opts.quietness)
popen = RosePopener(event_handler=report)
select = HostSelector(event_handler=report, popen=popen)
try:
host_score_list = select(names=args,
rank_method=opts.rank_method,
thresholds=opts.thresholds,
ssh_cmd_timeout=opts.timeout)
except (NoHostError, NoHostSelectError) as exc:
report(exc)
if opts.debug_mode:
traceback.print_exc(exc)
sys.exit(1)
opts.choice = int(opts.choice)
report(choice(host_score_list[0:opts.choice])[0] + "\n", level=0)
def _launch(name, event_handler=None, *args, **kwargs):
popen = RosePopener(event_handler)
command = popen.get_cmd(name, *args)
return popen.run_bg(*command, **kwargs)
class Analyse(object):
"""A comparison engine for Rose."""
def __init__(self,
config,
opts,
args,
method_paths,
reporter=None,
popen=None):
if reporter is None:
self.reporter = Reporter(opts.verbosity - opts.quietness)
else:
self.reporter = reporter
if popen is None:
self.popen = RosePopener(event_handler=self.reporter)
else:
self.popen = popen
self.opts = opts
self.args = args
self.config = config
self.load_tasks()
modules = []
for path in method_paths:
for filename in glob.glob(path + "/*.py"):
modules.append(filename)
self.load_user_comparison_modules(modules)
def analyse(self):
"""Perform comparisons given a list of tasks."""
rc = 0
for task in self.tasks:
if self.check_extract(task):
# Internal AnalysisEngine extract+comparison test
# Extract data from results and from kgoX
task = self.do_extract(task, "result")
for i in range(task.numkgofiles):
var = "kgo" + str(i + 1)
task = self.do_extract(task, var)
task = self.do_comparison(task)
else:
# External program(s) doing test
# Command to run
command = task.extract
result = re.search(r"\$file", command)
# If the command contains $file, it is run separately for both
# the KGO and the result files
if result:
# Replace $file token with resultfile or kgofile
resultcommand = self._expand_tokens(
command, task, "result")
kgocommand = self._expand_tokens(command, task, "kgo1")
# Run the command on the resultfile
task.resultdata = self._run_command(resultcommand)
# Run the command on the KGO file
task.kgo1data = self._run_command(kgocommand)
else:
# The command works on both files at the same time
# Replace tokens $kgofile and $resultfile for actual values
command = self._expand_tokens(command, task)
# Run the command
task.resultdata = self._run_command(command)
# Run the comparison
task = self.do_comparison(task)
self.reporter(TaskCompletionEvent(task),
prefix="[%s]" % (task.userstatus))
if task.numericstatus != PASS:
rc += 1
return rc, self.tasks
def check_extract(self, task):
"""Check if an extract name is present in a user method."""
for module_name, class_name, method, help in self.user_methods:
if task.extract == class_name:
return True
return False
def do_comparison(self, task):
"""Run the comparison."""
for module_name, class_name, method, help in self.user_methods:
comparison_name = ".".join([module_name, class_name])
if task.comparison == class_name:
for module in self.modules:
if module.__name__ == module_name:
comparison_inst = getattr(module, class_name)()
comparison_meth = getattr(comparison_inst, "run")(task)
return task
def do_extract(self, task, var):
"""Extract the specified data."""
for module_name, class_name, method, help in self.user_methods:
extract_name = ".".join([module_name, class_name])
if task.extract == class_name:
for module in self.modules:
if module.__name__ == module_name:
extract_inst = getattr(module, class_name)()
extract_meth = getattr(extract_inst, "run")(task, var)
return task
def _run_command(self, command):
"""Run an external command using rose.popen."""
output, stderr = self.popen.run_ok(command, shell=True)
output = "".join(output).splitlines()
return output
def _expand_tokens(self, inputstring, task, var=None):
"""Expands tokens $resultfile, $file and $kgoXfile."""
filename = ''
if var:
filename = getattr(task, var + "file")
expansions = {'resultfile': task.resultfile, 'file': filename}
for i in range(1, task.numkgofiles + 1):
key = "kgo" + str(i) + "file"
value = getattr(task, key)
expansions[key] = value
inputstring = inputstring.format(**expansions)
return inputstring
def _find_file(self, var, task):
"""Finds a file given a variable name containing the filename.
Given a variable name and task object, this returns the filename it
points to, including expanding any * characters with glob.
"""
filevar = var + "file"
if hasattr(task, filevar):
configvar = var + "fileconfig"
setattr(task, configvar, getattr(task, filevar))
filenames = glob.glob(env_var_process(getattr(task, filevar)))
if len(filenames) > 0:
setattr(task, filevar, os.path.abspath(filenames[0]))
return task
def load_tasks(self):
"""Loads AnalysisTasks from files.
Given a list of files, return AnalysisTasks generated from those files.
This also expands environment variables in filenames, but saves the
original contents for use when writing out config files
"""
tasks = []
for task in self.config.value.keys():
if task is "env":
continue
if task.startswith("file:"):
continue
newtask = AnalysisTask()
newtask.name = task
value = self.config.get_value([task, "resultfile"])
# If the task is ignored, this will be None, so continue
# on to the next task
if value is None:
continue
if "{}" in value:
newtask.resultfile = value.replace("{}", self.args[0])
else:
newtask.resultfile = value
newtask = self._find_file("result", newtask)
newtask.extract = self.config.get_value([task, "extract"])
result = re.search(r":", newtask.extract)
if result:
newtask.subextract = ":".join(newtask.extract.split(":")[1:])
newtask.extract = newtask.extract.split(":")[0]
newtask.comparison = self.config.get_value([task, "comparison"])
newtask.tolerance = env_var_process(
self.config.get_value([task, "tolerance"]))
newtask.warnonfail = (self.config.get_value([task, "warnonfail"])
in ["yes", "true"])
# Allow for multiple KGO, e.g. kgo1file, kgo2file, for
# statistical comparisons of results
newtask.numkgofiles = 0
for i in range(1, MAX_KGO_FILES):
kgovar = "kgo" + str(i)
kgofilevar = kgovar + "file"
if self.config.get([task, kgofilevar]):
value = self.config.get([task, kgofilevar])[:]
if "{}" in value:
setattr(newtask, kgofilevar,
value.replace("{}", self.args[0]))
else:
setattr(newtask, kgofilevar, value)
newtask.numkgofiles += 1
newtask = self._find_file(kgovar, newtask)
else:
break
tasks.append(newtask)
self.tasks = tasks
return tasks
def load_user_comparison_modules(self, files):
"""Import comparison modules and store them."""
modules = []
for filename in files:
directory = os.path.dirname(filename)
if (not directory.endswith(USRCOMPARISON_DIRNAME)
or not filename.endswith(USRCOMPARISON_EXT)):
continue
comparison_name = os.path.basename(filename).rpartition(
USRCOMPARISON_EXT)[0]
sys.path.insert(0, os.path.abspath(directory))
try:
modules.append(__import__(comparison_name))
except ImportError as e:
self.reporter(e)
sys.path.pop(0)
modules.sort()
self.modules = modules
user_methods = []
for module in modules:
comparison_name = module.__name__
contents = inspect.getmembers(module, inspect.isclass)
for obj_name, obj in contents:
att_name = "run"
if hasattr(obj, att_name) and callable(getattr(obj, att_name)):
doc_string = obj.__doc__
user_methods.append(
(comparison_name, obj_name, att_name, doc_string))
self.user_methods = user_methods
return user_methods
def write_config(self, filename, tasks):
"""Write an analysis config file based on a list of tasks provided"""
config = rose.config.ConfigNode()
for task in tasks:
sectionname = task.name
if task.resultfileconfig:
config.set([sectionname, "resultfile"], task.resultfileconfig)
for i in range(1, task.numkgofiles + 1):
origvar = "kgo" + str(i) + "fileconfig"
valvar = "kgo" + str(i) + "file"
if hasattr(task, origvar):
config.set([sectionname, valvar], getattr(task, origvar))
if task.extract:
config.set([sectionname, "extract"], task.extract)
if task.subextract:
config.set([sectionname, "extract"],
task.extract + ":" + task.subextract)
if task.comparison:
config.set([sectionname, "comparison"], task.comparison)
if task.tolerance:
config.set([sectionname, "tolerance"], task.tolerance)
if task.warnonfail:
config.set([sectionname, "warnonfail"], "true")
rose.config.dump(config, filename)
'auto_cli_doc',
'cylc_lang',
'minicylc',
'practical',
'rose_lang',
'rose_domain',
'script_include',
'sub_lang'
]
# Select best available SVG image converter.
for svg_converter, extension in [
('rsvg', 'sphinxcontrib.rsvgconverter'),
('inkscape', 'sphinxcontrib.inkscapeconverter')]:
try:
assert RosePopener.which(svg_converter)
__import__(extension)
except (AssertionError, ImportError):
# converter or extension not available
pass
else:
extensions.append(extension)
break
else:
# no extensions or converters available, fall-back to default
# vector graphics will be converted to bitmaps in all documents
extensions.append('sphinx.ext.imgconverter')
# Slide (hieroglyph) settings.
slide_theme = 'single-level'
slide_link_to_html = True
'auto_cli_doc',
'cylc_lang',
'minicylc',
'practical',
'rose_lang',
'rose_domain',
'script_include',
'sub_lang'
]
# Select best available SVG image converter.
for svg_converter, extension in [('rsvg', 'sphinxcontrib.rsvgconverter'),
('inkscape',
'sphinxcontrib.inkscapeconverter')]:
try:
assert RosePopener.which(svg_converter)
__import__(extension)
except (AssertionError, ImportError):
# converter or extension not available
pass
else:
extensions.append(extension)
break
else:
# no extensions or converters available, fall-back to default
# vector graphics will be converted to bitmaps in all documents
extensions.append('sphinx.ext.imgconverter')
# Slide (hieroglyph) settings.
slide_theme = 'single-level'
slide_link_to_html = True
class StemRunner(object):
"""Set up options for running a STEM job through Rose."""
def __init__(self, opts, reporter=None, popen=None, fs_util=None):
self.opts = opts
if reporter is None:
self.reporter = Reporter(opts.verbosity - opts.quietness)
else:
self.reporter = reporter
if popen is None:
self.popen = RosePopener(event_handler = self.reporter)
else:
self.popen = popen
if fs_util is None:
self.fs_util = FileSystemUtil(event_handler = self.reporter)
else:
self.fs_util = fs_util
def _add_define_option(self, var, val):
"""Add a define option passed to the SuiteRunner."""
if self.opts.defines:
self.opts.defines.append(SUITE_RC_PREFIX + var + '=' + val )
else:
self.opts.defines= [ SUITE_RC_PREFIX + var + '=' + val ]
self.reporter(ConfigVariableSetEvent(var, val))
return
def _ascertain_project(self, item):
"""Set the project name and top-level from 'fcm loc-layout'"""
project = ''
if re.search(r'^\.', item):
item = os.path.abspath(os.path.join(os.getcwd(), item))
result = re.search(r'\[(\w+)\]', item)
if result:
project = result.group(1)
item = re.sub(r'\[\w+\]', r'', item)
return project, item
rc, output, stderr = self.popen.run('fcm', 'loc-layout', item)
if rc != 0:
raise ProjectNotFoundException(item, stderr)
result = re.search(r'url:\s*(file|svn|https|http|svn\+ssh)(://.*)',
output)
# Generate a unique name for this project based on fcm kp
if result:
urlstring = result.group(1) + result.group(2)
rc, kpoutput, stderr = self.popen.run('fcm', 'kp', urlstring)
kpresult = re.search(r'location{primary}\[(.*)\]\s*=', kpoutput)
if kpresult:
project = kpresult.group(1)
if not project:
raise ProjectNotFoundException(item)
result = re.search(r'peg_rev:\s*(.*)', output)
if '@' in item and result:
revision = '@' + result.group(1)
base = re.sub(r'@.*', r'', item)
else:
revision = ''
base = item
# If we're in a subdirectory of the source tree, find it and
# remove it leaving the top-level location
result = re.search(r'target:\s*(.*)', output)
target=''
if result:
target = result.group(1)
subtree=''
result2 = re.search(r'sub_tree:\s*(.*)', output)
if result2:
subtree = result2.group(1)
item = re.sub(subtree, r'', target)
# Remove trailing forwards-slash
item = re.sub(r'/$',r'',item)
return project, item, base, revision
def _generate_name(self):
"""Generate a suite name from the name of the first source tree."""
dummy, basedir, dummy2, dummy3 = self._ascertain_project(os.getcwd())
name = os.path.basename(basedir)
return name
def _this_suite(self):
"""Find the location of the suite in the first source tree."""
# Get base of first source
basedir = ''
if self.opts.source:
basedir = self.opts.source[0]
else:
dummy, basedir, dum2, dum3 = self._ascertain_project(os.getcwd())
suitedir = os.path.join(basedir, DEFAULT_TEST_DIR)
suitefile = os.path.join(suitedir, "rose-suite.conf")
if not os.path.isfile(suitefile):
raise RoseSuiteConfNotFoundException(suitedir)
return suitedir
def process(self):
"""Process STEM options into 'rose suite-run' options."""
# Generate options for source trees
repos = {}
if not self.opts.source:
self.opts.source = ['.']
self.opts.project = list()
for i, url in enumerate(self.opts.source):
project, url, base, rev = self._ascertain_project(url)
self.opts.source[i] = url
self.opts.project.append(project)
if project in repos:
repos[project].append(url)
else:
repos[project] = [ url ]
self._add_define_option('SOURCE_' + project.upper() + '_REV',
'"' + rev + '"')
self._add_define_option('SOURCE_' + project.upper() + '_BASE',
'"' + base + '"')
self.reporter(SourceTreeAddedAsBranchEvent(url))
for project, branches in repos.iteritems():
var = 'SOURCE_' + project.upper()
branchstring = RosePopener.list_to_shell_str(branches)
self._add_define_option(var, '"' + branchstring + '"')
# Generate the variable containing tasks to run
if self.opts.group:
if not self.opts.defines:
self.opts.defines = []
self.opts.defines.append(SUITE_RC_PREFIX + 'RUN_NAMES=' +
str(self.opts.group))
# Change into the suite directory
if self.opts.conf_dir:
self.fs_util.chdir(self.opts.conf_dir)
self.reporter(SuiteSelectionEvent(self.opts.conf_dir))
else:
thissuite = self._this_suite()
self.fs_util.chdir(thissuite)
self.opts.conf_dir = thissuite
self.reporter(SuiteSelectionEvent(thissuite))
# Create a default name for the suite; allow override by user
if not self.opts.name:
self.opts.name = self._generate_name()
return self.opts
class StemRunner(object):
"""Set up options for running a STEM job through Rose."""
def __init__(self, opts, reporter=None, popen=None, fs_util=None):
self.opts = opts
if reporter is None:
self.reporter = Reporter(opts.verbosity - opts.quietness)
else:
self.reporter = reporter
if popen is None:
self.popen = RosePopener(event_handler=self.reporter)
else:
self.popen = popen
if fs_util is None:
self.fs_util = FileSystemUtil(event_handler=self.reporter)
else:
self.fs_util = fs_util
def _add_define_option(self, var, val):
"""Add a define option passed to the SuiteRunner."""
if self.opts.defines:
self.opts.defines.append(SUITE_RC_PREFIX + var + '=' + val)
else:
self.opts.defines = [SUITE_RC_PREFIX + var + '=' + val]
self.reporter(ConfigVariableSetEvent(var, val))
return
def _get_base_dir(self, item):
"""Given a source tree return the following from 'fcm loc-layout':
* url
* sub_tree
* peg_rev
* root
* project
"""
rc, output, stderr = self.popen.run('fcm', 'loc-layout', item)
if rc != 0:
raise ProjectNotFoundException(item, stderr)
ret = {}
for line in output.splitlines():
if not ":" in line:
continue
key, value = line.split(":", 1)
if key:
if value:
ret[key] = value.strip()
return ret
def _get_project_from_url(self, source_dict):
"""Run 'fcm keyword-print' to work out the project name."""
repo = source_dict['root']
if source_dict['project']:
repo += '/' + source_dict['project']
rc, kpoutput, stderr = self.popen.run('fcm', 'kp', source_dict['url'])
project = None
for line in kpoutput.splitlines():
if line.rstrip().endswith(repo):
kpresult = re.search(r'^location{primary}\[(.*)\]', line)
if kpresult:
project = kpresult.group(1)
break
return project
def _deduce_mirror(self, source_dict, project):
"""Deduce the mirror location of this source tree."""
# Root location for project
proj_root = source_dict['root'] + '/' + source_dict['project']
# Swap project to mirror
project = re.sub(r'\.x$', r'.xm', project)
mirror_repo = "fcm:" + project
# Generate mirror location
mirror = re.sub(proj_root, mirror_repo, source_dict['url'])
# Add forwards slash after .xm if missing
if '.xm/' not in mirror:
mirror = re.sub(r'\.xm', r'.xm/', mirror)
return mirror
def _ascertain_project(self, item):
"""Set the project name and top-level from 'fcm loc-layout'.
Returns:
* project name
* top-level location of the source tree with revision number
* top-level location of the source tree without revision number
* revision number
"""
project = ''
if re.search(r'^\.', item):
item = os.path.abspath(os.path.join(os.getcwd(), item))
source_dict = self._get_base_dir(item)
project = self._get_project_from_url(source_dict)
mirror = self._deduce_mirror(source_dict, project)
if not project:
raise ProjectNotFoundException(item)
if 'peg_rev' in source_dict and '@' in item:
revision = '@' + source_dict['peg_rev']
base = re.sub(r'@.*', r'', item)
else:
revision = ''
base = item
# Remove subtree from base and item
if 'sub_tree' in source_dict:
item = re.sub(r'(.*)%s/?$' % (source_dict['sub_tree']),
r'\1',
item,
count=1)
base = re.sub(r'(.*)%s/?$' % (source_dict['sub_tree']),
r'\1',
base,
count=1)
# Remove trailing forwards-slash
item = re.sub(r'/$', r'', item)
base = re.sub(r'/$', r'', base)
# Remove anything after a point
project = re.sub(r'\..*', r'', project)
return project, item, base, revision, mirror
def _generate_name(self):
"""Generate a suite name from the name of the first source tree."""
try:
basedir = self._ascertain_project(os.getcwd())[1]
except ProjectNotFoundException:
if self.opts.conf_dir:
basedir = os.path.abspath(self.opts.conf_dir)
else:
basedir = os.getcwd()
name = os.path.basename(basedir)
self.reporter(NameSetEvent(name))
return name
def _this_suite(self):
"""Find the location of the suite in the first source tree."""
# Get base of first source
basedir = ''
if self.opts.source:
basedir = self.opts.source[0]
else:
basedir = self._ascertain_project(os.getcwd())[1]
suitedir = os.path.join(basedir, DEFAULT_TEST_DIR)
suitefile = os.path.join(suitedir, "rose-suite.conf")
if not os.path.isfile(suitefile):
raise RoseSuiteConfNotFoundException(suitedir)
self._check_suite_version(suitefile)
return suitedir
def _read_site_config_and_return_options(self):
"""Read the site rose.conf file."""
return ResourceLocator.default().get_conf().get_value(
["rose-stem", "automatic-options"])
def _check_suite_version(self, fname):
"""Check the suite is compatible with this version of rose-stem."""
if not os.path.isfile(fname):
raise RoseSuiteConfNotFoundException(os.path.dirname(fname))
config = rose.config.load(fname)
suite_rose_stem_version = config.get(['ROSE_STEM_VERSION'])
if suite_rose_stem_version:
suite_rose_stem_version = int(suite_rose_stem_version.value)
else:
suite_rose_stem_version = None
if not suite_rose_stem_version == ROSE_STEM_VERSION:
raise RoseStemVersionException(suite_rose_stem_version)
def process(self):
"""Process STEM options into 'rose suite-run' options."""
# Generate options for source trees
repos = {}
if not self.opts.source:
self.opts.source = ['.']
self.opts.project = list()
for i, url in enumerate(self.opts.source):
project, url, base, rev, mirror = self._ascertain_project(url)
self.opts.source[i] = url
self.opts.project.append(project)
if project in repos:
repos[project].append(url)
else:
repos[project] = [url]
self._add_define_option('SOURCE_' + project.upper() + '_REV',
'"' + rev + '"')
self._add_define_option('SOURCE_' + project.upper() + '_BASE',
'"' + base + '"')
self._add_define_option(
'SOURCE_' + project.upper() + '_MIRROR',
'"' + mirror + '"')
self.reporter(SourceTreeAddedAsBranchEvent(url))
for project, branches in repos.iteritems():
var = 'SOURCE_' + project.upper()
branchstring = RosePopener.list_to_shell_str(branches)
self._add_define_option(var, '"' + branchstring + '"')
# Generate the variable containing tasks to run
if self.opts.group:
if not self.opts.defines:
self.opts.defines = []
expanded_groups = []
for i in self.opts.group:
expanded_groups.extend(i.split(','))
self.opts.defines.append(SUITE_RC_PREFIX + 'RUN_NAMES=' +
str(expanded_groups))
# Load the config file and return any automatic-options
auto_opts = self._read_site_config_and_return_options()
if auto_opts:
automatic_options = auto_opts.split()
for option in automatic_options:
elements = option.split("=")
if len(elements) == 2:
self._add_define_option(elements[0],
'"' + elements[1] + '"')
# Change into the suite directory
if self.opts.conf_dir:
self.reporter(SuiteSelectionEvent(self.opts.conf_dir))
self._check_suite_version(
os.path.join(self.opts.conf_dir, 'rose-suite.conf'))
else:
thissuite = self._this_suite()
self.fs_util.chdir(thissuite)
self.reporter(SuiteSelectionEvent(thissuite))
# Create a default name for the suite; allow override by user
if not self.opts.name:
self.opts.name = self._generate_name()
return self.opts
class StemRunner(object):
"""Set up options for running a STEM job through Rose."""
def __init__(self, opts, reporter=None, popen=None, fs_util=None):
self.opts = opts
if reporter is None:
self.reporter = Reporter(opts.verbosity - opts.quietness)
else:
self.reporter = reporter
if popen is None:
self.popen = RosePopener(event_handler=self.reporter)
else:
self.popen = popen
if fs_util is None:
self.fs_util = FileSystemUtil(event_handler=self.reporter)
else:
self.fs_util = fs_util
def _add_define_option(self, var, val):
"""Add a define option passed to the SuiteRunner."""
if self.opts.defines:
self.opts.defines.append(SUITE_RC_PREFIX + var + '=' + val)
else:
self.opts.defines = [SUITE_RC_PREFIX + var + '=' + val]
self.reporter(ConfigVariableSetEvent(var, val))
return
def _get_base_dir(self, item):
"""Given a source tree return the following from 'fcm loc-layout':
* url
* sub_tree
* peg_rev
* root
* project
"""
rc, output, stderr = self.popen.run('fcm', 'loc-layout', item)
if rc != 0:
raise ProjectNotFoundException(item, stderr)
ret = {}
for line in output.splitlines():
if ":" not in line:
continue
key, value = line.split(":", 1)
if key:
if value:
ret[key] = value.strip()
return ret
def _get_project_from_url(self, source_dict):
"""Run 'fcm keyword-print' to work out the project name."""
repo = source_dict['root']
if source_dict['project']:
repo += '/' + source_dict['project']
rc, kpoutput, stderr = self.popen.run('fcm', 'kp', source_dict['url'])
project = None
for line in kpoutput.splitlines():
if line.rstrip().endswith(repo):
kpresult = re.search(r'^location{primary}\[(.*)\]', line)
if kpresult:
project = kpresult.group(1)
break
return project
def _deduce_mirror(self, source_dict, project):
"""Deduce the mirror location of this source tree."""
# Root location for project
proj_root = source_dict['root'] + '/' + source_dict['project']
# Swap project to mirror
project = re.sub(r'\.x$', r'.xm', project)
mirror_repo = "fcm:" + project
# Generate mirror location
mirror = re.sub(proj_root, mirror_repo, source_dict['url'])
# Add forwards slash after .xm if missing
if '.xm/' not in mirror:
mirror = re.sub(r'\.xm', r'.xm/', mirror)
return mirror
def _ascertain_project(self, item):
"""Set the project name and top-level from 'fcm loc-layout'.
Returns:
* project name
* top-level location of the source tree with revision number
* top-level location of the source tree without revision number
* revision number
"""
project = None
try:
project, item = item.split("=", 1)
except ValueError:
pass
if re.search(r'^\.', item):
item = os.path.abspath(os.path.join(os.getcwd(), item))
if project is not None:
print "[WARN] Forcing project for '{0}' to be '{1}'".format(
item, project)
return project, item, item, '', ''
source_dict = self._get_base_dir(item)
project = self._get_project_from_url(source_dict)
mirror = self._deduce_mirror(source_dict, project)
if not project:
raise ProjectNotFoundException(item)
if 'peg_rev' in source_dict and '@' in item:
revision = '@' + source_dict['peg_rev']
base = re.sub(r'@.*', r'', item)
else:
revision = ''
base = item
# Remove subtree from base and item
if 'sub_tree' in source_dict:
item = re.sub(
r'(.*)%s/?$' % (source_dict['sub_tree']), r'\1', item, count=1)
base = re.sub(
r'(.*)%s/?$' % (source_dict['sub_tree']), r'\1', base, count=1)
# Remove trailing forwards-slash
item = re.sub(r'/$', r'', item)
base = re.sub(r'/$', r'', base)
# Remove anything after a point
project = re.sub(r'\..*', r'', project)
return project, item, base, revision, mirror
def _generate_name(self):
"""Generate a suite name from the name of the first source tree."""
try:
basedir = self._ascertain_project(os.getcwd())[1]
except ProjectNotFoundException:
if self.opts.conf_dir:
basedir = os.path.abspath(self.opts.conf_dir)
else:
basedir = os.getcwd()
name = os.path.basename(basedir)
self.reporter(NameSetEvent(name))
return name
def _this_suite(self):
"""Find the location of the suite in the first source tree."""
# Get base of first source
basedir = ''
if self.opts.source:
basedir = self.opts.source[0]
else:
basedir = self._ascertain_project(os.getcwd())[1]
suitedir = os.path.join(basedir, DEFAULT_TEST_DIR)
suitefile = os.path.join(suitedir, "rose-suite.conf")
if not os.path.isfile(suitefile):
raise RoseSuiteConfNotFoundException(suitedir)
self._check_suite_version(suitefile)
return suitedir
def _read_site_config_and_return_options(self):
"""Read the site rose.conf file."""
return ResourceLocator.default().get_conf().get_value(
["rose-stem", "automatic-options"])
def _check_suite_version(self, fname):
"""Check the suite is compatible with this version of rose-stem."""
if not os.path.isfile(fname):
raise RoseSuiteConfNotFoundException(os.path.dirname(fname))
config = rose.config.load(fname)
suite_rose_stem_version = config.get(['ROSE_STEM_VERSION'])
if suite_rose_stem_version:
suite_rose_stem_version = int(suite_rose_stem_version.value)
else:
suite_rose_stem_version = None
if not suite_rose_stem_version == ROSE_STEM_VERSION:
raise RoseStemVersionException(suite_rose_stem_version)
def process(self):
"""Process STEM options into 'rose suite-run' options."""
# Generate options for source trees
repos = {}
if not self.opts.source:
self.opts.source = ['.']
self.opts.project = list()
for i, url in enumerate(self.opts.source):
project, url, base, rev, mirror = self._ascertain_project(url)
self.opts.source[i] = url
self.opts.project.append(project)
if project in repos:
repos[project].append(url)
else:
repos[project] = [url]
self._add_define_option('SOURCE_' + project.upper() + '_REV',
'"' + rev + '"')
self._add_define_option('SOURCE_' + project.upper() + '_BASE',
'"' + base + '"')
self._add_define_option('SOURCE_' + project.upper() +
'_MIRROR', '"' + mirror + '"')
self.reporter(SourceTreeAddedAsBranchEvent(url))
for project, branches in repos.iteritems():
var = 'SOURCE_' + project.upper()
branchstring = RosePopener.list_to_shell_str(branches)
self._add_define_option(var, '"' + branchstring + '"')
# Generate the variable containing tasks to run
if self.opts.group:
if not self.opts.defines:
self.opts.defines = []
expanded_groups = []
for i in self.opts.group:
expanded_groups.extend(i.split(','))
self.opts.defines.append(SUITE_RC_PREFIX + 'RUN_NAMES=' +
str(expanded_groups))
# Load the config file and return any automatic-options
auto_opts = self._read_site_config_and_return_options()
if auto_opts:
automatic_options = auto_opts.split()
for option in automatic_options:
elements = option.split("=")
if len(elements) == 2:
self._add_define_option(
elements[0], '"' + elements[1] + '"')
# Change into the suite directory
if self.opts.conf_dir:
self.reporter(SuiteSelectionEvent(self.opts.conf_dir))
self._check_suite_version(
os.path.join(self.opts.conf_dir, 'rose-suite.conf'))
else:
thissuite = self._this_suite()
self.fs_util.chdir(thissuite)
self.reporter(SuiteSelectionEvent(thissuite))
# Create a default name for the suite; allow override by user
if not self.opts.name:
self.opts.name = self._generate_name()
return self.opts
class StemRunner(object):
"""Set up options for running a STEM job through Rose."""
def __init__(self, opts, reporter=None, popen=None, fs_util=None):
self.opts = opts
if reporter is None:
self.reporter = Reporter(opts.verbosity - opts.quietness)
else:
self.reporter = reporter
if popen is None:
self.popen = RosePopener(event_handler = self.reporter)
else:
self.popen = popen
if fs_util is None:
self.fs_util = FileSystemUtil(event_handler = self.reporter)
else:
self.fs_util = fs_util
def _add_define_option(self, var, val):
"""Add a define option passed to the SuiteRunner."""
if self.opts.defines:
self.opts.defines.append(SUITE_RC_PREFIX + var + '=' + val )
else:
self.opts.defines= [ SUITE_RC_PREFIX + var + '=' + val ]
self.reporter(ConfigVariableSetEvent(var, val))
return
def _get_base_dir(self, item):
"""Given a source tree return the following from 'fcm loc-layout':
* url
* sub_tree
* peg_rev
* root
* project
"""
rc, output, stderr = self.popen.run('fcm', 'loc-layout', item)
if rc != 0:
raise ProjectNotFoundException(item, stderr)
ret = {}
for line in output.splitlines():
if not ":" in line:
continue
key, value = line.split(":", 1)
if key:
if value:
ret[key] = value.strip()
return ret
def _get_project_from_url(self, source_dict):
"""Run 'fcm keyword-print' to work out the project name."""
repo = source_dict['root']
if source_dict['project']:
repo += '/' + source_dict['project']
rc, kpoutput, stderr = self.popen.run('fcm', 'kp', source_dict['url'])
project = None
for line in kpoutput.splitlines():
if line.rstrip().endswith(repo):
kpresult = re.search(r'^location{primary}\[(.*)\]', line)
if kpresult:
project = kpresult.group(1)
break
return project
def _ascertain_project(self, item):
"""Set the project name and top-level from 'fcm loc-layout'.
Returns:
* project name
* top-level location of the source tree with revision number
* top-level location of the source tree without revision number
* revision number
"""
project = ''
if re.search(r'^\.', item):
item = os.path.abspath(os.path.join(os.getcwd(), item))
source_dict = self._get_base_dir(item)
project = self._get_project_from_url(source_dict)
if not project:
raise ProjectNotFoundException(item)
if 'peg_rev' in source_dict and '@' in item:
revision = '@' + source_dict['peg_rev']
base = re.sub(r'@.*', r'', item)
else:
revision = ''
base = item
# Remove subtree from base and item
if 'sub_tree' in source_dict:
item = item.replace(source_dict['sub_tree'], '', 1)
base = base.replace(source_dict['sub_tree'], '', 1)
# Remove trailing forwards-slash
item = re.sub(r'/$',r'',item)
base = re.sub(r'/$',r'',base)
return project, item, base, revision
def _generate_name(self):
"""Generate a suite name from the name of the first source tree."""
dummy, basedir, dummy2, dummy3 = self._ascertain_project(os.getcwd())
name = os.path.basename(basedir)
return name
def _this_suite(self):
"""Find the location of the suite in the first source tree."""
# Get base of first source
basedir = ''
if self.opts.source:
basedir = self.opts.source[0]
else:
dummy, basedir, dum2, dum3 = self._ascertain_project(os.getcwd())
suitedir = os.path.join(basedir, DEFAULT_TEST_DIR)
suitefile = os.path.join(suitedir, "rose-suite.conf")
if not os.path.isfile(suitefile):
raise RoseSuiteConfNotFoundException(suitedir)
return suitedir
def process(self):
"""Process STEM options into 'rose suite-run' options."""
# Generate options for source trees
repos = {}
if not self.opts.source:
self.opts.source = ['.']
self.opts.project = list()
for i, url in enumerate(self.opts.source):
project, url, base, rev = self._ascertain_project(url)
self.opts.source[i] = url
self.opts.project.append(project)
if project in repos:
repos[project].append(url)
else:
repos[project] = [ url ]
self._add_define_option('SOURCE_' + project.upper() + '_REV',
'"' + rev + '"')
self._add_define_option('SOURCE_' + project.upper() + '_BASE',
'"' + base + '"')
self.reporter(SourceTreeAddedAsBranchEvent(url))
for project, branches in repos.iteritems():
var = 'SOURCE_' + project.upper()
branchstring = RosePopener.list_to_shell_str(branches)
self._add_define_option(var, '"' + branchstring + '"')
# Generate the variable containing tasks to run
if self.opts.group:
if not self.opts.defines:
self.opts.defines = []
self.opts.defines.append(SUITE_RC_PREFIX + 'RUN_NAMES=' +
str(self.opts.group))
# Change into the suite directory
if self.opts.conf_dir:
self.reporter(SuiteSelectionEvent(self.opts.conf_dir))
else:
thissuite = self._this_suite()
self.fs_util.chdir(thissuite)
self.reporter(SuiteSelectionEvent(thissuite))
# Create a default name for the suite; allow override by user
if not self.opts.name:
self.opts.name = self._generate_name()
return self.opts
def __init__(self, event_handler=None):
self.event_handler = event_handler
self.popen = RosePopener(self.event_handler)
self.suite_engine_proc = SuiteEngineProcessor.get_processor(
event_handler=self.event_handler, popen=self.popen)
class Analyse(object):
"""A comparison engine for Rose."""
def __init__(self, config, opts, args, method_paths, reporter=None,
popen=None):
if reporter is None:
self.reporter = Reporter(opts.verbosity - opts.quietness)
else:
self.reporter = reporter
if popen is None:
self.popen = RosePopener(event_handler=self.reporter)
else:
self.popen = popen
self.opts = opts
self.args = args
self.config = config
self.load_tasks()
modules = []
for path in method_paths:
for filename in glob.glob(path + "/*.py"):
modules.append(filename)
self.load_user_comparison_modules(modules)
def analyse(self):
"""Perform comparisons given a list of tasks."""
rc = 0
for task in self.tasks:
if self.check_extract(task):
# Internal AnalysisEngine extract+comparison test
# Extract data from results and from kgoX
task = self.do_extract(task, "result")
for i in range(task.numkgofiles):
var = "kgo" + str(i + 1)
task = self.do_extract(task, var)
task = self.do_comparison(task)
else:
# External program(s) doing test
# Command to run
command = task.extract
result = re.search(r"\$file", command)
# If the command contains $file, it is run separately for both
# the KGO and the result files
if result:
# Replace $file token with resultfile or kgofile
resultcommand = self._expand_tokens(command, task,
"result")
kgocommand = self._expand_tokens(command, task, "kgo1")
# Run the command on the resultfile
task.resultdata = self._run_command(resultcommand)
# Run the command on the KGO file
task.kgo1data = self._run_command(kgocommand)
else:
# The command works on both files at the same time
# Replace tokens $kgofile and $resultfile for actual values
command = self._expand_tokens(command, task)
# Run the command
task.resultdata = self._run_command(command)
# Run the comparison
task = self.do_comparison(task)
self.reporter(TaskCompletionEvent(task),
prefix="[%s]" % (task.userstatus))
if task.numericstatus != PASS:
rc += 1
return rc, self.tasks
def check_extract(self, task):
"""Check if an extract name is present in a user method."""
for module_name, class_name, method, help in self.user_methods:
if task.extract == class_name:
return True
return False
def do_comparison(self, task):
"""Run the comparison."""
for module_name, class_name, method, help in self.user_methods:
comparison_name = ".".join([module_name, class_name])
if task.comparison == class_name:
for module in self.modules:
if module.__name__ == module_name:
comparison_inst = getattr(module, class_name)()
comparison_meth = getattr(comparison_inst, "run")(task)
return task
def do_extract(self, task, var):
"""Extract the specified data."""
for module_name, class_name, method, help in self.user_methods:
extract_name = ".".join([module_name, class_name])
if task.extract == class_name:
for module in self.modules:
if module.__name__ == module_name:
extract_inst = getattr(module, class_name)()
extract_meth = getattr(extract_inst, "run")(task, var)
return task
def _run_command(self, command):
"""Run an external command using rose.popen."""
output, stderr = self.popen.run_ok(command, shell=True)
output = "".join(output).splitlines()
return output
def _expand_tokens(self, inputstring, task, var=None):
"""Expands tokens $resultfile, $file and $kgoXfile."""
filename = ''
if var:
filename = getattr(task, var + "file")
expansions = {'resultfile': task.resultfile, 'file': filename}
for i in range(1, task.numkgofiles + 1):
key = "kgo" + str(i) + "file"
value = getattr(task, key)
expansions[key] = value
inputstring = inputstring.format(**expansions)
return inputstring
def _find_file(self, var, task):
"""Finds a file given a variable name containing the filename.
Given a variable name and task object, this returns the filename it
points to, including expanding any * characters with glob.
"""
filevar = var + "file"
if hasattr(task, filevar):
configvar = var + "fileconfig"
setattr(task, configvar, getattr(task, filevar))
filenames = glob.glob(env_var_process(getattr(task, filevar)))
if len(filenames) > 0:
setattr(task, filevar, os.path.abspath(filenames[0]))
return task
def load_tasks(self):
"""Loads AnalysisTasks from files.
Given a list of files, return AnalysisTasks generated from those files.
This also expands environment variables in filenames, but saves the
original contents for use when writing out config files
"""
tasks = []
for task in self.config.value.keys():
if task is "env":
continue
if task.startswith("file:"):
continue
newtask = AnalysisTask()
newtask.name = task
value = self.config.get_value([task, "resultfile"])
# If the task is ignored, this will be None, so continue
# on to the next task
if value is None:
continue
if "{}" in value:
newtask.resultfile = value.replace("{}", self.args[0])
else:
newtask.resultfile = value
newtask = self._find_file("result", newtask)
newtask.extract = self.config.get_value([task, "extract"])
result = re.search(r":", newtask.extract)
if result:
newtask.subextract = ":".join(newtask.extract.split(":")[1:])
newtask.extract = newtask.extract.split(":")[0]
newtask.comparison = self.config.get_value([task, "comparison"])
newtask.tolerance = env_var_process(self.config.get_value(
[task, "tolerance"]))
newtask.warnonfail = (
self.config.get_value([task, "warnonfail"]) in ["yes", "true"])
# Allow for multiple KGO, e.g. kgo1file, kgo2file, for
# statistical comparisons of results
newtask.numkgofiles = 0
for i in range(1, MAX_KGO_FILES):
kgovar = "kgo" + str(i)
kgofilevar = kgovar + "file"
if self.config.get([task, kgofilevar]):
value = self.config.get([task, kgofilevar])[:]
if "{}" in value:
setattr(
newtask, kgofilevar,
value.replace("{}", self.args[0]))
else:
setattr(newtask, kgofilevar, value)
newtask.numkgofiles += 1
newtask = self._find_file(kgovar, newtask)
else:
break
tasks.append(newtask)
self.tasks = tasks
return tasks
def load_user_comparison_modules(self, files):
"""Import comparison modules and store them."""
modules = []
for filename in files:
directory = os.path.dirname(filename)
if (not directory.endswith(USRCOMPARISON_DIRNAME) or
not filename.endswith(USRCOMPARISON_EXT)):
continue
comparison_name = os.path.basename(filename).rpartition(
USRCOMPARISON_EXT)[0]
sys.path.insert(0, os.path.abspath(directory))
try:
modules.append(__import__(comparison_name))
except ImportError as e:
self.reporter(e)
sys.path.pop(0)
modules.sort()
self.modules = modules
user_methods = []
for module in modules:
comparison_name = module.__name__
contents = inspect.getmembers(module, inspect.isclass)
for obj_name, obj in contents:
att_name = "run"
if hasattr(obj, att_name) and callable(getattr(obj, att_name)):
doc_string = obj.__doc__
user_methods.append((comparison_name, obj_name, att_name,
doc_string))
self.user_methods = user_methods
return user_methods
def write_config(self, filename, tasks):
"""Write an analysis config file based on a list of tasks provided"""
config = rose.config.ConfigNode()
for task in tasks:
sectionname = task.name
if task.resultfileconfig:
config.set([sectionname, "resultfile"], task.resultfileconfig)
for i in range(1, task.numkgofiles + 1):
origvar = "kgo" + str(i) + "fileconfig"
valvar = "kgo" + str(i) + "file"
if hasattr(task, origvar):
config.set([sectionname, valvar], getattr(task, origvar))
if task.extract:
config.set([sectionname, "extract"], task.extract)
if task.subextract:
config.set([sectionname, "extract"],
task.extract + ":" + task.subextract)
if task.comparison:
config.set([sectionname, "comparison"], task.comparison)
if task.tolerance:
config.set([sectionname, "tolerance"], task.tolerance)
if task.warnonfail:
config.set([sectionname, "warnonfail"], "true")
rose.config.dump(config, filename)
def process(self):
"""Process STEM options into 'rose suite-run' options."""
# Generate options for source trees
repos = {}
if not self.opts.source:
self.opts.source = ['.']
self.opts.project = list()
for i, url in enumerate(self.opts.source):
project, url, base, rev, mirror = self._ascertain_project(url)
self.opts.source[i] = url
self.opts.project.append(project)
if project in repos:
repos[project].append(url)
else:
repos[project] = [url]
self._add_define_option('SOURCE_' + project.upper() + '_REV',
'"' + rev + '"')
self._add_define_option('SOURCE_' + project.upper() + '_BASE',
'"' + base + '"')
self._add_define_option(
'SOURCE_' + project.upper() + '_MIRROR',
'"' + mirror + '"')
self.reporter(SourceTreeAddedAsBranchEvent(url))
for project, branches in repos.iteritems():
var = 'SOURCE_' + project.upper()
branchstring = RosePopener.list_to_shell_str(branches)
self._add_define_option(var, '"' + branchstring + '"')
# Generate the variable containing tasks to run
if self.opts.group:
if not self.opts.defines:
self.opts.defines = []
expanded_groups = []
for i in self.opts.group:
expanded_groups.extend(i.split(','))
self.opts.defines.append(SUITE_RC_PREFIX + 'RUN_NAMES=' +
str(expanded_groups))
# Load the config file and return any automatic-options
auto_opts = self._read_site_config_and_return_options()
if auto_opts:
automatic_options = auto_opts.split()
for option in automatic_options:
elements = option.split("=")
if len(elements) == 2:
self._add_define_option(elements[0],
'"' + elements[1] + '"')
# Change into the suite directory
if self.opts.conf_dir:
self.reporter(SuiteSelectionEvent(self.opts.conf_dir))
self._check_suite_version(
os.path.join(self.opts.conf_dir, 'rose-suite.conf'))
else:
thissuite = self._this_suite()
self.fs_util.chdir(thissuite)
self.reporter(SuiteSelectionEvent(thissuite))
# Create a default name for the suite; allow override by user
if not self.opts.name:
self.opts.name = self._generate_name()
return self.opts
