def test_ut_sourced_if_used(self):
# Check list of plugins for regexp sourcing common functions and skip them
nonsourcing = []
for plugin in plugins:
if not risu.regexpfile(filename=plugin["plugin"],
regexp=".*common-functions"):
nonsourcing.append(plugin["plugin"])
commonfunctions = []
for script in risu.findplugins(
folders=[os.path.join(risu.risudir, "common.d")],
fileextension=".sh",
):
filename = script["plugin"]
with open(filename, "r") as f:
for line in f:
find = re.match("^(([a-z]+_+)+[a-z]*)", line)
if find and find.groups()[0] != "":
commonfunctions.append(find.groups()[0])
usingcf = []
for plugin in nonsourcing:
for func in commonfunctions:
if risu.regexpfile(filename=plugin, regexp=".*%s" % func):
usingcf.append(plugin)
assert sorted(set(usingcf)) == []
def listplugins(options=None):
"""
List available plugins
:param options: argparse options provided
:return: plugin object generator
"""
prio = 0
if options:
try:
prio = options.prio
except:
pass
if options and options.extraplugintree:
folders = [
pluginsdir,
os.path.join(options.extraplugintree, extension)
]
else:
folders = [pluginsdir]
plugins = risu.findplugins(
folders=folders,
fileextension=".sh",
extension=extension,
prio=prio,
options=options,
)
yield plugins
def listplugins(options=None):
"""
List available plugins
:param options: argparse options provided
:return: plugin object generator
"""
prio = 0
if options:
try:
prio = options.prio
except:
pass
if options and options.extraplugintree:
folders = [
pluginsdir,
os.path.join(options.extraplugintree, extension)
]
else:
folders = [pluginsdir]
yield risu.findplugins(
folders=folders,
executables=True,
extension="tripleo-common-healthcheck",
prio=prio,
options=options,
)
def test_findplugins_positive_filter_exclude(self):
plugins = risu.findplugins(
[testplugins], exclude=["exit_passed", "exit_skipped"]
)
for plugin in plugins:
assert "exit_passed" not in plugin and "exit_skipped" not in plugin
def listplugins(options=None):
"""
List available plugins
:param options: argparse options provided
:return: plugin object generator
"""
prio = 0
if options:
try:
prio = options.prio
except:
pass
if options and options.extraplugintree:
folders = [
pluginsdir,
os.path.join(options.extraplugintree, extension)
]
else:
folders = [pluginsdir]
yield risu.findplugins(
folders=folders,
executables=False,
fileextension=".txt",
extension="rhv-log-collector-analyzer",
prio=prio,
options=options,
)
def test_jsons_for_missbehaviours(self):
mypath = os.path.dirname(__file__)
print(mypath)
jsons = risu.findplugins(folders=[mypath],
executables=False,
fileextension=".json")
flag = 0
for risujson in jsons:
try:
results = json.load(open(risujson["plugin"], "r"))["results"]
except:
print("Skipping json: %s as cannot be loaded by risu" %
risujson)
results = []
for result in results:
data = results[result]["result"]["out"]
if data != "":
print("JSON: %s" % risujson["plugin"])
print("PLUGIN: %s" % results[result]["plugin"])
print("STDOUT: %s" % data)
flag = 1
# Force it to fail after we've printed so we can notice error failing
if "plugins/metadata" in results[result]["plugin"]:
# If it's a metadata plugin, it's expected to have STDOUT
flag = 0
assert flag == 0
def test_findplugins_ext(self):
plugins = []
folder = [os.path.join(risu.risudir, "plugins", "core")]
for each in risu.findplugins(folders=folder,
fileextension=".sh",
include=[".sh"],
exclude=["potato"]):
plugins.append(each)
assert len(plugins) != 0
def listplugins(options=None):
"""
List available plugins
:param options: argparse options provided
:return: plugin object generator
"""
prio = 0
if options:
try:
prio = options.prio
except:
pass
plugins = []
if options and options.extraplugintree:
folders = [
pluginsdir,
os.path.join(options.extraplugintree, extension)
]
else:
folders = [pluginsdir]
for plugin in risu.findplugins(
folders=folders,
executables=False,
fileextension=".json",
extension=extension,
prio=prio,
options=options,
):
filename = plugin["plugin"]
data = json.load(open(filename, "r"))
if "logPath" in data and "rules" in data:
path = data["logPath"]
for rule in data["rules"]:
# Clone plugin dictionary:
newplugin = dict(plugin)
newplugin["name"] = "Check %s for %s" % (path, rule["pattern"])
newplugin["category"] = "node-problem-detector"
newplugin["path"] = "%s%s" % ("${RISU_ROOT}", path)
newplugin["description"] = "%s: %s" % (
plugin["description"],
path.replace("${RISU_ROOT}", ""),
)
newplugin["id"] = "%s%s" % (
plugin["id"],
risu.calcid(string=rule["pattern"]),
)
newplugin["pattern"] = rule["pattern"]
newplugin["reason"] = rule["reason"]
plugins.append(dict(newplugin))
yield plugins
def test_plugins_have_dual_parenthesis_for_if(self):
pluginpath = [os.path.join(risu.risudir, "plugins", "core")]
pluginscit = []
for plugin in risu.findplugins(folders=pluginpath):
filename = plugin["plugin"]
regexp = r"if \( "
if risu.regexpfile(filename=filename, regexp=regexp):
pluginscit.append(filename)
assert len(pluginscit) == 0
def test_domagui_againstjsons(self):
print("Running magui against set of jsons, might take a while...")
# Find all jsons
mypath = os.path.dirname(__file__)
alljsons = risu.findplugins(folders=[mypath],
executables=False,
fileextension=".json")
jsons = []
# Convert from plugin list to json list
for jsonfile in alljsons:
jsons.append(jsonfile["plugin"])
# Call with no arguments
res = magui.domagui(sosreports=jsons, risuplugins=[])
assert res != {}
def listplugins(options=None):
"""
List available plugins
:param options: argparse options provided
:return: plugin object generator
"""
prio = 0
if options:
try:
prio = options.prio
except:
pass
if options and options.extraplugintree:
folders = [pluginsdir, os.path.join(options.extraplugintree, extension)]
else:
folders = [pluginsdir]
plugins = risu.findplugins(
folders=folders,
executables=False,
fileextension=".txt",
extension=extension,
prio=prio,
options=options,
)
# check for multiple files specified as per the 'path' by using "," as separator
newplugins = []
for plugin in plugins:
if "," not in plugin["path"]:
newplugins.append(plugin)
else:
# Path contains ',' so we fake extra plugins for each path
for path in plugin["path"].split(","):
# Clone plugin dictionary:
newplugin = dict(plugin)
newplugin["name"] = "Check %s" % path.replace("${RISU_ROOT}", "")
newplugin["path"] = path
newplugin["description"] = "%s: %s" % (
plugin["description"],
path.replace("${RISU_ROOT}", ""),
)
newplugin["id"] = "%s-%s" % (plugin["id"], risu.calcid(string=path))
newplugins.append(newplugin)
yield newplugins
def test_plugins_have_executable_bit(self):
pluginpath = [os.path.join(risu.risudir, "plugins", "core")]
plugins = []
for folder in pluginpath:
for root, dirnames, filenames in os.walk(folder, followlinks=True):
for filename in filenames:
filepath = os.path.join(root, filename)
if ".risu_tests" not in filepath:
plugins.append(filepath)
plugins = sorted(set(plugins))
pluginscit = []
for plugin in risu.findplugins(folders=pluginpath):
pluginscit.append(plugin["plugin"])
pluginscit = sorted(set(pluginscit))
assert plugins == pluginscit
def test_jsons_for_printresults(self):
mypath = os.path.dirname(__file__)
print(mypath)
jsons = risu.findplugins(folders=[mypath],
executables=False,
fileextension=".json")
risujson = jsons[0]
try:
results = json.load(open(risujson["plugin"], "r"))["results"]
except:
print("Skipping json: %s as cannot be loaded by risu" % risujson)
results = []
options = risu.parse_args(default=True)
risu.printresults(results, options)
assert True
def test_findplugins_positive_filter_include(self):
plugins = risu.findplugins([testplugins], include=["exit_passed"])
assert len(plugins) == 1
import os
import re
import shutil
import subprocess
import risuclient.shell as risu
regexpyear = "[0-9][0-9][0-9][0-9]-"
regexpemail = "\\<(.*@.*)\\>"
# Find all plugins
print("Finding all possible files to modify...")
# plugins = risu.findallplugins()
plugins = risu.findplugins(
folders=[os.path.abspath(os.path.dirname(__file__))],
executables=False,
exclude=[".git", ".tox", ".pyc", ".history", "doc/templates"],
include=[".yml", ".py", ".sh", ".txt"],
)
os.environ["LANG"] = "en_US.UTF-8"
# Iterate over found plugins
for plugin in plugins:
if "risu/plugins" not in plugin["plugin"]:
name = ""
date = ""
command = (
"cd $(dirname %s) && git blame -e %s | awk '{print $2\" \"$3\" \"$4}'|egrep -o '<.*>.*[0-9][0-9][0-9][0-9]-' | sed 's/ */ /g' | cut -d ' ' -f 1-2 | sort -u|grep -v not.committed.yet"
import shutil
import subprocess
import sys
import tempfile
from unittest import TestCase
import maguiclient.magui as magui
import risuclient.shell as risu
sys.path.append(os.path.abspath(os.path.dirname(__file__) + "/" + "../"))
testplugins = os.path.join(risu.risudir, "plugins", "test")
plugins = os.path.join(risu.risudir, "plugins", "core")
folder = os.path.join(os.path.abspath(os.path.dirname(__file__)), "setup")
uttest = risu.findplugins(folders=[folder])
citplugs = risu.findplugins(folders=[plugins])
okay = random.randint(10, 29)
failed = random.randint(30, 49)
skipped = random.randint(50, 69)
info = random.randint(70, 89)
# Setup commands and expected return codes
rcs = {"pass": okay, "fail": failed, "skipped": skipped, "info": info}
class RisuTest(TestCase):
def test_all_plugins_snapshot(self):
tmpdir = tempfile.mkdtemp(prefix="risu-tmp")
tmpdir2 = tempfile.mkdtemp(prefix="risu-tmp")
def main():
"""
Main code stub
"""
options = parse_args()
# Configure ENV language before anything else
os.environ["LANG"] = "%s" % options.lang
# Reinstall language in case it has changed
trad = gettext.translation("risu",
localedir,
fallback=True,
languages=[options.lang])
try:
_ = trad.ugettext
except AttributeError:
_ = trad.gettext
# Configure logging
logging.basicConfig(level=options.loglevel)
if not options.quiet:
show_logo()
# Each argument in sosreport is a sosreport
magplugs, magtriggers = risu.initPymodules(
extensions=risu.getPymodules(options=options, folders=[PluginsFolder]))
if options.list_plugins:
for plugin in magplugs:
print("-", plugin.__name__.split(".")[-1])
if options.description:
desc = plugin.help()
if desc:
print(risu.indent(text=desc, amount=4))
return
# Prefill enabled risu plugins from args
if not risu.extensions:
extensions = risu.initPymodules()[0]
else:
extensions = risu.extensions
# Grab the data
sosreports = options.sosreports
# If we've provided a hosts file, use ansible to grab the data from them
if options.hosts:
ansible = risu.which("ansible-playbook")
if not ansible:
LOG.err(_("No ansible-playbook support found, skipping"))
else:
LOG.info("Grabbing data from remote hosts with Ansible")
# Grab data from ansible hosts
# Disable Ansible retry files creation:
os.environ["ANSIBLE_RETRY_FILES_ENABLED"] = "0"
if options.loglevel == "DEBUG":
# Keep ansible remote files for debug
os.environ["ANSIBLE_KEEP_REMOTE_FILES"] = "1"
command = "%s -i %s %s" % (
ansible,
options.hosts,
os.path.join(maguidir, "remote.yml"),
)
LOG.debug("Running: %s with 600 seconds timeout" % command)
risu.execonshell(filename=command, timeout=600)
# Now check the hosts we got logs from:
hosts = risu.findplugins(
folders=glob.glob("/tmp/risu/hostrun/*"),
executables=False,
fileextension=".json",
)
for host in hosts:
sosreports.append(os.path.dirname(host["plugin"]))
# Get all data from hosts for all plugins, etc
if options.output:
dooutput = options.output
else:
dooutput = False
if len(sosreports) > int(options.max_hosts):
print("Maximum number of sosreports provided, exiting")
sys.exit(0)
risuplugins = []
# Prefill with all available plugins and the ones we want to filter for
for extension in extensions:
risuplugins.extend(extension.listplugins())
global allplugins
allplugins = risuplugins
# By default, flatten plugin list for all extensions
newplugins = []
for each in risuplugins:
newplugins.extend(each)
risuplugins = newplugins
def runmaguiandplugs(
sosreports,
risuplugins,
filename=dooutput,
extranames=None,
serveruri=False,
onlysave=False,
result=None,
anon=False,
grouped={},
):
"""
Runs magui and magui plugins
:param grouped: Grouped results from sosreports to speedup processing (domagui)
:param anon: anonymize results on execution
:param serveruri: Server uri to POST the analysis
:param sosreports: sosreports to process
:param risuplugins: risuplugins to run
:param filename: filename to save to
:param extranames: additional filenames used
:param onlysave: Bool: Defines if we just want to save results
:param result: Results to write to disk
:return: results of execution
"""
start_time = time.time()
if not onlysave and not result:
# Run with all plugins so that we get all data back
grouped = domagui(sosreports=sosreports,
risuplugins=risuplugins,
grouped=grouped)
# Run Magui plugins
result = []
for plugin in magplugs:
plugstart_time = time.time()
# Get output from plugin
data = filterresults(
data=grouped,
triggers=magtriggers[plugin.__name__.split(".")[-1]])
returncode, out, err = plugin.run(data=data,
quiet=options.quiet)
updates = {"rc": returncode, "out": out, "err": err}
subcategory = os.path.split(plugin.__file__)[0].replace(
os.path.join(maguidir, "plugins", ""), "")
if subcategory:
if len(os.path.normpath(subcategory).split(os.sep)) > 1:
category = os.path.normpath(subcategory).split(
os.sep)[0]
else:
category = subcategory
subcategory = ""
else:
category = ""
mydata = {
"plugin":
plugin.__name__.split(".")[-1],
"name":
"magui: %s" %
os.path.basename(plugin.__name__.split(".")[-1]),
"id":
hashlib.sha512(
plugin.__file__.replace(
maguidir, "").encode("UTF-8")).hexdigest(),
"description":
plugin.help(),
"long_name":
plugin.help(),
"result":
updates,
"time":
time.time() - plugstart_time,
"category":
category,
"subcategory":
subcategory,
}
result.append(mydata)
if filename:
branding = _(" ")
risu.write_results(
results=result,
filename=filename,
source="magui",
path=sosreports,
time=time.time() - start_time,
branding=branding,
web=True,
extranames=extranames,
serveruri=serveruri,
anon=anon,
)
return result, grouped
print(_("\nStarting check updates and comparison"))
metadataplugins = []
for plugin in risuplugins:
if plugin["backend"] == "metadata":
metadataplugins.append(plugin)
# Prepare metadata execution to find groups
results, grouped = runmaguiandplugs(
sosreports=sosreports,
risuplugins=metadataplugins,
filename=options.output,
serveruri=options.call_home,
)
# Now we've Magui saved for the whole execution provided in 'results' var
# Start working on autogroups
for result in results:
if result["plugin"] == "metadata-outputs":
autodata = result["result"]["err"]
print(_("\nGenerating autogroups:\n"))
groups = autogroups(autodata)
processedgroups = {}
# TODO(iranzo): Review this
# This code was used to provide a field in json for risu.html to get
# other groups in dropdown, but is not in use so commenting meanwhile
filenames = []
# loop over filenames first so that full results are saved and freed from memory
for group in groups:
basefilename = os.path.splitext(options.output)
filename = basefilename[0] + "-" + group + basefilename[1]
runautogroup = True
for progroup in processedgroups:
if sorted(set(groups[group])) == sorted(
set(processedgroups[progroup])):
runautogroup = False
runautofile = progroup
if runautogroup:
# Analysis will be generated
filenames.append(filename)
print("\nRunning full comparison:... %s" % options.output)
# Run full (not only metadata plugins) so that we've the data stored and save filenames in magui.json
results, grouped = runmaguiandplugs(
sosreports=sosreports,
risuplugins=risuplugins,
extranames=filenames,
filename=options.output,
serveruri=options.call_home,
)
# Here 'grouped' obtained from above contains the full set of data
# Results stored, removing variable to free up memory
del results
# reset list of processed groups
# while len(data) != 0:
# print "loop: ", loop
# loop = loop +1
# target, data, todel = findtarget(data)
processedgroups = {}
basefilename = os.path.splitext(options.output)
while len(groups) != 0:
target, newgroups, todel = findtarget(groups)
if target and target != "":
group = target
filename = basefilename[0] + "-" + group + basefilename[1]
print(_("\nRunning for group: %s" % filename))
runautogroup = True
for progroup in processedgroups:
if groups[target] == processedgroups[progroup]:
runautogroup = False
runautofile = progroup
if runautogroup:
# Analysis was missing for this group, run it
# pass grouped as 'dict' to avoid mutable
newgrouped = copy.deepcopy(grouped)
runmaguiandplugs(
sosreports=groups[target],
risuplugins=risuplugins,
filename=filename,
extranames=filenames,
anon=options.anon,
grouped=newgrouped,
)
else:
# Copy file instead of run as it was already existing
LOG.debug("Copying old file from %s to %s" %
(runautofile, filename))
shutil.copyfile(runautofile, filename)
processedgroups[filename] = groups[target]
if todel:
# We can remove a sosreport from the dataset
for plugin in grouped:
if todel in grouped[plugin]["sosreport"]:
del grouped[plugin]["sosreport"][todel]
del newgroups[target]
# Put remaining groups to work
groups = dict(newgroups)
del groups
del processedgroups
print(_("\nFinished autogroup generation."))
def run(data, quiet=False, options=None): # do not edit this line
"""
Executes plugin
:param quiet: be more silent on returned information
:param data: data to process
:return: returncode, out, err
"""
# prefill plugins we had used:
plugins = []
for item in data:
plugin = {"plugin": data[item]["plugin"], "id": data[item]["id"]}
plugins.append(plugin)
if options and options.extraplugintree:
folders = [
pluginsdir,
os.path.join(options.extraplugintree, extension)
]
else:
folders = [pluginsdir]
# Find available profile definitions
try:
include = options.include
except:
include = None
try:
exclude = options.exclude
except:
exclude = None
profiles = risu.findplugins(
folders=folders,
executables=False,
fileextension=".txt",
include=include,
exclude=exclude,
)
for item in profiles:
uid = risu.getids(plugins=[item])[0]
profile = item["plugin"]
plugin = dict(item)
# Precreate storage for this profile
name = "Profiles: %s" % os.path.basename(
os.path.splitext(profile.replace(pluginsdir, ""))[0])
subcategory = ""
category = name
data[uid] = {
"category": category,
"hash": item["hash"],
"plugin": item["plugin"],
"name": name,
"result": {
"rc": 0,
"err": "",
"out": ""
},
"time": 0,
"backend": "profile",
"id": uid,
"subcategory": subcategory,
}
metadata = {
"description":
risu.regexpfile(filename=plugin["plugin"],
regexp=r"\A# description:")[14:].strip(),
"long_name":
risu.regexpfile(filename=plugin["plugin"],
regexp=r"\A# long_name:")[12:].strip(),
"bugzilla":
risu.regexpfile(filename=plugin["plugin"],
regexp=r"\A# bugzilla:")[11:].strip(),
"priority":
int(
risu.regexpfile(filename=plugin["plugin"],
regexp=r"\A# priority:")[11:].strip() or 0),
}
data[uid].update(metadata)
# start with OK status
okay = int(os.environ["RC_OKAY"])
failed = int(os.environ["RC_FAILED"])
skipped = int(os.environ["RC_SKIPPED"])
info = int(os.environ["RC_INFO"])
# Start asembling data for the plugins relevant for profile
data[uid]["result"]["err"] = ""
ids = plugidsforprofile(profile=profile, plugins=plugins)
new_results = []
overallitems = []
for id in ids:
if id in data:
if "sysinfo" in name and data[id]["result"]["rc"] == skipped:
# Do nothing as we don't want to show skipped in sysinfo
pass
else:
new_results.append({
"plugin_id":
id,
"plugin":
data[id]["plugin"].replace(
os.path.join(risu.risudir, "plugins"), ""),
"err":
data[id]["result"]["err"].strip(),
"rc":
data[id]["result"]["rc"],
})
overallitems.append(data[id]["result"]["rc"])
if "sysinfo" in name:
if okay in overallitems or failed in overallitems or info in overallitems:
overall = info
else:
# No plugins matched, so skip it
overall = skipped
else:
if failed in overallitems:
overall = failed
elif info in overallitems:
overall = info
elif skipped in overallitems:
overall = skipped
else:
overall = okay
data[uid]["result"]["err"] = json.dumps(new_results)
data[uid]["components"] = ids
data[uid]["result"]["rc"] = overall
return data
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import os
import re
import sys
from unittest import TestCase
sys.path.append(os.path.abspath(os.path.dirname(__file__) + "/" + "../"))
import risuclient.shell as risu
testplugins = os.path.join(risu.risudir, "plugins", "test")
pluginsdir = os.path.join(risu.risudir, "plugins", "core")
plugins = risu.findplugins(folders=[pluginsdir])
class RisuTest(TestCase):
def test_ut_sourced_if_used(self):
# Check list of plugins for regexp sourcing common functions and skip them
nonsourcing = []
for plugin in plugins:
if not risu.regexpfile(filename=plugin["plugin"],
regexp=".*common-functions"):
nonsourcing.append(plugin["plugin"])
commonfunctions = []
for script in risu.findplugins(
def test_findplugins_negative(self):
assert risu.findplugins("__does_not_exist__") == []
import sys
import tempfile
from unittest import TestCase
import risuclient.shell as risu
sys.path.append(os.path.abspath(os.path.dirname(__file__) + "/" + "../"))
# To create your own test, update NAME with plugin name and copy this file to test_$NAME.py
NAME = "test_cf_is_lineinfile"
testplugins = os.path.join(risu.risudir, "plugins", "test")
plugins = os.path.join(risu.risudir, "plugins", "core")
folder = os.path.join(os.path.abspath(os.path.dirname(__file__)), "setup")
uttest = risu.findplugins(folders=[folder], include=[NAME])[0]["plugin"]
us = os.path.basename(uttest)
citplugs = risu.findplugins(folders=[folder], include=[us])
# Setup commands and expected return codes
rcs = {
"pass": risu.RC_OKAY,
"fail": risu.RC_FAILED,
"skipped": risu.RC_SKIPPED,
"info": risu.RC_INFO,
}
def runtest(testtype="False"):
"""
Actually run the test for UT
def test_findplugins_positive(self):
assert len(risu.findplugins([testplugins])) != 0
