def run(plugin): # do not edit this line
"""
Executes plugin
:return: returncode, out, err
"""
return citellus.execonshell(filename=plugin["plugin"])
def run(plugin): # do not edit this line
"""
Executes plugin
:param plugin: plugin dictionary
:return: returncode, out, err
"""
ansible = citellus.which("ansible-playbook")
if not ansible:
return citellus.RC_SKIPPED, '', _('ansible-playbook support not found')
if citellus.CITELLUS_LIVE == 0 and citellus.regexpfile(
filename=plugin['plugin'], regexp="CITELLUS_ROOT"):
# We're running in snapshoot and playbook has CITELLUS_ROOT
skipped = 0
elif citellus.CITELLUS_LIVE == 1:
if citellus.regexpfile(
filename=plugin['plugin'],
regexp="CITELLUS_HYBRID") or not citellus.regexpfile(
filename=plugin['plugin'], regexp="CITELLUS_ROOT"):
# We're running in Live mode and either plugin supports HYBRID or has no CITELLUS_ROOT
skipped = 0
else:
# We do not satisfy conditions, exit early
skipped = 1
else:
# We do not satisfy conditions, exit early
skipped = 1
if skipped == 1:
return citellus.RC_SKIPPED, '', _(
'Plugin does not satisfy conditions for running')
command = "%s -i localhost, --connection=local %s" % (ansible,
plugin['plugin'])
# Disable Ansible retry files creation:
os.environ['ANSIBLE_RETRY_FILES_ENABLED'] = "0"
# Call exec to run playbook
returncode, out, err = citellus.execonshell(filename=command)
# Do formatting of results to remove ansible-playbook -i localhost, and adjust return codes to citellus standards
if returncode == 2:
returncode = citellus.RC_FAILED
elif returncode == 0:
returncode = citellus.RC_OKAY
# Convert stdout to stderr for citellus handling
err = out
out = ''
# Rewrite error messages to not contain all playbook execution but just the actual error
if 'FAILED!' in err:
start = err.find('FAILED!', 0) + 11
end = err.find('PLAY RECAP', 0) - 10
newtext = err[start:end]
err = newtext
return returncode, out, err
def run(plugin): # do not edit this line
"""
Executes plugin
:param plugin: plugin dictionary
:return: returncode, out, err
"""
if citellus.CITELLUS_LIVE == 1:
# We're running in Live mode
skipped = 0
else:
# We do not satisfy conditions, exit early
skipped = 1
if skipped == 1:
return citellus.RC_SKIPPED, '', _('Plugin does not satisfy conditions for running')
command = "sh %s " % plugin['plugin']
# Call exec to run playbook
returncode, out, err = citellus.execonshell(filename=command)
# Do formatting of results to adjust return codes to citellus standards
if returncode == 1:
returncode = citellus.RC_FAILED
elif returncode == 0:
returncode = citellus.RC_OKAY
# Convert stdout to stderr for citellus handling
err = out
out = ''
return returncode, out, err
def run(plugin): # do not edit this line
"""
Executes plugin
:param plugin: plugin dictionary
:return: returncode, out, err
"""
rhvlc = citellus.which("rhv-log-collector-analyzer-live")
# rhv-log-collector-analyzer-live --json
if not rhvlc:
return (
citellus.RC_SKIPPED,
"",
_("rhv-log-collector-analyzer-live support not found"),
)
if citellus.CITELLUS_LIVE == 0:
# We're running in snapshoot
skipped = 1
elif citellus.CITELLUS_LIVE == 1:
# We're running in Live mode
skipped = 0
else:
# We do not satisfy conditions, exit early
skipped = 1
if skipped == 1:
return (
citellus.RC_SKIPPED,
"",
_("Plugin does not satisfy conditions for running"),
)
command = "%s --json" % rhvlc
# Call exec to run playbook
returncode, out, err = citellus.execonshell(filename=command)
# Do formatting of results and adjust return codes to citellus standards
if returncode == 2:
returncode = citellus.RC_FAILED
elif returncode == 0:
returncode = citellus.RC_OKAY
# Convert stdout to stderr for citellus handling
try:
err = out
except:
err = "Failed to convert output from log-analyzer"
returncode = citellus.RC_SKIPPED
out = ""
return returncode, out, err
def run(plugin): # do not edit this line
"""
Executes plugin
:param plugin: plugin dictionary
:return: returncode, out, err
"""
gorun = citellus.which("go")
if not gorun:
return citellus.RC_SKIPPED, "", _("Golang support not found")
filename = plugin["plugin"]
# Call exec to run playbook
mypath = os.getcwd()
path = os.path.dirname(filename)
file = os.path.basename(filename)
# Compiling
binary = os.path.splitext(filename)[0]
os.chdir(path)
try:
os.remove(binary)
except:
pass
command = "%s build %s" % (gorun, file)
citellus.execonshell(filename=command)
# Go back to our folder
os.chdir(mypath)
# Running
returncode, out, err = citellus.execonshell(filename=binary)
return returncode, out, err
def main():
"""
Main code stub
"""
options = parse_args()
# Configure ENV language before anything else
os.environ['LANG'] = "%s" % options.lang
# Reinstall language in case it has changed
trad = gettext.translation('citellus',
localedir,
fallback=True,
languages=[options.lang])
try:
_ = trad.ugettext
except AttributeError:
_ = trad.gettext
# Configure logging
logging.basicConfig(level=options.loglevel)
if not options.quiet:
show_logo()
# Each argument in sosreport is a sosreport
magplugs, magtriggers = citellus.initPymodules(
extensions=citellus.getPymodules(options=options,
folders=[PluginsFolder]))
if options.list_plugins:
for plugin in magplugs:
print("-", plugin.__name__.split(".")[-1])
if options.description:
desc = plugin.help()
if desc:
print(citellus.indent(text=desc, amount=4))
return
# Prefill enabled citellus plugins from args
if not citellus.extensions:
extensions = citellus.initPymodules()[0]
else:
extensions = citellus.extensions
# Grab the data
sosreports = options.sosreports
# If we've provided a hosts file, use ansible to grab the data from them
if options.hosts:
ansible = citellus.which("ansible-playbook")
if not ansible:
LOG.err(_("No ansible-playbook support found, skipping"))
else:
LOG.info("Grabbing data from remote hosts with Ansible")
# Grab data from ansible hosts
# Disable Ansible retry files creation:
os.environ['ANSIBLE_RETRY_FILES_ENABLED'] = "0"
if options.loglevel == 'DEBUG':
# Keep ansible remote files for debug
os.environ['ANSIBLE_KEEP_REMOTE_FILES'] = "1"
command = "%s -i %s %s" % (ansible, options.hosts,
os.path.join(maguidir, 'remote.yml'))
LOG.debug("Running: %s with 600 seconds timeout" % command)
citellus.execonshell(filename=command, timeout=600)
# Now check the hosts we got logs from:
hosts = citellus.findplugins(
folders=glob.glob('/tmp/citellus/hostrun/*'),
executables=False,
fileextension='.json')
for host in hosts:
sosreports.append(os.path.dirname(host['plugin']))
# Get all data from hosts for all plugins, etc
if options.output:
dooutput = options.output
else:
dooutput = False
if len(sosreports) > options.max_hosts:
print("Maximum number of sosreports provided, exiting")
sys.exit(0)
citellusplugins = []
# Prefill with all available plugins and the ones we want to filter for
for extension in extensions:
citellusplugins.extend(extension.listplugins())
global allplugins
allplugins = citellusplugins
# By default, flatten plugin list for all extensions
newplugins = []
for each in citellusplugins:
newplugins.extend(each)
citellusplugins = newplugins
def runmaguiandplugs(sosreports,
citellusplugins,
filename=dooutput,
extranames=None,
serveruri=False,
onlysave=False,
result=None,
anon=False):
"""
Runs magui and magui plugins
:param serveruri:
:param sosreports: sosreports to process
:param citellusplugins: citellusplugins to run
:param filename: filename to save to
:param extranames: additional filenames used
:param onlysave: Bool: Defines if we just want to save results
:param result: Results to write to disk
:return: results of execution
"""
start_time = time.time()
if not onlysave and not result:
# Run with all plugins so that we get all data back
grouped = domagui(sosreports=sosreports,
citellusplugins=citellusplugins)
# Run Magui plugins
result = []
for plugin in magplugs:
plugstart_time = time.time()
# Get output from plugin
data = filterresults(
data=grouped,
triggers=magtriggers[plugin.__name__.split(".")[-1]])
returncode, out, err = plugin.run(data=data,
quiet=options.quiet)
updates = {'rc': returncode, 'out': out, 'err': err}
subcategory = os.path.split(plugin.__file__)[0].replace(
os.path.join(maguidir, 'plugins', ''), '')
if subcategory:
if len(os.path.normpath(subcategory).split(os.sep)) > 1:
category = os.path.normpath(subcategory).split(
os.sep)[0]
else:
category = subcategory
subcategory = ""
else:
category = ""
mydata = {
'plugin':
plugin.__name__.split(".")[-1],
'name':
"magui: %s" %
os.path.basename(plugin.__name__.split(".")[-1]),
'id':
hashlib.md5(
plugin.__file__.replace(
maguidir, '').encode('UTF-8')).hexdigest(),
'description':
plugin.help(),
'long_name':
plugin.help(),
'result':
updates,
'time':
time.time() - plugstart_time,
'category':
category,
'subcategory':
subcategory
}
result.append(mydata)
if filename:
branding = _(" ")
citellus.write_results(results=result,
filename=filename,
source='magui',
path=sosreports,
time=time.time() - start_time,
branding=branding,
web=True,
extranames=extranames,
serveruri=serveruri,
anon=anon)
return result
print(_("\nStarting check updates and comparison"))
results = runmaguiandplugs(sosreports=sosreports,
citellusplugins=citellusplugins,
filename=options.output,
serveruri=options.call_home)
# Now we've Magui saved for the whole execution provided in 'results' var
# Start working on autogroups
for result in results:
if result['plugin'] == 'metadata-outputs':
autodata = result['result']['err']
print(_("\nGenerating autogroups:\n"))
groups = autogroups(autodata)
processedgroups = {}
filenames = []
# loop over filenames first so that full results are saved and freed from memory
for group in groups:
basefilename = os.path.splitext(options.output)
filename = basefilename[0] + "-" + group + basefilename[1]
runautogroup = True
for progroup in processedgroups:
if groups[group] == processedgroups[progroup]:
runautogroup = False
runautofile = progroup
if runautogroup:
# Analisys will be generated
filenames.append(filename)
if len(filenames) > 0:
# We've written additional files, so save again magui.json with additional references
runmaguiandplugs(sosreports=sosreports,
citellusplugins=citellusplugins,
filename=options.output,
extranames=filenames,
onlysave=True,
result=results,
anon=options.anon)
# Results stored, removing variable
del results
print("\nFull results written to %s" % options.output)
# reset list of processed groups
processedgroups = {}
for group in groups:
basefilename = os.path.splitext(options.output)
filename = basefilename[0] + "-" + group + basefilename[1]
print(_("\nRunning for group: %s" % filename))
runautogroup = True
for progroup in processedgroups:
if groups[group] == processedgroups[progroup]:
runautogroup = False
runautofile = progroup
if runautogroup:
# Analisys was missing for this group, run
runmaguiandplugs(sosreports=groups[group],
citellusplugins=citellusplugins,
filename=filename,
extranames=options.output,
anon=options.anon)
filenames.append(filename)
else:
# Copy file instead of run as it was already existing
LOG.debug("Copying old file from %s to %s" %
(runautofile, filename))
shutil.copyfile(runautofile, filename)
processedgroups[filename] = groups[group]
del groups
del processedgroups
print(_("\nFinished autogroup generation."))
def main():
"""
Main code stub
"""
options = parse_args()
# Configure ENV language before anything else
os.environ["LANG"] = "%s" % options.lang
# Reinstall language in case it has changed
trad = gettext.translation("citellus",
localedir,
fallback=True,
languages=[options.lang])
try:
_ = trad.ugettext
except AttributeError:
_ = trad.gettext
# Configure logging
logging.basicConfig(level=options.loglevel)
if not options.quiet:
show_logo()
# Each argument in sosreport is a sosreport
magplugs, magtriggers = citellus.initPymodules(
extensions=citellus.getPymodules(options=options,
folders=[PluginsFolder]))
if options.list_plugins:
for plugin in magplugs:
print("-", plugin.__name__.split(".")[-1])
if options.description:
desc = plugin.help()
if desc:
print(citellus.indent(text=desc, amount=4))
return
# Prefill enabled citellus plugins from args
if not citellus.extensions:
extensions = citellus.initPymodules()[0]
else:
extensions = citellus.extensions
# Grab the data
sosreports = options.sosreports
# If we've provided a hosts file, use ansible to grab the data from them
if options.hosts:
ansible = citellus.which("ansible-playbook")
if not ansible:
LOG.err(_("No ansible-playbook support found, skipping"))
else:
LOG.info("Grabbing data from remote hosts with Ansible")
# Grab data from ansible hosts
# Disable Ansible retry files creation:
os.environ["ANSIBLE_RETRY_FILES_ENABLED"] = "0"
if options.loglevel == "DEBUG":
# Keep ansible remote files for debug
os.environ["ANSIBLE_KEEP_REMOTE_FILES"] = "1"
command = "%s -i %s %s" % (
ansible,
options.hosts,
os.path.join(maguidir, "remote.yml"),
)
LOG.debug("Running: %s with 600 seconds timeout" % command)
citellus.execonshell(filename=command, timeout=600)
# Now check the hosts we got logs from:
hosts = citellus.findplugins(
folders=glob.glob("/tmp/citellus/hostrun/*"),
executables=False,
fileextension=".json",
)
for host in hosts:
sosreports.append(os.path.dirname(host["plugin"]))
# Get all data from hosts for all plugins, etc
if options.output:
dooutput = options.output
else:
dooutput = False
if len(sosreports) > int(options.max_hosts):
print("Maximum number of sosreports provided, exiting")
sys.exit(0)
citellusplugins = []
# Prefill with all available plugins and the ones we want to filter for
for extension in extensions:
citellusplugins.extend(extension.listplugins())
global allplugins
allplugins = citellusplugins
# By default, flatten plugin list for all extensions
newplugins = []
for each in citellusplugins:
newplugins.extend(each)
citellusplugins = newplugins
def runmaguiandplugs(
sosreports,
citellusplugins,
filename=dooutput,
extranames=None,
serveruri=False,
onlysave=False,
result=None,
anon=False,
grouped={},
):
"""
Runs magui and magui plugins
:param grouped: Grouped results from sosreports to speedup processing (domagui)
:param anon: anonymize results on execution
:param serveruri: Server uri to POST the analysis
:param sosreports: sosreports to process
:param citellusplugins: citellusplugins to run
:param filename: filename to save to
:param extranames: additional filenames used
:param onlysave: Bool: Defines if we just want to save results
:param result: Results to write to disk
:return: results of execution
"""
start_time = time.time()
if not onlysave and not result:
# Run with all plugins so that we get all data back
grouped = domagui(sosreports=sosreports,
citellusplugins=citellusplugins,
grouped=grouped)
# Run Magui plugins
result = []
for plugin in magplugs:
plugstart_time = time.time()
# Get output from plugin
data = filterresults(
data=grouped,
triggers=magtriggers[plugin.__name__.split(".")[-1]])
returncode, out, err = plugin.run(data=data,
quiet=options.quiet)
updates = {"rc": returncode, "out": out, "err": err}
subcategory = os.path.split(plugin.__file__)[0].replace(
os.path.join(maguidir, "plugins", ""), "")
if subcategory:
if len(os.path.normpath(subcategory).split(os.sep)) > 1:
category = os.path.normpath(subcategory).split(
os.sep)[0]
else:
category = subcategory
subcategory = ""
else:
category = ""
mydata = {
"plugin":
plugin.__name__.split(".")[-1],
"name":
"magui: %s" %
os.path.basename(plugin.__name__.split(".")[-1]),
"id":
hashlib.sha512(
plugin.__file__.replace(
maguidir, "").encode("UTF-8")).hexdigest(),
"description":
plugin.help(),
"long_name":
plugin.help(),
"result":
updates,
"time":
time.time() - plugstart_time,
"category":
category,
"subcategory":
subcategory,
}
result.append(mydata)
if filename:
branding = _(" ")
citellus.write_results(
results=result,
filename=filename,
source="magui",
path=sosreports,
time=time.time() - start_time,
branding=branding,
web=True,
extranames=extranames,
serveruri=serveruri,
anon=anon,
)
return result, grouped
print(_("\nStarting check updates and comparison"))
metadataplugins = []
for plugin in citellusplugins:
if plugin["backend"] == "metadata":
metadataplugins.append(plugin)
# Prepare metadata execution to find groups
results, grouped = runmaguiandplugs(
sosreports=sosreports,
citellusplugins=metadataplugins,
filename=options.output,
serveruri=options.call_home,
)
# Now we've Magui saved for the whole execution provided in 'results' var
# Start working on autogroups
for result in results:
if result["plugin"] == "metadata-outputs":
autodata = result["result"]["err"]
print(_("\nGenerating autogroups:\n"))
groups = autogroups(autodata)
processedgroups = {}
# TODO(iranzo): Review this
# This code was used to provide a field in json for citellus.html to get
# other groups in dropdown, but is not in use so commenting meanwhile
filenames = []
# loop over filenames first so that full results are saved and freed from memory
for group in groups:
basefilename = os.path.splitext(options.output)
filename = basefilename[0] + "-" + group + basefilename[1]
runautogroup = True
for progroup in processedgroups:
if sorted(set(groups[group])) == sorted(
set(processedgroups[progroup])):
runautogroup = False
runautofile = progroup
if runautogroup:
# Analysis will be generated
filenames.append(filename)
print("\nRunning full comparison:... %s" % options.output)
# Run full (not only metadata plugins) so that we've the data stored and save filenames in magui.json
results, grouped = runmaguiandplugs(
sosreports=sosreports,
citellusplugins=citellusplugins,
extranames=filenames,
filename=options.output,
serveruri=options.call_home,
)
# Here 'grouped' obtained from above contains the full set of data
# Results stored, removing variable to free up memory
del results
# reset list of processed groups
# while len(data) != 0:
# print "loop: ", loop
# loop = loop +1
# target, data, todel = findtarget(data)
processedgroups = {}
basefilename = os.path.splitext(options.output)
while len(groups) != 0:
target, newgroups, todel = findtarget(groups)
group = target
filename = basefilename[0] + "-" + group + basefilename[1]
print(_("\nRunning for group: %s" % filename))
runautogroup = True
for progroup in processedgroups:
if groups[target] == processedgroups[progroup]:
runautogroup = False
runautofile = progroup
if runautogroup:
# Analysis was missing for this group, run it
# pass grouped as 'dict' to avoid mutable
newgrouped = copy.deepcopy(grouped)
runmaguiandplugs(
sosreports=groups[target],
citellusplugins=citellusplugins,
filename=filename,
extranames=filenames,
anon=options.anon,
grouped=newgrouped,
)
else:
# Copy file instead of run as it was already existing
LOG.debug("Copying old file from %s to %s" %
(runautofile, filename))
shutil.copyfile(runautofile, filename)
processedgroups[filename] = groups[target]
if todel:
# We can remove a sosreport from the dataset
for plugin in grouped:
if todel in grouped[plugin]["sosreport"]:
del grouped[plugin]["sosreport"][todel]
del newgroups[target]
# Put remaining groups to work
groups = dict(newgroups)
del groups
del processedgroups
print(_("\nFinished autogroup generation."))
def test_execonshellfailure(self):
returncode, out, err = citellus.execonshell('/proc/cmdline')
assert returncode == 3
def main():
"""
Main code stub
"""
start_time = time.time()
options = parse_args()
# Configure logging
logging.basicConfig(level=options.loglevel)
if not options.quiet:
show_logo()
# Each argument in sosreport is a sosreport
magplugs, magtriggers = initPlugins(options)
if options.list_plugins:
for plugin in magplugs:
print("-", plugin.__name__.split(".")[-1])
if options.description:
desc = plugin.help()
if desc:
print(citellus.indent(text=desc, amount=4))
return
# Prefill enabled citellus plugins from args
if not citellus.extensions:
extensions, exttriggers = citellus.initExtensions()
else:
extensions = citellus.extensions
citellusplugins = []
for extension in extensions:
citellusplugins.extend(extension.listplugins(options))
global allplugins
allplugins = citellusplugins
# By default, flatten plugin list for all extensions
newplugins = []
for each in citellusplugins:
newplugins.extend(each)
citellusplugins = newplugins
# Grab the data
sosreports = options.sosreports
if options.hosts:
ansible = citellus.which("ansible-playbook")
if not ansible:
LOG.err("No ansible-playbook support found, skipping")
else:
LOG.info("Grabbing data from remote hosts with Ansible")
# Grab data from ansible hosts
# Disable Ansible retry files creation:
os.environ['ANSIBLE_RETRY_FILES_ENABLED'] = "0"
if options.loglevel == 'DEBUG':
# Keep ansible remote files for debug
os.environ['ANSIBLE_KEEP_REMOTE_FILES'] = "1"
command = "%s -i %s %s" % (ansible, options.hosts,
os.path.join(maguidir, 'remote.yml'))
LOG.debug("Running: %s " % command)
citellus.execonshell(filename=command)
# Now check the hosts we got logs from:
hosts = citellus.findplugins(
folders=glob.glob('/tmp/citellus/hostrun/*'),
executables=False,
fileextension='.json')
for host in hosts:
sosreports.append(os.path.dirname(host['plugin']))
grouped = domagui(sosreports=sosreports,
citellusplugins=citellusplugins,
options=options)
# Run Magui plugins
result = []
for plugin in magplugs:
start_time = time.time()
# Get output from plugin
data = filterresults(
data=grouped, triggers=magtriggers[plugin.__name__.split(".")[-1]])
returncode, out, err = plugin.run(data=data, quiet=options.quiet)
updates = {'rc': returncode, 'out': out, 'err': err}
adddata = True
if options.quiet:
if returncode in [citellus.RC_OKAY, citellus.RC_SKIPPED]:
adddata = False
subcategory = os.path.split(plugin.__file__)[0].replace(
os.path.join(maguidir, 'plugins', ''), '')
if subcategory:
if len(os.path.normpath(subcategory).split(os.sep)) > 1:
category = os.path.normpath(subcategory).split(os.sep)[0]
else:
category = subcategory
subcategory = ""
else:
category = ""
if adddata:
result.append({
'plugin':
plugin.__name__.split(".")[-1],
'id':
hashlib.md5(
plugin.__file__.replace(maguidir,
'').encode('UTF-8')).hexdigest(),
'description':
plugin.help(),
'result':
updates,
'time':
time.time() - start_time,
'category':
category,
'subcategory':
subcategory
})
if options.output:
citellus.write_results(results=result,
filename=options.output,
source='magui',
path=sosreports,
time=time.time() - start_time)
pprint.pprint(result, width=1)
def main():
"""
Main code stub
"""
start_time = time.time()
options = parse_args()
# Configure ENV language before anything else
os.environ['LANG'] = "%s" % options.lang
# Reinstall language in case it has changed
trad = gettext.translation('citellus', localedir, fallback=True, languages=[options.lang])
try:
_ = trad.ugettext
except AttributeError:
_ = trad.gettext
# Configure logging
logging.basicConfig(level=options.loglevel)
if not options.quiet:
show_logo()
# Each argument in sosreport is a sosreport
magplugs, magtriggers = initPlugins(options)
if options.list_plugins:
for plugin in magplugs:
print("-", plugin.__name__.split(".")[-1])
if options.description:
desc = plugin.help()
if desc:
print(citellus.indent(text=desc, amount=4))
return
# Prefill enabled citellus plugins from args
if not citellus.extensions:
extensions, exttriggers = citellus.initExtensions()
else:
extensions = citellus.extensions
# Grab the data
sosreports = options.sosreports
# If we've provided a hosts file, use ansible to grab the data from them
if options.hosts:
ansible = citellus.which("ansible-playbook")
if not ansible:
LOG.err(_("No ansible-playbook support found, skipping"))
else:
LOG.info("Grabbing data from remote hosts with Ansible")
# Grab data from ansible hosts
# Disable Ansible retry files creation:
os.environ['ANSIBLE_RETRY_FILES_ENABLED'] = "0"
if options.loglevel == 'DEBUG':
# Keep ansible remote files for debug
os.environ['ANSIBLE_KEEP_REMOTE_FILES'] = "1"
command = "%s -i %s %s" % (ansible, options.hosts, os.path.join(maguidir, 'remote.yml'))
LOG.debug("Running: %s " % command)
citellus.execonshell(filename=command)
# Now check the hosts we got logs from:
hosts = citellus.findplugins(folders=glob.glob('/tmp/citellus/hostrun/*'), executables=False, fileextension='.json')
for host in hosts:
sosreports.append(os.path.dirname(host['plugin']))
# Get all data from hosts for all plugins, etc
if options.output:
citellusplugins = []
# Prefill with all available plugins and the ones we want to filter for
for extension in extensions:
citellusplugins.extend(extension.listplugins())
global allplugins
allplugins = citellusplugins
# By default, flatten plugin list for all extensions
newplugins = []
for each in citellusplugins:
newplugins.extend(each)
citellusplugins = newplugins
def runmaguiandplugs(sosreports, citellusplugins, filename=options.output, extranames=None):
"""
Runs magui and magui plugins
:param sosreports: sosreports to process
:param citellusplugins: citellusplugins to run
:param filename: filename to save to
:param extranames: additional filenames used
:return: results of execution
"""
# Run with all plugins so that we get all data back
grouped = domagui(sosreports=sosreports, citellusplugins=citellusplugins)
# Run Magui plugins
result = []
for plugin in magplugs:
start_time = time.time()
# Get output from plugin
data = filterresults(data=grouped, triggers=magtriggers[plugin.__name__.split(".")[-1]])
returncode, out, err = plugin.run(data=data, quiet=options.quiet)
updates = {'rc': returncode,
'out': out,
'err': err}
subcategory = os.path.split(plugin.__file__)[0].replace(os.path.join(maguidir, 'plugins', ''), '')
if subcategory:
if len(os.path.normpath(subcategory).split(os.sep)) > 1:
category = os.path.normpath(subcategory).split(os.sep)[0]
else:
category = subcategory
subcategory = ""
else:
category = ""
mydata = {'plugin': plugin.__name__.split(".")[-1],
'name': "magui: %s" % os.path.basename(plugin.__name__.split(".")[-1]),
'id': hashlib.md5(plugin.__file__.replace(maguidir, '').encode('UTF-8')).hexdigest(),
'description': plugin.help(),
'long_name': plugin.help(),
'result': updates,
'time': time.time() - start_time,
'category': category,
'subcategory': subcategory}
result.append(mydata)
branding = _(" ")
citellus.write_results(results=result, filename=filename, source='magui', path=sosreports, time=time.time() - start_time, branding=branding, web=True, extranames=extranames)
return result
results = runmaguiandplugs(sosreports=sosreports, citellusplugins=citellusplugins, filename=options.output)
# Now we've Magui saved for the whole execution provided in 'results' var
# Start working on autogroups
for result in results:
if result['plugin'] == 'metadata-outputs':
autodata = result['result']['err']
print(_("Running magui for autogroups:\n"))
groups = autogroups(autodata)
processedgroups = {}
filenames = []
for group in groups:
basefilename = os.path.splitext(options.output)
filename = basefilename[0] + "-" + group + basefilename[1]
print(filename)
runautogroup = True
for progroup in processedgroups:
if groups[group] == processedgroups[progroup]:
runautogroup = False
runautofile = progroup
if runautogroup:
# Analisys was missing for this group, run
runmaguiandplugs(sosreports=groups[group], citellusplugins=citellusplugins, filename=filename, extranames=options.output)
filenames.append(filename)
else:
# Copy file instead of run as it was already existing
LOG.debug("Copying old file from %s to %s" % (runautofile, filename))
shutil.copyfile(runautofile, filename)
processedgroups[filename] = groups[group]
print(_("\nFinished autogroup generation."))
if len(filenames) > 0:
# We've written additional files, so save again magui.json with additional references
# TODO: Intead of running magui and plugins again (should be fast, but not 'smart', we should save the json with the extra data.)
# As we've the extra data writing inside the function we might have to rewrite several steps so we went the code-reuse path
results = runmaguiandplugs(sosreports=sosreports, citellusplugins=citellusplugins, filename=options.output, extranames=filenames)
# Here preprocess output to use filtering, etc
# "result" does contain all data for both all citellus plugins and all magui plugins, need to filter for output on CLI only
# As we don't have a proper place to store output and we're running the full set of tests only when output is going
# to be stored (and then, the screen output is based on the already cached citellus results), it's probably not worth at this point to change this
citellusplugins = []
# Prefill with all available plugins and the ones we want to filter for
for extension in extensions:
citellusplugins.extend(extension.listplugins(options))
allplugins = citellusplugins
# By default, flatten plugin list for all extensions
newplugins = []
for each in citellusplugins:
newplugins.extend(each)
citellusplugins = newplugins
# Run with only the enabled plugins so that we get all data back for printing on console
grouped = domagui(sosreports=sosreports, citellusplugins=citellusplugins, options=options)
# Run Magui plugins
result = []
for plugin in magplugs:
start_time = time.time()
# Get output from plugin
data = filterresults(data=grouped, triggers=magtriggers[plugin.__name__.split(".")[-1]])
returncode, out, err = plugin.run(data=data, quiet=options.quiet)
updates = {'rc': returncode,
'out': out,
'err': err}
adddata = True
if options.quiet:
if returncode in [citellus.RC_OKAY, citellus.RC_SKIPPED]:
adddata = False
if adddata:
# If RC is to be stored, process further
subcategory = os.path.split(plugin.__file__)[0].replace(os.path.join(maguidir, 'plugins', ''), '')
if subcategory:
if len(os.path.normpath(subcategory).split(os.sep)) > 1:
category = os.path.normpath(subcategory).split(os.sep)[0]
else:
category = subcategory
subcategory = ""
else:
category = ""
mydata = {'plugin': plugin.__name__.split(".")[-1],
'id': hashlib.md5(plugin.__file__.replace(maguidir, '').encode('UTF-8')).hexdigest(),
'description': plugin.help(),
'result': updates,
'time': time.time() - start_time,
'category': category,
'subcategory': subcategory}
result.append(mydata)
pprint.pprint(result, width=1)
def main():
"""
Main code stub
"""
start_time = time.time()
options = parse_args()
# Configure ENV language before anything else
os.environ['LANG'] = "%s" % options.lang
# Reinstall language in case it has changed
trad = gettext.translation('citellus',
localedir,
fallback=True,
languages=[options.lang])
try:
_ = trad.ugettext
except AttributeError:
_ = trad.gettext
# Configure logging
logging.basicConfig(level=options.loglevel)
if not options.quiet:
show_logo()
# Each argument in sosreport is a sosreport
magplugs, magtriggers = initPlugins(options)
if options.list_plugins:
for plugin in magplugs:
print("-", plugin.__name__.split(".")[-1])
if options.description:
desc = plugin.help()
if desc:
print(citellus.indent(text=desc, amount=4))
return
# Prefill enabled citellus plugins from args
if not citellus.extensions:
extensions, exttriggers = citellus.initExtensions()
else:
extensions = citellus.extensions
# Grab the data
sosreports = options.sosreports
if options.hosts:
ansible = citellus.which("ansible-playbook")
if not ansible:
LOG.err(_("No ansible-playbook support found, skipping"))
else:
LOG.info("Grabbing data from remote hosts with Ansible")
# Grab data from ansible hosts
# Disable Ansible retry files creation:
os.environ['ANSIBLE_RETRY_FILES_ENABLED'] = "0"
if options.loglevel == 'DEBUG':
# Keep ansible remote files for debug
os.environ['ANSIBLE_KEEP_REMOTE_FILES'] = "1"
command = "%s -i %s %s" % (ansible, options.hosts,
os.path.join(maguidir, 'remote.yml'))
LOG.debug("Running: %s " % command)
citellus.execonshell(filename=command)
# Now check the hosts we got logs from:
hosts = citellus.findplugins(
folders=glob.glob('/tmp/citellus/hostrun/*'),
executables=False,
fileextension='.json')
for host in hosts:
sosreports.append(os.path.dirname(host['plugin']))
# Get all data from hosts for all plugins, etc
if options.output:
citellusplugins = []
# Prefill with all available plugins and the ones we want to filter for
for extension in extensions:
citellusplugins.extend(extension.listplugins())
global allplugins
allplugins = citellusplugins
# By default, flatten plugin list for all extensions
newplugins = []
for each in citellusplugins:
newplugins.extend(each)
citellusplugins = newplugins
# Run with all plugins so that we get all data back
grouped = domagui(sosreports=sosreports,
citellusplugins=citellusplugins)
# Run Magui plugins
result = []
for plugin in magplugs:
start_time = time.time()
# Get output from plugin
data = filterresults(
data=grouped,
triggers=magtriggers[plugin.__name__.split(".")[-1]])
returncode, out, err = plugin.run(data=data, quiet=options.quiet)
updates = {'rc': returncode, 'out': out, 'err': err}
subcategory = os.path.split(plugin.__file__)[0].replace(
os.path.join(maguidir, 'plugins', ''), '')
if subcategory:
if len(os.path.normpath(subcategory).split(os.sep)) > 1:
category = os.path.normpath(subcategory).split(os.sep)[0]
else:
category = subcategory
subcategory = ""
else:
category = ""
mydata = {
'plugin':
plugin.__name__.split(".")[-1],
'id':
hashlib.md5(
plugin.__file__.replace(maguidir,
'').encode('UTF-8')).hexdigest(),
'description':
plugin.help(),
'result':
updates,
'time':
time.time() - start_time,
'category':
category,
'subcategory':
subcategory
}
result.append(mydata)
branding = _(" ")
citellus.write_results(results=result,
filename=options.output,
source='magui',
path=sosreports,
time=time.time() - start_time,
branding=branding,
web=True)
# Here preprocess output to use filtering, etc
# "result" does contain all data for both all citellus plugins and all magui plugins, need to filter for output on CLI only
# As we don't have a proper place to store output and we're running the full set of tests only when output is going
# to be stored (and then, the screen output is based on the already cached citellus results), it's probably not worth at this point to change this
citellusplugins = []
# Prefill with all available plugins and the ones we want to filter for
for extension in extensions:
citellusplugins.extend(extension.listplugins(options))
global allplugins
allplugins = citellusplugins
# By default, flatten plugin list for all extensions
newplugins = []
for each in citellusplugins:
newplugins.extend(each)
citellusplugins = newplugins
# Run with all plugins so that we get all data back
grouped = domagui(sosreports=sosreports,
citellusplugins=citellusplugins,
options=options)
# Run Magui plugins
result = []
for plugin in magplugs:
start_time = time.time()
# Get output from plugin
data = filterresults(
data=grouped, triggers=magtriggers[plugin.__name__.split(".")[-1]])
returncode, out, err = plugin.run(data=data, quiet=options.quiet)
updates = {'rc': returncode, 'out': out, 'err': err}
adddata = True
if options.quiet:
if returncode in [citellus.RC_OKAY, citellus.RC_SKIPPED]:
adddata = False
if adddata:
# If RC is to be stored, process further
subcategory = os.path.split(plugin.__file__)[0].replace(
os.path.join(maguidir, 'plugins', ''), '')
if subcategory:
if len(os.path.normpath(subcategory).split(os.sep)) > 1:
category = os.path.normpath(subcategory).split(os.sep)[0]
else:
category = subcategory
subcategory = ""
else:
category = ""
mydata = {
'plugin':
plugin.__name__.split(".")[-1],
'id':
hashlib.md5(
plugin.__file__.replace(maguidir,
'').encode('UTF-8')).hexdigest(),
'description':
plugin.help(),
'result':
updates,
'time':
time.time() - start_time,
'category':
category,
'subcategory':
subcategory
}
result.append(mydata)
pprint.pprint(result, width=1)
